diff --git a/.gitignore b/.gitignore index f56e44c3b03efd1c1274d3c9b34922f967cb0e1a..cc28223ecff86188d7534e38a8d85da366b4ff88 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,8 @@ build.ninja ### copied files -functionsystem/src/common/proto/* \ No newline at end of file +functionsystem/src/common/proto/* + +### build files +common/litebus/build +functionsystem/build \ No newline at end of file diff --git a/functionsystem/apps/meta_service/cmd/main.go b/functionsystem/apps/meta_service/cmd/main.go new file mode 100644 index 0000000000000000000000000000000000000000..5e3138f016e926394006619cfa1a94427b2e4c6b --- /dev/null +++ b/functionsystem/apps/meta_service/cmd/main.go @@ -0,0 +1,138 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// package main start of meta-service +package main + +import ( + "flag" + "fmt" + "os" + + "meta_service/common/logger/log" + "meta_service/common/signals" + "meta_service/common/versions" + "meta_service/etcd" + "meta_service/function_repo/config" + "meta_service/function_repo/storage" + "meta_service/router" +) + +const ( + defaultListenIP = "0.0.0.0" + defaultFileName = "meta-service" +) + +func initLogConfig(logFile string) error { + if err := log.InitRunLogByParam(logFile, defaultFileName); err != nil { + fmt.Printf("failed to init run logger, logConfigPath: %s, error: %s\n", logFile, err.Error()) + return err + } + return nil +} + +func initRouterStorage() error { + etcdclient, err := etcd.NewClient() + if err != nil { + fmt.Printf("failed to new etcd client, error: %s", err.Error()) + return err + } + + err = storage.InitStorage(etcdclient) + if err != nil { + fmt.Printf("failed to init storage, error: %s", err.Error()) + return err + } + return nil +} + +func initMetaStorage() error { + if !config.RepoCfg.MetaEtcdEnable { + return nil + } + metaEtcdClient, err := etcd.NewMetaClient() + if err != nil { + log.GetLogger().Errorf("failed to new mete etcd client, error: %s", err.Error()) + return err + } + + err = storage.InitMetaStorage(metaEtcdClient) + if err != nil { + log.GetLogger().Errorf("failed to init storage, error: %s", err.Error()) + return err + } + return nil +} + +func initStorage() error { + err := initRouterStorage() + if err != nil { + fmt.Printf("failed to init router storage, error: %s", err.Error()) + return err + } + err = initMetaStorage() + if err != nil { + fmt.Printf("failed to init meta storage, error: %s", err.Error()) + return err + } + return nil +} + +func initMetaService(metaServiceConfigFile, logConfigFile string) bool { + if err := initLogConfig(logConfigFile); err != nil { + return false + } + if err := config.InitConfig(metaServiceConfigFile); err != nil { + fmt.Printf("failed to initialize config, error:%s\n", err.Error()) + return false + } + if err := initStorage(); err != nil { + fmt.Printf("failed to initialize storage, error:%s\n", err.Error()) + return false + } + return true +} + +func run() { + stopCh := signals.WaitForSignal() + fmt.Printf("version: %s branch: %s commit_id: %s\n", versions.GetBuildVersion(), versions.GetGitBranch(), + versions.GetGitHash()) + ip := config.RepoCfg.ServerCfg.IP + if len(ip) == 0 { + ip = os.Getenv("POD_IP") + fmt.Printf("failed to get pod ip from config, try to use %s as listen IP", ip) + } + if len(ip) == 0 { + ip = defaultListenIP + fmt.Printf("failed to get pod ip from env POD_IP, try to use %s as listen IP", ip) + } + addr := fmt.Sprintf("%s:%d", ip, config.RepoCfg.ServerCfg.Port) + fmt.Printf("starting router at %s ...\n", addr) + + router.Run(addr, stopCh) +} + +func main() { + fmt.Print("starting...\n") + configFile := flag.String("config_path", "/home/yuanrong/config/config.json", "configuration path") + logConfigFile := flag.String("log_config_path", "/home/yuanrong/config/log.json", "log configuration path") + flag.Parse() + + if !initMetaService(*configFile, *logConfigFile) { + return + } + + run() +} diff --git a/functionsystem/apps/meta_service/common/base/base_model.go b/functionsystem/apps/meta_service/common/base/base_model.go new file mode 100644 index 0000000000000000000000000000000000000000..d4a4e06f9273739acc17f6b1cd14c0ed3f306a05 --- /dev/null +++ b/functionsystem/apps/meta_service/common/base/base_model.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package base provides base model. +package base + +// FunctionCommonInfo function common info +type FunctionCommonInfo struct { + ID string `json:"id"` + CreateTime string `json:"createTime"` + UpdateTime string `json:"updateTime"` + FunctionURN string `json:"functionUrn"` + FunctionName string `json:"name"` + TenantID string `json:"tenantId"` + BusinessID string `json:"businessId"` + ProductID string `json:"productId"` + ReversedConcurrency int `json:"reversedConcurrency"` + Description string `json:"description"` + LastModified string `json:"lastModified"` + Published string `json:"Published"` + MinInstance int `json:"minInstance"` + MaxInstance int `json:"maxInstance"` + ConcurrentNum int `json:"concurrentNum"` + Status string `json:"status"` + InstanceNum int `json:"instanceNum"` + Tag map[string]string `json:"tag"` + FunctionVersionURN string `json:"functionVersionUrn"` + RevisionID string `json:"revisionId"` + CodeSize int64 `json:"codeSize"` + CodeSha256 string `json:"codeSha256"` + BucketID string `json:"bucketId"` + ObjectID string `json:"objectId"` + Handler string `json:"handler"` + Layers []string `json:"layers"` + CPU int `json:"cpu"` + Memory int `json:"memory"` + Runtime string `json:"runtime"` + Timeout int `json:"timeout"` + VersionNumber string `json:"versionNumber"` + VersionDesc string `json:"versionDesc"` + Environment map[string]string `json:"environment"` + CustomResources map[string]float64 `json:"customResources"` + StatefulFlag int `json:"statefulFlag"` +} diff --git a/functionsystem/apps/meta_service/common/codec/interface.go b/functionsystem/apps/meta_service/common/codec/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..77e271beb5768c2b426e100eb02071eddc147456 --- /dev/null +++ b/functionsystem/apps/meta_service/common/codec/interface.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package codec contains generic methods encoding and decoding storage keys and values. +package codec + +// Codec encode and decode types into keys and values. +type Codec interface { + Encode(v interface{}) (string, error) + Decode(data string, v interface{}) error +} diff --git a/functionsystem/apps/meta_service/common/codec/json.go b/functionsystem/apps/meta_service/common/codec/json.go new file mode 100644 index 0000000000000000000000000000000000000000..9242461dc141b47a09cb23a2a00837288882a9f7 --- /dev/null +++ b/functionsystem/apps/meta_service/common/codec/json.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package codec + +import ( + "encoding/json" +) + +type jsonC struct{} + +// NewJSONCodec encodes and decodes types to/from JSON strings. +func NewJSONCodec() Codec { + return jsonC{} +} + +// Encode implements Codec +func (c jsonC) Encode(v interface{}) (string, error) { + b, err := json.Marshal(v) + if err != nil { + return "", err + } + return string(b), nil +} + +// Decode implements Codec +func (c jsonC) Decode(data string, v interface{}) error { + return json.Unmarshal([]byte(data), v) +} diff --git a/functionsystem/apps/meta_service/common/codec/reflect.go b/functionsystem/apps/meta_service/common/codec/reflect.go new file mode 100644 index 0000000000000000000000000000000000000000..05eb5559311e4a2d19c5da6cab1b03c63ddb8a38 --- /dev/null +++ b/functionsystem/apps/meta_service/common/codec/reflect.go @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package codec + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" +) + +const ( + reflectSeperator = "/" +) + +var ( + reflectEncodeIntFunc = func(field reflect.Value) string { + if field.Int() != 0 { + return formatInt(field.Int()) + } + return "" + } + reflectEncodeUintFunc = func(field reflect.Value) string { + if field.Uint() != 0 { + return formatUint(field.Uint()) + } + return "" + } + reflecteEncodeMap = map[reflect.Kind]func(field reflect.Value) string{ + reflect.String: func(field reflect.Value) string { return field.String() }, + reflect.Bool: func(field reflect.Value) string { return formatBool(field.Bool()) }, + reflect.Int: reflectEncodeIntFunc, + reflect.Int64: reflectEncodeIntFunc, + reflect.Int32: reflectEncodeIntFunc, + reflect.Int16: reflectEncodeIntFunc, + reflect.Int8: reflectEncodeIntFunc, + reflect.Uint: reflectEncodeUintFunc, + reflect.Uint64: reflectEncodeUintFunc, + reflect.Uint32: reflectEncodeUintFunc, + reflect.Uint16: reflectEncodeUintFunc, + reflect.Uint8: reflectEncodeUintFunc, + } + reflectDecodeIntFunc = func(field *reflect.Value, s string) error { + if s == "" { + return nil + } + num, err := parseInt(s) + if err != nil { + return fmt.Errorf("failed to parse field %s to int, %s", field.Type().Name(), err.Error()) + } + field.SetInt(num) + return nil + } + reflectDecodeUintFunc = func(field *reflect.Value, s string) error { + if s == "" { + return nil + } + num, err := parseUint(s) + if err != nil { + return fmt.Errorf("failed to parse field %s to uint, %s", field.Type().Name(), err.Error()) + } + field.SetUint(num) + return nil + } + reflectDecodeMap = map[reflect.Kind]func(field *reflect.Value, s string) error{ + reflect.String: func(field *reflect.Value, s string) error { + field.SetString(s) + return nil + }, + reflect.Bool: func(field *reflect.Value, s string) error { + b, err := parseBool(s) + if err != nil { + return fmt.Errorf("failed to parse field %s to bool, %s", field.Type().Name(), err.Error()) + } + field.SetBool(b) + return nil + }, + reflect.Int: reflectDecodeIntFunc, + reflect.Int64: reflectDecodeIntFunc, + reflect.Int32: reflectDecodeIntFunc, + reflect.Int16: reflectDecodeIntFunc, + reflect.Int8: reflectDecodeIntFunc, + reflect.Uint: reflectDecodeUintFunc, + reflect.Uint64: reflectDecodeUintFunc, + reflect.Uint32: reflectDecodeUintFunc, + reflect.Uint16: reflectDecodeUintFunc, + reflect.Uint8: reflectDecodeUintFunc, + } +) + +type reflectC struct { + prefix string + hasSuffix bool +} + +// NewReflectCodec encodes and decodes types to/from "/" separated strings. +func NewReflectCodec(prefix string, hasSuffix bool) Codec { + return reflectC{prefix, hasSuffix} +} + +// Encode implements Codec +func (c reflectC) Encode(v interface{}) (string, error) { + val := reflect.ValueOf(v) + tokens, err := c.encode(val) + if err != nil { + return "", err + } + + var trim int + for j := len(tokens) - 1; j >= 0; j-- { + if tokens[j] == "" { + trim++ + } else { + break + } + } + tokens = append([]string{c.prefix}, tokens...) + if c.hasSuffix { + return strings.Join(tokens[:len(tokens)-trim], reflectSeperator) + reflectSeperator, nil + } else { + return strings.Join(tokens[:len(tokens)-trim], reflectSeperator), nil + } +} + +func (c reflectC) encode(val reflect.Value) ([]string, error) { + if val.Kind() != reflect.Struct { + return nil, errors.New("interface should be a struct") + } + + tokens := make([]string, 0, val.NumField()) + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + if field.Kind() == reflect.Struct { + nested, err := c.encode(field) + if err != nil { + return nil, err + } + tokens = append(tokens, nested...) + continue + } + fn, exist := reflecteEncodeMap[field.Kind()] + if !exist { + return nil, fmt.Errorf("unsupported type %v", field.Kind()) + } + tokens = append(tokens, fn(field)) + } + + return tokens, nil +} + +// Decode implements Codec +func (c reflectC) Decode(data string, v interface{}) error { + if v == nil { + return errors.New("interface should be a pointer of struct") + } + val := reflect.ValueOf(v) + if val.Kind() != reflect.Ptr { + return errors.New("interface should be a pointer of struct") + } + val = val.Elem() + if val.Kind() != reflect.Struct { + return errors.New("interface should be a pointer of struct") + } + if len(data) < len(c.prefix)+1 || !strings.HasPrefix(data, c.prefix+reflectSeperator) || + (c.hasSuffix && !strings.HasSuffix(data, reflectSeperator)) { + return fmt.Errorf("data %s is invalid", data) + } + tokens := strings.Split(data[len(c.prefix):], reflectSeperator) + if c.hasSuffix { + tokens = tokens[1 : len(tokens)-1] + } else { + tokens = tokens[1:len(tokens)] + } + pos := 0 + return c.decode(val, &pos, tokens) +} + +func (c reflectC) decode(val reflect.Value, pos *int, tokens []string) error { + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + if field.Kind() == reflect.Struct { + if err := c.decode(field, pos, tokens); err != nil { + return err + } + continue + } + fn, exist := reflectDecodeMap[field.Kind()] + if !exist { + return fmt.Errorf("unsupported type %v", field.Kind()) + } + if *pos >= len(tokens) { + break + } + if err := fn(&field, tokens[*pos]); err != nil { + return err + } + *pos++ + } + return nil +} + +func formatBool(b bool) string { + if b { + return "1" + } + return "0" +} + +func parseBool(s string) (bool, error) { + if s == "1" { + return true, nil + } + if s == "0" { + return false, nil + } + return false, fmt.Errorf("expect 0 or 1, got %s", s) +} + +// Strategy: Prepend a number with its length formatted to rune so it becomes sortable. +const ( + base = 'a' + nbase = 10 +) + +func formatInt(i int64) string { + s := strconv.FormatInt(i, nbase) + if i >= 0 { + return string(base+rune(len(s))) + s + } + return string(base-rune(len(s))) + s +} + +func parseInt(s string) (int64, error) { + if len(s) < 1 { + return 0, errors.New("expect at least 1 rune") + } + s = s[1:] + return strconv.ParseInt(s, nbase, 0) +} + +func formatUint(i uint64) string { + s := strconv.FormatUint(i, nbase) + return string(base+rune(len(s))) + s +} + +func parseUint(s string) (uint64, error) { + if len(s) < 1 { + return 0, errors.New("expect at least 1 rune") + } + s = s[1:] + return strconv.ParseUint(s, nbase, 0) +} diff --git a/functionsystem/apps/meta_service/common/codec/reflect_test.go b/functionsystem/apps/meta_service/common/codec/reflect_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3e75d4aa0993092025bc4ef8c6157bcf34c0ffba --- /dev/null +++ b/functionsystem/apps/meta_service/common/codec/reflect_test.go @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package codec + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReflectCodec(t *testing.T) { + codec := NewReflectCodec("prefix", true) + type Student struct { + Name string + Age int + Gender bool + } + + s, err := codec.Encode(Student{Name: "Alice", Age: 15}) + require.NoError(t, err) + assert.Equal(t, "prefix/Alice/c15/0/", s) + + var alice Student + err = codec.Decode(s, &alice) + require.NoError(t, err) + assert.Equal(t, "Alice", alice.Name) + assert.Equal(t, 15, alice.Age) + assert.Equal(t, false, alice.Gender) +} + +func TestReflectCodecTrim(t *testing.T) { + codec := NewReflectCodec("prefix", true) + type Student struct { + Name string + Age int + Phone string + } + + { + s, err := codec.Encode(Student{Name: "Alice", Age: 15}) + require.NoError(t, err) + assert.Equal(t, "prefix/Alice/c15/", s) + + var alice Student + err = codec.Decode(s, &alice) + require.NoError(t, err) + assert.Equal(t, "Alice", alice.Name) + assert.Equal(t, 15, alice.Age) + assert.Equal(t, "", alice.Phone) + } + { + s, err := codec.Encode(Student{Name: "Alice"}) + require.NoError(t, err) + assert.Equal(t, "prefix/Alice/", s) + + var alice Student + err = codec.Decode(s, &alice) + require.NoError(t, err) + assert.Equal(t, "Alice", alice.Name) + assert.Equal(t, 0, alice.Age) + assert.Equal(t, "", alice.Phone) + } +} + +func TestReflectCodecNil(t *testing.T) { + codec := NewReflectCodec("prefix", true) + type Student struct { + Name string + Age int + } + + s, err := codec.Encode(Student{Age: 15}) + require.NoError(t, err) + assert.Equal(t, "prefix//c15/", s) + + var alice Student + err = codec.Decode(s, &alice) + require.NoError(t, err) + assert.Equal(t, "", alice.Name) + assert.Equal(t, 15, alice.Age) +} + +func TestReflectCodecNested(t *testing.T) { + codec := NewReflectCodec("prefix", true) + type Address struct { + Street string + Code uint + } + type Student struct { + Name string + Age int + Address Address + } + + { + s, err := codec.Encode(Student{Name: "Alice", Age: 15, Address: Address{Street: "abc road", Code: 12345}}) + require.NoError(t, err) + assert.Equal(t, "prefix/Alice/c15/abc road/f12345/", s) + + var alice Student + err = codec.Decode(s, &alice) + require.NoError(t, err) + assert.Equal(t, "Alice", alice.Name) + assert.Equal(t, 15, alice.Age) + assert.Equal(t, "abc road", alice.Address.Street) + assert.Equal(t, uint(12345), alice.Address.Code) + } + { + s, err := codec.Encode(Student{Name: "Alice", Address: Address{Street: "abc road", Code: 12345}}) + require.NoError(t, err) + assert.Equal(t, "prefix/Alice//abc road/f12345/", s) + + var alice Student + err = codec.Decode(s, &alice) + require.NoError(t, err) + assert.Equal(t, "Alice", alice.Name) + assert.Equal(t, 0, alice.Age) + assert.Equal(t, "abc road", alice.Address.Street) + assert.Equal(t, uint(12345), alice.Address.Code) + } + { + s, err := codec.Encode(Student{Name: "Alice", Address: Address{Street: "abc road"}}) + require.NoError(t, err) + assert.Equal(t, "prefix/Alice//abc road/", s) + + var alice Student + err = codec.Decode(s, &alice) + require.NoError(t, err) + assert.Equal(t, "Alice", alice.Name) + assert.Equal(t, 0, alice.Age) + assert.Equal(t, "abc road", alice.Address.Street) + } +} diff --git a/functionsystem/apps/meta_service/common/constants/constant_test.go b/functionsystem/apps/meta_service/common/constants/constant_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1255fc8ba1c130a116925f873376d05829de308c --- /dev/null +++ b/functionsystem/apps/meta_service/common/constants/constant_test.go @@ -0,0 +1 @@ +package constants diff --git a/functionsystem/apps/meta_service/common/constants/constants.go b/functionsystem/apps/meta_service/common/constants/constants.go new file mode 100644 index 0000000000000000000000000000000000000000..b09d3255163641a34f86bbb59bba00d08c1393db --- /dev/null +++ b/functionsystem/apps/meta_service/common/constants/constants.go @@ -0,0 +1,453 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constants implements vars of all +package constants + +import ( + "os" + "strconv" + "time" +) + +const ( + // ZoneKey zone key + ZoneKey = "KUBERNETES_IO_AVAILABLEZONE" + // ZoneNameLen define zone length + ZoneNameLen = 255 + // DefaultAZ default az + DefaultAZ = "defaultaz" + + // PodIPEnvKey define pod ip env key + PodIPEnvKey = "POD_IP" + + // HostNameEnvKey defines the hostname env key + HostNameEnvKey = "HOSTNAME" + + // NodeID defines the node name env key + NodeID = "NODE_ID" + + // HostIPEnvKey defines the host ip env key + HostIPEnvKey = "HOST_IP" + + // PodNamespaceEnvKey define pod namespace env key + PodNamespaceEnvKey = "POD_NAMESPACE" + + // ResourceLimitsMemory Memory limit, in bytes + ResourceLimitsMemory = "MEMORY_LIMIT_BYTES" + + // ResourceLimitsCPU CPU limit, in m(1/1000) + ResourceLimitsCPU = "CPU_LIMIT" + + // FuncBranchEnvKey is branch env key + FuncBranchEnvKey = "FUNC_BRANCH" + + // DataSystemBranchEnvKey is branch env key + DataSystemBranchEnvKey = "DATASYSTEM_CAPABILITY" + + // HTTPort busproxy httpserver listen port + HTTPort = "22423" + // GRPCPort busproxy gRPCserver listen port + GRPCPort = "22769" + // WorkerAgentPort is the listen port of worker agent grpc server + WorkerAgentPort = "22888" + // DataSystemPort is the port of data system + DataSystemPort = "31501" + // LocalSchedulerPort is the listen port string of local scheduler grpc server + LocalSchedulerPort = GRPCPort + // DomainSchedulerPort is the listen port of domain scheduler grpc server + DomainSchedulerPort = 22771 + // MaxPort maximum number of ports + MaxPort = 65535 + // SchedulerAddressSeparator is the separator of domain scheduler address + SchedulerAddressSeparator = ":" + // PlatformTenantID is tenant ID of platform function + PlatformTenantID = "0" + + // RuntimeLogOptTail - + RuntimeLogOptTail = "Tail" + // RuntimeLayerDirName - + RuntimeLayerDirName = "layer" + // RuntimeFuncDirName - + RuntimeFuncDirName = "func" + + // FunctionTaskAppID - + FunctionTaskAppID = "function-task" + + // TenantID config from function task + TenantID = "0" + + // BackpressureCode indicate that frontend should choose another proxy/worker and retry + BackpressureCode = 211429 + // HeaderBackpressure indicate that proxy can backpressure this request + HeaderBackpressure = "X-Backpressure" + + // SrcInstanceID gRPC context of metadata + SrcInstanceID = "src_instance_id" + // ReturnObjID gRPC context of metadata + ReturnObjID = "return_obj_id" + + // DelWorkerAgentEvent delete workerAgent + DelWorkerAgentEvent = "WorkerAgent-Del" + // UpdWorkerAgentEvent update workerAgent + UpdWorkerAgentEvent = "WorkerAgent-Upd" + + // DefaultLatestVersion is default function name + DefaultLatestVersion = "$latest" + // DefaultLatestFaaSVersion is default faas function name + DefaultLatestFaaSVersion = "latest" + // DefaultJavaRuntimeName is default java runtime name + DefaultJavaRuntimeName = "java1.8" + // DefaultJavaRuntimeNameForFaas is defualt + DefaultJavaRuntimeNameForFaas = "java8" +) + +// grpc parameters +const ( + // MaxMsgSize grpc client max message size(bit) + MaxMsgSize = 1024 * 1024 * 2 + // MaxWindowSize grpc flow control window size(bit) + MaxWindowSize = 1024 * 1024 * 2 + // MaxBufferSize grpc read/write buffer size(bit) + MaxBufferSize = 1024 * 1024 * 2 +) + +// functionBus userData key flag +const ( + // FrontendCallFlag invoke from task + FrontendCallFlag = "FrontendCallFlag" +) + +const ( + // DynamicRouterParamPrefix 动态路由参数前缀 + DynamicRouterParamPrefix = "/:" +) + +// HTTP invoke request header key +const ( + // HeaderExecutedDuration - + HeaderExecutedDuration = "X-Executed-Duration" + // HeaderTraceID - + HeaderTraceID = "X-Trace-Id" + // HeaderEventSourceID - + HeaderEventSourceID = "X-Event-Source-Id" + // HeaderBusinessID - + HeaderBusinessID = "X-Business-ID" + // HeaderTenantID - + HeaderTenantID = "X-Tenant-ID" + // HeaderTenantId - + HeaderTenantId = "X-Tenant-Id" + // HeaderPoolLabel - + HeaderPoolLabel = "X-Pool-Label" + // HeaderLogType - + HeaderLogType = "X-Log-Type" + // HeaderLogResult - + HeaderLogResult = "X-Log-Result" + // HeaderTriggerFlag - + HeaderTriggerFlag = "X-Trigger-Flag" + // HeaderInnerCode - + HeaderInnerCode = "X-Inner-Code" + // HeaderInvokeURN - + HeaderInvokeURN = "X-Tag-VersionUrn" + // HeaderStateKey - + HeaderStateKey = "X-State-Key" + // HeaderCallType is the request type + HeaderCallType = "X-Call-Type" + // HeaderLoadDuration duration of loading function + HeaderLoadDuration = "X-Load-Duration" + // HeaderNodeLabel is node label + HeaderNodeLabel = "X-Node-Label" + // HeaderForceDeploy is Force Deploy + HeaderForceDeploy = "X-Force-Deploy" + // HeaderAuthorization is authorization + HeaderAuthorization = "authorization" + // HeaderFutureID is futureID of invocation + HeaderFutureID = "X-Future-ID" + // HeaderAsync indicate whether it is an async request + HeaderAsync = "X-ASYNC" + // HeaderRuntimeID represents runtime instance identification + HeaderRuntimeID = "X-Runtime-ID" + // HeaderRuntimePort represents runtime rpc port + HeaderRuntimePort = "X-Runtime-Port" + // HeaderCPUSize is cpu size specified by invoke + HeaderCPUSize = "X-Instance-CPU" + // HeaderMemorySize is cpu memory specified by invoke + HeaderMemorySize = "X-Instance-Memory" + HeaderFileDigest = "X-File-Digest" + HeaderProductID = "X-Product-Id" + HeaderPrivilege = "X-Privilege" + HeaderUserID = "X-User-Id" + HeaderVersion = "X-Version" + HeaderKind = "X-Kind" + // HeaderCompatibleRuntimes - + HeaderCompatibleRuntimes = "X-Header-Compatible-Runtimes" + // HeaderDescription - + HeaderDescription = "X-Description" + // HeaderLicenseInfo - + HeaderLicenseInfo = "X-License-Info" + // HeaderGroupID is group id + HeaderGroupID = "X-Group-ID" + // ApplicationJSON - + ApplicationJSON = "application/json" + // ContentType - + ContentType = "Content-Type" + // PriorityHeader - + PriorityHeader = "priority" + // HeaderDataContentType - + HeaderDataContentType = "X-Content-Type" + // ErrorDuration duration when error happened, + // used with key $LoadDuration + ErrorDuration = -1 +) + +// Extra Request Header +const ( + // HeaderRequestID - + HeaderRequestID = "x-request-id" + // HeaderAccessKey - + HeaderAccessKey = "x-access-key" + // HeaderSecretKey - + HeaderSecretKey = "x-secret-key" + // HeaderAuthToken - + HeaderAuthToken = "x-auth-token" + // HeaderSecurityToken - + HeaderSecurityToken = "x-security-token" + // HeaderStorageType code storage type + HeaderStorageType = "x-storage-type" +) + +const ( + // FunctionStatusUnavailable function status is unavailable + FunctionStatusUnavailable = "unavailable" + + // FunctionStatusAvailable function status is available + FunctionStatusAvailable = "available" +) + +const ( + // OndemandKey is used in ondemand scenario + OndemandKey = "ondemand" +) + +// stage +const ( + InitializeStage = "initialize" +) + +// default UIDs and GIDs +const ( + DefaultWorkerGID = 1002 + DefaultRuntimeUID = 1003 + DefaultRuntimeUName = "snuser" + DefaultRuntimeGID = 1003 +) + +const ( + // WorkerManagerApplier mark the instance is created by minInstance + WorkerManagerApplier = "worker-manager" +) + +const ( + DialBaseDelay = 300 * time.Millisecond + DialMultiplier = 1.2 + DialJitter = 0.1 + DialMaxDelay = 15 * time.Second + RuntimeDialMaxDelay = 100 * time.Second +) + +// constants of network connection +const ( + // DefaultConnectInterval is the default connect interval + DefaultConnectInterval = 3 * time.Second + // DefaultDialInterval is the default grpc dial request interval + DefaultDialInterval = 3 * time.Second + // DefaultRetryTimes is the default request retry times + DefaultRetryTimes = 3 + ConnectIntervalTime = 1 * time.Second +) + +// request message +const ( + // RequestCPU - + RequestCPU = "CPU" + // RequestMemory - + RequestMemory = "Memory" + // MinCustomResourcesSize is min gpu size of invoke + MinCustomResourcesSize = 0 + + // CpuUnitConvert - + CpuUnitConvert = 1000 + // MemoryUnitConvert - + MemoryUnitConvert = 1024 + + // minInvokeCPUSize is default min cpu size of invoke (One CPU core corresponds to 1000) + minInvokeCPUSize = 300 + // MaxInvokeCPUSize is max cpu size of invoke (One CPU core corresponds to 1000) + MaxInvokeCPUSize = 16000 + // minInvokeMemorySize is default min memory size of invoke (MB) + minInvokeMemorySize = 128 + // MaxInvokeMemorySize is max memory size of invoke (MB) + MaxInvokeMemorySize = 1024 * 1024 * 1024 + // InstanceConcurrency - + InstanceConcurrency = "Concurrency" + // DefaultMapSize default map size + DefaultMapSize = 2 + // DefaultSliceSize default slice size + DefaultSliceSize = 16 + // MaxUploadMemorySize is max memory size of upload (MB) + MaxUploadMemorySize = 10 * 1024 * 1024 + // S3StorageType the code is stored in the minio + S3StorageType = "s3" + // LocalStorageType the code is stored in the disk + LocalStorageType = "local" + // CopyStorageType the code is stored in the disk and need to copy to container path + CopyStorageType = "copy" + // Faas kind of function creation + Faas = "faas" +) + +// prefixes of ETCD keys +const ( + WorkerETCDKeyPrefix = "/sn/workeragent" + NodeETCDKeyPrefix = "/sn/node" + // InstanceETCDKeyPrefix is the prefix of etcd key for instance + InstanceETCDKeyPrefix = "/sn/instance" + // ResourceGroupETCDKeyPrefix is the prefix of etcd key for resource group + ResourceGroupETCDKeyPrefix = "/sn/resourcegroup" + // WorkersEtcdKeyPrefix is the prefix of etcd key for workers + WorkersEtcdKeyPrefix = "/sn/workers" + // AliasEtcdKeyPrefix is the key prefix of aliases in etcd + AliasEtcdKeyPrefix = "/sn/aliases" + // MetaFuncKey key used to match functions within ETCD + MetaFuncKey = "/sn/functions/business/yrk/tenant/%s/function/%s/version/%s" + // BusinessTypeServe - + BusinessTypeServe = "serve" +) + +// constants of posix custom runtime +const ( + PosixCustomRuntime = "posix-custom-runtime" + GORuntime = "go" + JavaRuntime = "java" + _ +) + +const ( + // OriginSchedulePolicy use origin scheduler policy + OriginSchedulePolicy = 0 + // NewSchedulePolicy use new scheduler policy + NewSchedulePolicy = 1 +) + +const ( + // LocalSchedulerLevel local scheduler level is 0 + LocalSchedulerLevel = iota + // LowDomainSchedulerLevel low domain scheduler level is 0 + LowDomainSchedulerLevel +) + +const ( + // Base10 is the decimal base number when use FormatInt + Base10 = 10 +) + +// MinInvokeCPUSize is min cpu size of invoke (One CPU core corresponds to 1000) +// Return default minInvokeCPUSize or system env[MinInvokeCPUSize] +var MinInvokeCPUSize = func() float64 { + minInvokeCPUSizeStr := os.Getenv("MinInvokeCPUSize") + if minInvokeCPUSizeStr != "" { + value, err := strconv.Atoi(minInvokeCPUSizeStr) + if err != nil { + return minInvokeCPUSize + } + return float64(value) + } + return minInvokeCPUSize +}() + +// MinInvokeMemorySize is min memory size of invoke (MB) +// Return default minInvokeMemorySize or system env[MinInvokeMemorySize] +var MinInvokeMemorySize = func() float64 { + minInvokeMemorySizeStr := os.Getenv("MinInvokeMemorySize") + if minInvokeMemorySizeStr != "" { + value, err := strconv.Atoi(minInvokeMemorySizeStr) + if err != nil { + return minInvokeMemorySize + } + return float64(value) + } + return minInvokeMemorySize +}() + +// SelfNodeIP - node IP +var SelfNodeIP = os.Getenv(HostIPEnvKey) + +// SelfNodeID - node ID +var SelfNodeID = os.Getenv(NodeID) + +// used for app-job +const ( + // UserMetadataKey key used for the app createOpts + UserMetadataKey = "USER_PROVIDED_METADATA" + // EntryPointKey entrypoint for starting app + EntryPointKey = "ENTRYPOINT" + // AppFN - + FunctionNameApp = "app" + // AppFuncId - + AppFuncId = "12345678901234561234567890123456/0-system-faasExecutorPosixCustom/$latest" + // AppType type for invoking create-app + AppType = "SUBMISSION" + // AppStatusPending - + AppStatusPending = "PENDING" + // AppStatusRunning - + AppStatusRunning = "RUNNING" + // AppStatusSucceeded - + AppStatusSucceeded = "SUCCEEDED" + // AppStatusFailed - + AppStatusFailed = "FAILED" + // AppStatusStopped - + AppStatusStopped = "STOPPED" + + // AppInvokeTimeout 30min + AppInvokeTimeout = 1800 +) + +// AppInfo - Ray job JobDetails +type AppInfo struct { + Key string `json:"key"` + // Enum: "SUBMISSION" "DRIVER" + Type string `json:"type"` + Entrypoint string `json:"entrypoint"` + SubmissionID string `json:"submission_id"` + DriverInfo DriverInfo `json:"driver_info" valid:",optional"` + // Status Enum: "PENDING" "RUNNING" "STOPPED" "SUCCEEDED" "FAILED" + Status string `json:"status" valid:",optional"` + StartTime string `json:"start_time" valid:",optional"` + EndTime string `json:"end_time" valid:",optional"` + Metadata map[string]string `json:"metadata" valid:",optional"` + RuntimeEnv map[string]interface{} `json:"runtime_env" valid:",optional"` + DriverAgentHttpAddress string `json:"driver_agent_http_address" valid:",optional"` + DriverNodeID string `json:"driver_node_id" valid:",optional"` + DriverExitCode int32 `json:"driver_exit_code" valid:",optional"` + ErrorType string `json:"error_type" valid:",optional"` +} + +// DriverInfo - +type DriverInfo struct { + ID string `json:"id"` + NodeIPAddress string `json:"node_ip_address"` + PID string `json:"pid"` +} diff --git a/functionsystem/apps/meta_service/common/engine/etcd/etcd.go b/functionsystem/apps/meta_service/common/engine/etcd/etcd.go new file mode 100644 index 0000000000000000000000000000000000000000..6efd44f755a4a6b5f48125809eb60bbd59be270a --- /dev/null +++ b/functionsystem/apps/meta_service/common/engine/etcd/etcd.go @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "context" + "fmt" + "time" + + "meta_service/common/engine" + "meta_service/common/logger/log" + + commonetcd "meta_service/common/etcd3" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// Config is the configuration with etcd +type Config struct { + Timeout time.Duration + TransactionTimeout time.Duration +} + +const ( + // DefaultTimeout is the default etcd execution timeout + DefaultTimeout = 40 * time.Second + // DefaultTransactionTimeout is the default etcd transaction timeout + DefaultTransactionTimeout = 40 * time.Second +) + +// DefaultConfig is default etcd engine configuration +var DefaultConfig = Config{ + Timeout: DefaultTimeout, + TransactionTimeout: DefaultTransactionTimeout, +} + +type etcdE struct { + cli *commonetcd.EtcdClient + cfg Config +} + +// NewEtcdEngine creates a new etcd engine +func NewEtcdEngine(cli *commonetcd.EtcdClient, cfg Config) engine.Engine { + return &etcdE{cli: cli, cfg: cfg} +} + +// Get implements engine.Engine +func (e *etcdE) Get(ctx context.Context, etcdKey string) (string, error) { + etcdCtxInfo := commonetcd.CreateEtcdCtxInfoWithTimeout(ctx, e.cfg.Timeout) + values, err := e.cli.GetValues(etcdCtxInfo, etcdKey) + if err != nil { + return "", err + } + if len(values) == 0 { + log.GetLogger().Debugf("get key: %s, key not found", etcdKey) + return "", engine.ErrKeyNotFound + } + + log.GetLogger().Debugf("get key: %s", etcdKey) + return values[0], nil +} + +// Count implements engine.Engine +func (e *etcdE) Count(ctx context.Context, prefix string) (int64, error) { + etcdCtxInfo := commonetcd.CreateEtcdCtxInfoWithTimeout(ctx, e.cfg.Timeout) + resp, err := e.cli.GetResponse(etcdCtxInfo, prefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + if err != nil { + return 0, err + } + log.GetLogger().Debugf("count prefix: %s, count: %s", prefix, resp.Count) + return resp.Count, nil +} + +func (e *etcdE) firstInRange(ctx context.Context, prefix string, last bool) (string, string, error) { + etcdCtxInfo := commonetcd.CreateEtcdCtxInfoWithTimeout(ctx, e.cfg.Timeout) + var opts []clientv3.OpOption + if last { + opts = clientv3.WithLastKey() + } else { + opts = clientv3.WithFirstKey() + } + + resp, err := e.cli.GetResponse(etcdCtxInfo, prefix, opts...) + if err != nil { + return "", "", err + } + if len(resp.Kvs) == 0 { + if last { + log.GetLogger().Debugf("last in range prefix: %s, key not found", prefix) + } else { + log.GetLogger().Debugf("first in range prefix: %s, key not found", prefix) + } + return "", "", engine.ErrKeyNotFound + } + + if last { + log.GetLogger().Debugf("last in range prefix: %s, key: %s", prefix, string(resp.Kvs[0].Key)) + } else { + log.GetLogger().Debugf("first in range prefix: %s, key: %s", prefix, string(resp.Kvs[0].Key)) + } + return e.cli.DetachAZPrefix(string(resp.Kvs[0].Key)), string(resp.Kvs[0].Value), nil +} + +// FirstInRange implements engine.Engine +func (e *etcdE) FirstInRange(ctx context.Context, prefix string) (string, string, error) { + return e.firstInRange(ctx, prefix, false) +} + +// LastInRange implements engine.Engine +func (e *etcdE) LastInRange(ctx context.Context, prefix string) (string, string, error) { + return e.firstInRange(ctx, prefix, true) +} + +// PrepareStream implements engine.Engine +func (e *etcdE) PrepareStream( + ctx context.Context, prefix string, decode engine.DecodeHandleFunc, by engine.SortBy, +) engine.PrepareStmt { + sortOp, err := e.genSortOpOption(by) + if err != nil { + return &etcdStmt{ + fn: func() ([]interface{}, error) { + return nil, err + }, + } + } + + fn := func() ([]interface{}, error) { + etcdCtxInfo := commonetcd.CreateEtcdCtxInfoWithTimeout(ctx, e.cfg.Timeout) + resp, err := e.cli.GetResponse(etcdCtxInfo, prefix, clientv3.WithPrefix(), sortOp) + if err != nil { + return nil, err + } + + log.GetLogger().Debugf("stream prefix: %s, resp num: %v", prefix, len(resp.Kvs)) + + var res []interface{} + for _, kv := range resp.Kvs { + key := e.cli.DetachAZPrefix(string(kv.Key)) + i, err := decode(key, string(kv.Value)) + if err != nil { + return nil, err + } + res = append(res, i) + } + return res, nil + } + + return &etcdStmt{ + fn: fn, + } +} + +func (e *etcdE) genSortOpOption(by engine.SortBy) (clientv3.OpOption, error) { + var ( + order clientv3.SortOrder + target clientv3.SortTarget + ) + switch by.Order { + case engine.Ascend: + order = clientv3.SortAscend + case engine.Descend: + order = clientv3.SortDescend + default: + return nil, fmt.Errorf("invalid sort order: %v", by.Order) + } + switch by.Target { + case engine.SortName: + target = clientv3.SortByKey + case engine.SortCreate: + target = clientv3.SortByCreateRevision + case engine.SortModify: + target = clientv3.SortByModRevision + default: + return nil, fmt.Errorf("invalid sort target: %v", by.Target) + } + sortOp := clientv3.WithSort(target, order) + return sortOp, nil +} + +// Put implements engine.Engine +func (e *etcdE) Put(ctx context.Context, etcdKey string, value string) error { + etcdCtxInfo := commonetcd.CreateEtcdCtxInfoWithTimeout(ctx, e.cfg.Timeout) + err := e.cli.Put(etcdCtxInfo, etcdKey, value) + if err != nil { + log.GetLogger().Debugf("put key: %s", etcdKey) + } + return err +} + +// Delete implements engine.Engine +func (e *etcdE) Delete(ctx context.Context, etcdKey string) error { + etcdCtxInfo := commonetcd.CreateEtcdCtxInfoWithTimeout(ctx, e.cfg.Timeout) + err := e.cli.Delete(etcdCtxInfo, etcdKey) + if err != nil { + log.GetLogger().Debugf("delete etcd key: %s", etcdKey) + } + return err +} + +// BeginTx implements engine.Engine +func (e *etcdE) BeginTx(ctx context.Context) engine.Transaction { + return newTransaction(ctx, e.cli, e.cfg.TransactionTimeout) +} + +// Close implements engine.Engine +func (e *etcdE) Close() error { + return e.cli.Client.Close() +} diff --git a/functionsystem/apps/meta_service/common/engine/etcd/stream.go b/functionsystem/apps/meta_service/common/engine/etcd/stream.go new file mode 100644 index 0000000000000000000000000000000000000000..128e27fddd335491cdb86a5f47b4ccc36b89c6c2 --- /dev/null +++ b/functionsystem/apps/meta_service/common/engine/etcd/stream.go @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd implements engine.Engine +package etcd + +import ( + "io" + + "meta_service/common/engine" +) + +type etcdStmt struct { + fn func() ([]interface{}, error) + filters []engine.FilterFunc +} + +// Filter implements engine.PrepareStmt +func (s *etcdStmt) Filter(filter engine.FilterFunc) engine.PrepareStmt { + s.filters = append(s.filters, filter) + return s +} + +// Execute implements engine.PrepareStmt +func (s *etcdStmt) Execute() (engine.Stream, error) { + vs, err := s.fn() + if err != nil { + return nil, err + } + + var res []interface{} +outer: + for _, v := range vs { + for _, filter := range s.filters { + if !filter(v) { + continue outer + } + } + res = append(res, v) + } + return &etcdStream{vs: res}, nil +} + +type etcdStream struct { + vs []interface{} + pos int +} + +// Next implements engine.Stream +func (s *etcdStream) Next() (interface{}, error) { + defer func() { + s.pos++ + }() + + if s.pos == len(s.vs) { + return nil, io.EOF + } + return s.vs[s.pos], nil +} diff --git a/functionsystem/apps/meta_service/common/engine/etcd/transaction.go b/functionsystem/apps/meta_service/common/engine/etcd/transaction.go new file mode 100644 index 0000000000000000000000000000000000000000..05e9013610aac7de7a56a92b105c1dcf8e8a78a7 --- /dev/null +++ b/functionsystem/apps/meta_service/common/engine/etcd/transaction.go @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "context" + "time" + + "meta_service/common/constants" + "meta_service/common/engine" + "meta_service/common/etcd3" + "meta_service/common/logger/log" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +type reads struct { + resp *clientv3.GetResponse + modRev int64 + withPrefix bool +} + +const ( + writeOp = iota + delOp +) + +type writes struct { + value string + op int + withPrefix bool +} + +// Transaction utilities etcd v3 transaction to perform transaction with separated expressions +type Transaction struct { + etcdClient *etcd3.EtcdClient + + rds map[string]*reads + wrs map[string]*writes + + ctx context.Context + cancel func() +} + +// newTransaction creates a new transaction object to records all the method that will be performed in it. +func newTransaction(ctx context.Context, client *etcd3.EtcdClient, timeout time.Duration) *Transaction { + ctx, cancel := context.WithTimeout(ctx, timeout) + t := Transaction{etcdClient: client, ctx: ctx, cancel: cancel} + t.rds = make(map[string]*reads, constants.DefaultMapSize) + t.wrs = make(map[string]*writes, constants.DefaultMapSize) + return &t +} + +// Put caches a key-value pair, it will be replaced if the same key has been called within the transaction. +func (t *Transaction) Put(key string, value string) { + log.GetLogger().Debugf("transaction put key: %s, value: %s", key, value) + t.wrs[key] = &writes{value, writeOp, false} +} + +func getRespMaxModRev(resp *clientv3.GetResponse) int64 { + var rev int64 = 0 + for _, kv := range resp.Kvs { + if kv.ModRevision > rev { + rev = kv.ModRevision + } + } + return rev +} + +// Get returns value of the key from etcd, and cached it to perform 'If' statement in the transaction. +// The method will return cached value if the same key has been called within the transaction. +func (t *Transaction) Get(key string) (string, error) { + if v, ok := t.wrs[key]; ok { + if v.op == writeOp { + log.GetLogger().Debugf("cached: transaction get key: %s, value: %s", key, v.value) + return v.value, nil + } + log.GetLogger().Debugf("cached: transaction get key: %s, value: %s, key not found", key) + return "", engine.ErrKeyNotFound + } + if v, ok := t.rds[key]; ok { + if len(v.resp.Kvs) == 1 { + return string(v.resp.Kvs[0].Value), nil + } + return "", engine.ErrKeyNotFound + } + + etcdCtxInfo := etcd3.CreateEtcdCtxInfo(t.ctx) + resp, err := t.etcdClient.GetResponse(etcdCtxInfo, key) + if err != nil { + log.GetLogger().Errorf("failed to get from etcd, error: %s", err.Error()) + return "", err + } + t.rds[key] = &reads{resp, getRespMaxModRev(resp), false} + if len(resp.Kvs) == 1 { + log.GetLogger().Debugf("transaction get key: %s, value: %s, modRevision: %d", + key, string(resp.Kvs[0].Value), resp.Kvs[0].ModRevision) + return string(resp.Kvs[0].Value), nil + } + log.GetLogger().Debugf("transaction get key: %s, value: %s, key not found", key) + return "", engine.ErrKeyNotFound +} + +func (t *Transaction) getRespKVs(resp *clientv3.GetResponse) (keys, values []string) { + for _, kv := range resp.Kvs { + key := t.etcdClient.DetachAZPrefix(string(kv.Key)) + keys = append(keys, key) + values = append(values, string(kv.Value)) + log.GetLogger().Debugf("transaction get prefix key: %s, modRevision: %d", key, kv.ModRevision) + } + return +} + +// GetPrefix returns values of the key as a prefix from etcd, and cached them to perform 'If' statement in the +// transaction. The method will return cached values if the same key has been called within the transaction. +func (t *Transaction) GetPrefix(key string) (keys, values []string, err error) { + if v, ok := t.rds[key]; ok { + keys, values = t.getRespKVs(v.resp) + log.GetLogger().Debugf("cached: transaction get prefix: %s, resp num: %v", key, len(keys)) + return keys, values, nil + } + + etcdCtxInfo := etcd3.CreateEtcdCtxInfo(t.ctx) + resp, err := t.etcdClient.GetResponse(etcdCtxInfo, key, clientv3.WithPrefix()) + if err != nil { + log.GetLogger().Errorf("failed to get with prefix from etcd, error: %s", err.Error()) + return nil, nil, err + } + t.rds[key] = &reads{resp, getRespMaxModRev(resp), true} + keys, values = t.getRespKVs(resp) + log.GetLogger().Debugf("transaction get prefix: %s, resp num: %v", key, len(keys)) + return keys, values, nil +} + +// Del caches a key, it will be replaced if the same key has been called within the transaction. +func (t *Transaction) Del(key string) { + log.GetLogger().Debugf("transaction delete key: %s", key) + t.wrs[key] = &writes{"", delOp, false} +} + +// DelPrefix caches a key, it will be replaced if the same key has been called within the transaction. +func (t *Transaction) DelPrefix(key string) { + log.GetLogger().Debugf("transaction delete prefix: %s", key) + t.wrs[key] = &writes{"", delOp, true} +} + +func (t *Transaction) genCmp() []clientv3.Cmp { + cs := make([]clientv3.Cmp, 0, len(t.rds)) + for k, v := range t.rds { + k = t.etcdClient.AttachAZPrefix(k) + result := "=" + rev := v.modRev + if v.withPrefix && rev != 0 { + result = "<" + rev++ + } + + c := clientv3.Compare(clientv3.ModRevision(k), result, rev) + if v.withPrefix { + c = c.WithPrefix() + } + cs = append(cs, c) + } + return cs +} + +func (t *Transaction) genOp() []clientv3.Op { + ops := make([]clientv3.Op, 0, len(t.wrs)) + for k, v := range t.wrs { + k = t.etcdClient.AttachAZPrefix(k) + var op clientv3.Op + if v.op == writeOp { + op = clientv3.OpPut(k, v.value) + } else { + if v.withPrefix { + op = clientv3.OpDelete(k, clientv3.WithPrefix()) + } else { + op = clientv3.OpDelete(k) + } + } + ops = append(ops, op) + } + return ops +} + +// Commit do the 'If' statement with all Get and GetPrefix methods, +// and do the 'Then' statement with all Put, Del, DelPrefix methods. +func (t *Transaction) Commit() error { + resp, err := t.etcdClient.Client.KV.Txn(t.ctx).If(t.genCmp()...).Then(t.genOp()...).Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit transaction, error: %s", err.Error()) + return err + } + if !resp.Succeeded { + log.GetLogger().Errorf("transaction commit not succeeded") + t.printError() + return engine.ErrTransaction + } + log.GetLogger().Debugf("transaction get revision, revision: %d", resp.Header.Revision) + return nil +} + +// Cancel undoes the commit. +func (t *Transaction) Cancel() { + t.cancel() +} + +func (t *Transaction) printError() { + for k, v := range t.rds { + rev := v.modRev + if v.withPrefix { + if t.rereadPrefix(k, rev) != nil { + continue + } + } else { + if t.rereadKey(k, rev) != nil { + continue + } + } + } +} + +func (t *Transaction) rereadPrefix(k string, revision int64) error { + etcdCtxInfo := etcd3.CreateEtcdCtxInfo(t.ctx) + resp, err := t.etcdClient.GetResponse(etcdCtxInfo, k, clientv3.WithPrefix()) + if err != nil { + log.GetLogger().Errorf("failed to reread data with prefix from etcd, error: %s", err.Error()) + return err + } + t.printPrefixValue(resp, k, revision) + return nil +} + +func (t *Transaction) printPrefixValue(resp *clientv3.GetResponse, k string, revision int64) { + for index, kv := range resp.Kvs { + if kv.ModRevision > revision { + log.GetLogger().Errorf("reread etcd data by prefix, Key : %s", string(resp.Kvs[index].Key)) + log.GetLogger().Errorf("reread etcd data by prefix, Value: %s", string(resp.Kvs[index].Value)) + log.GetLogger().Errorf("reread etcd data by prefix, CreateRevision: %d, ModRevision: %d, Version: %d", + resp.Kvs[index].CreateRevision, resp.Kvs[index].ModRevision, resp.Kvs[index].Version) + t.printCachedPrefixValue(k, string(kv.Key)) + } + } +} + +func (t *Transaction) printCachedPrefixValue(prefix string, key string) { + if v, ok := t.rds[prefix]; ok { + for i, value := range v.resp.Kvs { + if string(value.Key) == string(key) { + log.GetLogger().Errorf("get cached data by prefix, Key : %s", string(v.resp.Kvs[i].Key)) + log.GetLogger().Errorf("get cached data by prefix, Value: %s", string(v.resp.Kvs[i].Value)) + log.GetLogger().Errorf("get cached data by prefix, CreateRevision: %d, ModRevision: %d, Version: %d", + v.resp.Kvs[i].CreateRevision, v.resp.Kvs[i].ModRevision, v.resp.Kvs[i].Version) + } + } + } else { + log.GetLogger().Errorf("invalid prefix, prefix : %s", prefix) + } +} + +func (t *Transaction) rereadKey(k string, revision int64) error { + etcdCtxInfo := etcd3.CreateEtcdCtxInfo(t.ctx) + resp, err := t.etcdClient.GetResponse(etcdCtxInfo, k) + if err != nil { + log.GetLogger().Errorf("invalid key, k: %s, error: %s", k, err.Error()) + return err + } + t.printKeyValue(resp, k, revision) + return nil +} + +func (t *Transaction) printKeyValue(resp *clientv3.GetResponse, k string, revision int64) { + for index, kv := range resp.Kvs { + if kv.ModRevision != revision { + log.GetLogger().Errorf("reread etcd data, Key : %s", string(resp.Kvs[index].Key)) + log.GetLogger().Errorf("reread etcd data, Value: %s", string(resp.Kvs[index].Value)) + log.GetLogger().Errorf("reread etcd data, CreateRevision: %d, ModRevision: %d, Version: %d", + resp.Kvs[index].CreateRevision, resp.Kvs[index].ModRevision, resp.Kvs[index].Version) + t.printCachedKeyValue(k) + } + } +} + +func (t *Transaction) printCachedKeyValue(k string) { + if v, ok := t.rds[k]; ok { + if len(v.resp.Kvs) == 1 { + log.GetLogger().Errorf("get cached data, Key : %s", string(v.resp.Kvs[0].Key)) + log.GetLogger().Errorf("get cached data, Value: %s", string(v.resp.Kvs[0].Value)) + log.GetLogger().Errorf("get cached data, CreateRevision: %d, ModRevision: %d, Version: %d", + v.resp.Kvs[0].CreateRevision, v.resp.Kvs[0].ModRevision, v.resp.Kvs[0].Version) + } + } else { + log.GetLogger().Errorf("invalid key, k: %s", k) + } +} diff --git a/functionsystem/apps/meta_service/common/engine/etcd/transaction_test.go b/functionsystem/apps/meta_service/common/engine/etcd/transaction_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5342f780ba2fbde59b465d972574169786f5de48 --- /dev/null +++ b/functionsystem/apps/meta_service/common/engine/etcd/transaction_test.go @@ -0,0 +1,184 @@ +package etcd + +import ( + "context" + "errors" + "reflect" + "testing" + + "meta_service/common/etcd3" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// 给一个prefix，从rds里找出对应的reads +func TestPrintCachedPrefixValue(t *testing.T) { + kv1 := &mvccpb.KeyValue{ + Key: []byte("mock-key"), + Value: []byte("mock-key"), + ModRevision: 1, + Version: 1, + } + tr := &Transaction{ + rds: map[string]*reads{ + "mock-prefix": {resp: &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{kv1}, + }}, + }, + } + Convey("Test PrintCachedPrefixValue", t, func() { + tr.printCachedPrefixValue("mock-prefix", "mock-key") + tr.printCachedPrefixValue("mock-prefix00", "mock-key") + }) +} + +func TestPrintPrefixValue(t *testing.T) { + Convey("Test printPrefixValue", t, func() { + kv1 := &mvccpb.KeyValue{ + Key: []byte("mock-key"), + Value: []byte("mock-key"), + CreateRevision: 1, + ModRevision: 1, + Version: 1, + } + resp := &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{kv1}, + } + tr := &Transaction{ + rds: map[string]*reads{ + "mock-prefix": {resp: resp}, + }, + } + tr.printPrefixValue(resp, "mock-prefix", 0) + }) +} + +func TestRereadPrefix(t *testing.T) { + kv1 := &mvccpb.KeyValue{ + Key: []byte("mock-key"), + Value: []byte("mock-key"), + } + resp := &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{kv1}, + } + tr := &Transaction{ + rds: map[string]*reads{ + "mock-prefix": {resp: resp}, + }, + ctx: context.TODO(), + etcdClient: &etcd3.EtcdClient{}, + } + Convey("Test RereadPrefix", t, func() { + Convey("with err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(w *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption, + ) (*clientv3.GetResponse, error) { + return nil, errors.New("mock err") + }) + defer patch.Reset() + err := tr.rereadPrefix("mock-prefix", 0) + So(err, ShouldNotBeNil) + }) + Convey("without err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(w *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption, + ) (*clientv3.GetResponse, error) { + return resp, nil + }) + defer patch.Reset() + err := tr.rereadPrefix("mock-prefix", 0) + So(err, ShouldBeNil) + }) + }) +} + +func TestRereadKey(t *testing.T) { + kv1 := &mvccpb.KeyValue{ + Key: []byte("mock-key"), + Value: []byte("mock-key"), + CreateRevision: 1, + ModRevision: 1, + Version: 1, + } + resp := &clientv3.GetResponse{ + Kvs: []*mvccpb.KeyValue{kv1}, + } + tr := &Transaction{ + rds: map[string]*reads{ + "mock-prefix": {resp: resp}, + }, + ctx: context.TODO(), + etcdClient: &etcd3.EtcdClient{}, + } + Convey("Test rereadKey", t, func() { + Convey("with err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(w *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption, + ) (*clientv3.GetResponse, error) { + return nil, errors.New("mock err") + }) + defer patch.Reset() + err := tr.rereadKey("mock-prefix", 0) + So(err, ShouldNotBeNil) + }) + Convey("without err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(w *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption, + ) (*clientv3.GetResponse, error) { + return resp, nil + }) + defer patch.Reset() + err := tr.rereadKey("mock-prefix", 0) + So(err, ShouldBeNil) + }) + Convey("with printCachedKeyValue err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(w *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption, + ) (*clientv3.GetResponse, error) { + return resp, nil + }) + defer patch.Reset() + delete(tr.rds, "mock-prefix") + err := tr.rereadKey("mock-prefix", 0) + So(err, ShouldBeNil) + }) + }) +} + +func TestPrintError(t *testing.T) { + kv1 := &mvccpb.KeyValue{ + Key: []byte("mock-key"), Value: []byte("mock-key"), CreateRevision: 1, + ModRevision: 1, Version: 1, + } + kv2 := &mvccpb.KeyValue{ + Key: []byte("mock-key2"), Value: []byte("mock-key2"), CreateRevision: 1, + ModRevision: 1, Version: 1, + } + resp := &clientv3.GetResponse{Kvs: []*mvccpb.KeyValue{kv1, kv2}} + tr := &Transaction{ + rds: map[string]*reads{ + "mock-prefix": {resp: resp, withPrefix: true}, + "mock-prefix2": {resp: resp, withPrefix: false}, + }, + ctx: context.TODO(), + etcdClient: &etcd3.EtcdClient{}, + } + Convey("Test printError", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&etcd3.EtcdClient{}), "GetResponse", + func(w *etcd3.EtcdClient, ctxInfo etcd3.EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption, + ) (*clientv3.GetResponse, error) { + return resp, nil + }) + defer patch.Reset() + tr.printError() + }) +} diff --git a/functionsystem/apps/meta_service/common/engine/interface.go b/functionsystem/apps/meta_service/common/engine/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..aa86307630b671f4ca2b7066c0771a04a19923bf --- /dev/null +++ b/functionsystem/apps/meta_service/common/engine/interface.go @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package engine defines interfaces for a storage kv engine +package engine + +import ( + "context" + "errors" +) + +var ( + // ErrKeyNotFound means key does not exist + ErrKeyNotFound = errors.New("key not found") + // ErrTransaction means a transaction request has failed. User should retry. + ErrTransaction = errors.New("failed to execute a transaction") +) + +// Engine defines a storage kv engine +type Engine interface { + // Get retrieves the value for a key. It returns ErrKeyNotFound if key does not exist. + Get(ctx context.Context, key string) (val string, err error) + + // Count retries the number of key value pairs from a prefix key. + Count(ctx context.Context, prefix string) (int64, error) + + // FirstInRange retries the first key value pairs sort by keys from a prefix. It returns ErrKeyNotFound if the + // range is empty. + FirstInRange(ctx context.Context, prefix string) (key, val string, err error) + + // LastInRange is similar to FirstInRange by searches backwards. It returns ErrKeyNotFound if the range is empty. + LastInRange(ctx context.Context, prefix string) (key, val string, err error) + + // PrepareStream creates a stream that users can combine it with "Filter" and "Execute" to fuzzy search from a + // range of key value pairs. + PrepareStream(ctx context.Context, prefix string, decode DecodeHandleFunc, by SortBy) PrepareStmt + + // Put writes a key value pairs. + Put(ctx context.Context, key string, value string) error + + // Delete removes a key value pair. + Delete(ctx context.Context, key string) error + + // BeginTx starts a new transaction. + BeginTx(ctx context.Context) Transaction + + // Close cleans up any resources holds by the engine. + Close() error +} + +// FilterFunc filters a range stream +type FilterFunc func(interface{}) bool + +// PrepareStmt allows fuzzy searching of a range of key value pairs. +type PrepareStmt interface { + Filter(filter FilterFunc) PrepareStmt + Execute() (Stream, error) +} + +// Stream defines interface to range a filtered key value pairs. +type Stream interface { + // Next returns the next value from a stream. It returns io.EOF when stream ends + Next() (interface{}, error) +} + +// DecodeHandleFunc decodes key value pairs to a user defined type. +type DecodeHandleFunc func(key, value string) (interface{}, error) + +// SortOrder is the sorting order of a stream. +type SortOrder int + +const ( + // Ascend starts from small to large. + Ascend SortOrder = iota + + // Descend starts from large to small. + Descend +) + +// SortTarget is the sorting target of a stream. +type SortTarget int + +const ( + // SortName tells a stream to sort by name. + SortName SortTarget = iota + + // SortCreate tells a stream to sort by create time. + SortCreate + + // SortModify tells a stream to sort by update time. + SortModify +) + +// SortBy is the sorting order and target of a stream. +type SortBy struct { + Order SortOrder + Target SortTarget +} + +// Transaction ensures consistent view and change of the data. +type Transaction interface { + Get(key string) (val string, err error) + GetPrefix(key string) (keys, values []string, err error) + Put(key string, value string) + Del(key string) + DelPrefix(key string) + Commit() error + Cancel() +} diff --git a/functionsystem/apps/meta_service/common/etcd3/config.go b/functionsystem/apps/meta_service/common/etcd3/config.go new file mode 100644 index 0000000000000000000000000000000000000000..2d184eb849413bd20daa8c5ed74d99278bb6a895 --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcd3/config.go @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "context" + "crypto/tls" + "crypto/x509" + + clientv3 "go.etcd.io/etcd/client/v3" + "k8s.io/apimachinery/pkg/util/wait" + + "meta_service/common/logger/log" + commontls "meta_service/common/tls" +) + +// EtcdConfig the info to get function instance +type EtcdConfig struct { + Servers []string `json:"servers" yaml:"servers" valid:"required"` + User string `json:"user" yaml:"user" valid:"optional"` + Passwd string `json:"password" yaml:"password" valid:"optional"` + AuthType string `json:"authType" yaml:"authType" valid:"optional"` + SslEnable bool `json:"sslEnable,omitempty" yaml:"sslEnable,omitempty" valid:"optional"` + LimitRate int `json:"limitRate,omitempty" yaml:"limitRate,omitempty" valid:"optional"` + LimitBurst int `json:"limitBurst,omitempty" yaml:"limitBurst,omitempty" valid:"optional"` + LimitTimeout int `json:"limitTimeout,omitempty" yaml:"limitTimeout,omitempty" valid:"optional"` + LeaseTTL int64 `json:"leaseTTL,omitempty" yaml:"leaseTTL,omitempty" valid:"optional"` + RenewTTL int64 `json:"renewTTL,omitempty" yaml:"renewTTL,omitempty" valid:"optional"` + CaFile string `json:"cafile,omitempty" yaml:"cafile,omitempty" valid:"optional"` + CertFile string `json:"certfile,omitempty" yaml:"certfile,omitempty" valid:"optional"` + KeyFile string `json:"keyfile,omitempty" yaml:"keyfile,omitempty" valid:"optional"` + PassphraseFile string `json:"passphraseFile,omitempty" yaml:"passphraseFile,omitempty" valid:"optional"` + AZPrefix string `json:"azPrefix,omitempty" yaml:"azPrefix,omitempty" valid:"optional"` + // DisableSync will not run the sync method to avoid endpoints being replaced by the domain name, default is FALSE + DisableSync bool +} + +// GetETCDCertificatePath get the certificate path from tlsConfig. +func GetETCDCertificatePath(config EtcdConfig, tlsConfig commontls.MutualTLSConfig) EtcdConfig { + if !config.SslEnable { + return config + } + config.CaFile = tlsConfig.RootCAFile + config.CertFile = tlsConfig.ModuleCertFile + config.KeyFile = tlsConfig.ModuleKeyFile + return config +} + +type etcdAuth interface { + getEtcdConfig() (*clientv3.Config, error) + renewToken(client *clientv3.Client, stop chan struct{}) +} + +type noAuth struct{} + +type tlsAuth struct { + cerfile string + keyfile string + cafile string +} + +type pwdAuth struct { + user string + passWd []byte +} + +// this support no tls +func (e *EtcdConfig) getEtcdAuthTypeNoTLS() etcdAuth { + if e.SslEnable { + return &tlsAuth{ + cerfile: e.CertFile, + keyfile: e.KeyFile, + cafile: e.CaFile, + } + } + if e.User == "" || e.Passwd == "" { + return &noAuth{} + } + return &pwdAuth{ + user: e.User, + passWd: []byte(e.Passwd), + } +} + +func (n *noAuth) getEtcdConfig() (*clientv3.Config, error) { + return &clientv3.Config{}, nil +} + +func (n *noAuth) renewToken(client *clientv3.Client, stop chan struct{}) { + return +} + +func (t *tlsAuth) getEtcdConfig() (*clientv3.Config, error) { + var pool *x509.CertPool + pool, err := commontls.GetX509CACertPool(t.cafile) + if err != nil { + log.GetLogger().Errorf("failed to getX509CACertPool: %s", err.Error()) + return nil, err + } + + var certs []tls.Certificate + certs, err = commontls.LoadServerTLSCertificate(t.cerfile, t.keyfile) + if err != nil { + log.GetLogger().Errorf("failed to loadServerTLSCertificate: %s", err.Error()) + return nil, err + } + + clientAuthMode := tls.NoClientCert + cfg := &clientv3.Config{ + TLS: &tls.Config{ + RootCAs: pool, + Certificates: certs, + ClientAuth: clientAuthMode, + }, + } + return cfg, nil +} + +func (t *tlsAuth) renewToken(client *clientv3.Client, stop chan struct{}) { + return +} + +func (p *pwdAuth) getEtcdConfig() (*clientv3.Config, error) { + cfg := &clientv3.Config{ + Username: p.user, + Password: string(p.passWd), + } + return cfg, nil +} + +// renewToken can keep client token not expired, because +// etcd server will renew simple token TTL if token is not expired. +func (p *pwdAuth) renewToken(client *clientv3.Client, stop chan struct{}) { + if stop == nil { + log.GetLogger().Errorf("stop chan is nil") + return + } + wait.Until(func() { + ctx, cancel := context.WithTimeout(context.Background(), DurationContextTimeout) + _, err := client.Get(ctx, renewKey) + cancel() + if err != nil { + log.GetLogger().Warnf("renew token error: %s", err.Error()) + } + }, tokenRenewTTL, stop) + log.GetLogger().Infof("stopped to renew token") +} diff --git a/functionsystem/apps/meta_service/common/etcd3/config_test.go b/functionsystem/apps/meta_service/common/etcd3/config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d85397924a1a833df63a13420282bca217c74a34 --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcd3/config_test.go @@ -0,0 +1,163 @@ +package etcd3 + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "testing" + + "meta_service/common/crypto" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + + commontls "meta_service/common/tls" +) + +func TestGetETCDCertificatePath(t *testing.T) { + Convey("Test ssl enable", t, func() { + config := EtcdConfig{ + SslEnable: true, + CaFile: "", + CertFile: "", + KeyFile: "", + } + tlsConfig := commontls.MutualTLSConfig{ + RootCAFile: "xxx.ca", + ModuleCertFile: "xxx.cert", + ModuleKeyFile: "xxx.key", + } + etcdConfig := GetETCDCertificatePath(config, tlsConfig) + So(etcdConfig.CaFile, ShouldEqual, tlsConfig.RootCAFile) + So(etcdConfig.CertFile, ShouldEqual, tlsConfig.ModuleCertFile) + So(etcdConfig.KeyFile, ShouldEqual, tlsConfig.ModuleKeyFile) + }) + Convey("Test ssl disable", t, func() { + config := EtcdConfig{ + SslEnable: false, + CaFile: "", + CertFile: "", + KeyFile: "", + } + tlsConfig := commontls.MutualTLSConfig{ + RootCAFile: "xxx.ca", + ModuleCertFile: "xxx.cert", + ModuleKeyFile: "xxx.key", + } + etcdConfig := GetETCDCertificatePath(config, tlsConfig) + So(etcdConfig.CaFile, ShouldNotEqual, tlsConfig.RootCAFile) + So(etcdConfig.CertFile, ShouldNotEqual, tlsConfig.ModuleCertFile) + So(etcdConfig.KeyFile, ShouldNotEqual, tlsConfig.ModuleKeyFile) + }) +} + +func TestGetEtcdAuthType(t *testing.T) { + Convey("Test getEtcdAuthType, tlsAuth", t, func() { + config := EtcdConfig{ + SslEnable: true, + CaFile: "", + CertFile: "", + KeyFile: "", + } + etcdAuth := config.getEtcdAuthType() + So(etcdAuth, ShouldNotBeNil) + }) + Convey("Test getEtcdAuthType, noAuth", t, func() { + config := EtcdConfig{ + SslEnable: false, + CaFile: "", + CertFile: "", + KeyFile: "", + User: "test", + Passwd: "", + } + etcdAuth := config.getEtcdAuthType() + So(etcdAuth, ShouldNotBeNil) + }) + Convey("Test getEtcdAuthType, pwdAuth", t, func() { + config := EtcdConfig{ + SslEnable: false, + CaFile: "", + CertFile: "", + KeyFile: "", + User: "test", + Passwd: "test", + } + etcdAuth := config.getEtcdAuthType() + So(etcdAuth, ShouldNotBeNil) + }) + Convey("Test getEtcdAuthType, clientTlsAuth", t, func() { + config := EtcdConfig{ + AuthType: "TLS", + SslEnable: false, + CaFile: "", + CertFile: "", + KeyFile: "", + User: "test", + Passwd: "test", + } + etcdAuth := config.getEtcdAuthType() + So(etcdAuth, ShouldNotBeNil) + }) +} + +func TestGetEtcdConfigTlsAuth(t *testing.T) { + tlsAuth := &tlsAuth{} + Convey("GetX509CACertPool success", t, func() { + patch := gomonkey.ApplyFunc(commontls.GetX509CACertPool, func(string) (*x509.CertPool, error) { + return nil, nil + }) + defer patch.Reset() + Convey("LoadServerTLSCertificate success", func() { + patch := gomonkey.ApplyFunc(commontls.LoadServerTLSCertificate, func(string, string) ([]tls.Certificate, error) { + return nil, nil + }) + defer patch.Reset() + _, err := tlsAuth.getEtcdConfig() + So(err, ShouldBeNil) + }) + Convey("LoadServerTLSCertificate fail", func() { + patch := gomonkey.ApplyFunc(commontls.LoadServerTLSCertificate, func(string, string) ([]tls.Certificate, error) { + return nil, errors.New("LoadServerTLSCertificate fail") + }) + defer patch.Reset() + _, err := tlsAuth.getEtcdConfig() + So(err, ShouldNotBeNil) + }) + }) +} + +func TestRenewTokenTlsAuth(t *testing.T) { + tlsAuth := &tlsAuth{} + cli := &EtcdClient{} + stopCh := make(chan struct{}) + + tlsAuth.renewToken(cli.Client, stopCh) +} + +func TestRenewTokenPwdAuth(t *testing.T) { + pwdAuth := &pwdAuth{} + cli := &EtcdClient{} + + pwdAuth.renewToken(cli.Client, nil) +} + +func TestGetEtcdConfigPwdAuth(t *testing.T) { + pwdAuth := &pwdAuth{} + Convey("Decrypt success", t, func() { + patch := gomonkey.ApplyFunc(crypto.Decrypt, func([]byte, []byte) (string, error) { + return "", nil + }) + defer patch.Reset() + _, err := pwdAuth.getEtcdConfig() + So(err, ShouldBeNil) + }) + Convey("Decrypt fail", t, func() { + patch := gomonkey.ApplyFunc(crypto.Decrypt, func([]byte, []byte) (string, error) { + return "", errors.New("decrypt fail") + }) + defer patch.Reset() + _, err := pwdAuth.getEtcdConfig() + So(err, ShouldNotBeNil) + }) +} diff --git a/functionsystem/apps/meta_service/common/etcd3/event.go b/functionsystem/apps/meta_service/common/etcd3/event.go new file mode 100644 index 0000000000000000000000000000000000000000..3ea7a17d6e6f9a7cd94a51d6c37bef543769887f --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcd3/event.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "time" + + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + // PUT event + PUT = iota + // DELETE event + DELETE + // ERROR unexpected event + ERROR + // SYNCED synced event + SYNCED +) + +// Event of databases +type Event struct { + Type int + Key string + Value []byte + PrevValue []byte + Rev int64 +} + +// parseKV converts a KeyValue retrieved from an initial sync() listing to a synthetic isCreated event. +func parseKV(kv *mvccpb.KeyValue) *Event { + return &Event{ + Type: PUT, + Key: string(kv.Key), + Value: kv.Value, + PrevValue: nil, + Rev: kv.ModRevision, + } +} + +func parseEvent(e *clientv3.Event) *Event { + eType := 0 + if e.Type == clientv3.EventTypeDelete { + eType = DELETE + } + ret := &Event{ + Type: eType, + Key: string(e.Kv.Key), + Value: e.Kv.Value, + Rev: e.Kv.ModRevision, + } + if e.PrevKv != nil { + ret.PrevValue = e.PrevKv.Value + } + return ret +} + +func parseErr(err error) *Event { + return &Event{Type: ERROR, Value: []byte(err.Error())} +} + +func parseSync(t time.Time) *Event { + return &Event{Type: SYNCED, Value: []byte(t.String())} +} diff --git a/functionsystem/apps/meta_service/common/etcd3/event_test.go b/functionsystem/apps/meta_service/common/etcd3/event_test.go new file mode 100644 index 0000000000000000000000000000000000000000..674578924f5f03cbbef4e7985306050adebb6aa9 --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcd3/event_test.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "fmt" + "testing" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + + "github.com/smartystreets/goconvey/convey" + "go.etcd.io/etcd/api/v3/mvccpb" +) + +func TestParseKV(t *testing.T) { + kv := &mvccpb.KeyValue{ + Key: []byte("/sn/workeragent/abc"), + Value: []byte("value_abc"), + ModRevision: 1, + } + res := parseKV(kv) + if res == nil { + t.Errorf("failed to parse kv") + } +} + +func TestParseEvent(t *testing.T) { + e := &clientv3.Event{ + Type: clientv3.EventTypeDelete, + Kv: &mvccpb.KeyValue{ + Key: []byte("/sn/workeragent/abc"), + Value: []byte("value_abc"), + ModRevision: 1, + }, + } + + res := parseEvent(e) + if res == nil { + t.Errorf("failed to parse event") + } + e = &clientv3.Event{ + Type: clientv3.EventTypePut, + Kv: &mvccpb.KeyValue{ + Key: []byte("/sn/workeragent/abc"), + Value: []byte("value_abc"), + ModRevision: 1, + }, + PrevKv: &mvccpb.KeyValue{ + Key: []byte("/sn/workeragent/def"), + Value: []byte("value_def"), + ModRevision: 1, + }, + } + res = parseEvent(e) + convey.ShouldNotBeNil(res) +} +func TestParseErr(t *testing.T) { + res := parseErr(fmt.Errorf("test")) + convey.ShouldNotBeNil(res) +} + +func TestParseSync(t *testing.T) { + res := parseSync(time.Time{}) + convey.ShouldNotBeNil(res) +} diff --git a/functionsystem/apps/meta_service/common/etcd3/watcher.go b/functionsystem/apps/meta_service/common/etcd3/watcher.go new file mode 100644 index 0000000000000000000000000000000000000000..76ecb0cb15daf136e91bcd5efec80ee4a5112e52 --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcd3/watcher.go @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd3 implements crud and watch operations based etcd clientv3 +package etcd3 + +import ( + "fmt" + "strings" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + "golang.org/x/net/context" + + "meta_service/common/logger/log" +) + +const ( + // We have set a buffer in order to reduce times of context switches. + incomingBufSize = 2000 + outgoingBufSize = 2000 +) + +// EtcdClientInterface is the interface of ETCD client +type EtcdClientInterface interface { + GetResponse(ctxInfo EtcdCtxInfo, etcdKey string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) + Put(ctxInfo EtcdCtxInfo, etcdKey string, value string, opts ...clientv3.OpOption) error + Delete(ctxInfo EtcdCtxInfo, etcdKey string, opts ...clientv3.OpOption) error +} + +// EtcdClient etcd client struct +type EtcdClient struct { + Client *clientv3.Client + AZPrefix string +} + +// EtcdWatchChan implements watch.Interface. +type EtcdWatchChan struct { + incomingEventChan chan *Event + ResultChan chan *Event + errChan chan error + watcher *EtcdClient + key string + initialRev int64 + recursive bool + ctx context.Context + cancel context.CancelFunc +} + +// EtcdCtxInfo etcd context info +type EtcdCtxInfo struct { + Ctx context.Context + Cancel context.CancelFunc +} + +const ( + // keepaliveTime is the time after which client pings the server to see if + // transport is alive. + keepaliveTime = 30 * time.Second + + // keepaliveTimeout is the time that the client waits for a response for the + // keep-alive attempt. + keepaliveTimeout = 10 * time.Second + + // dialTimeout is the timeout for establishing a connection. + // 20 seconds as times should be set shorter than that will cause TLS connections to fail + dialTimeout = 20 * time.Second + + // tokenRenewTTL the default TTL of etcd simple token is 300s, so the renew TTL should be smaller than 300s + tokenRenewTTL = 30 * time.Second + + // renewKey etcd server will renew simple token TTL if token is not expired. + // so use an random key path for querying in order to renew token. + renewKey = "/keyforrenew" + + // DurationContextTimeout etcd request timeout, default context duration timeout + DurationContextTimeout = 5 * time.Second +) + +// AttachAZPrefix - +func (w *EtcdClient) AttachAZPrefix(key string) string { + if len(w.AZPrefix) != 0 { + return fmt.Sprintf("/%s%s", w.AZPrefix, key) + } + return key +} + +// DetachAZPrefix - +func (w *EtcdClient) DetachAZPrefix(key string) string { + if len(w.AZPrefix) != 0 { + return strings.TrimPrefix(key, fmt.Sprintf("/%s", w.AZPrefix)) + } + return key +} + +// GetResponse get etcd value and return pointer of GetResponse struct +func (w *EtcdClient) GetResponse(ctxInfo EtcdCtxInfo, etcdKey string, + opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + etcdKey = w.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + + kv := clientv3.NewKV(w.Client) + leaderCtx := clientv3.WithRequireLeader(ctx) + getResp, err := kv.Get(leaderCtx, etcdKey, opts...) + + return getResp, err +} + +// Put put context key and value +func (w *EtcdClient) Put(ctxInfo EtcdCtxInfo, etcdKey string, value string, opts ...clientv3.OpOption) error { + etcdKey = w.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + + kv := clientv3.NewKV(w.Client) + leaderCtx := clientv3.WithRequireLeader(ctx) + _, err := kv.Put(leaderCtx, etcdKey, value, opts...) + return err +} + +// Delete delete key +func (w *EtcdClient) Delete(ctxInfo EtcdCtxInfo, etcdKey string, opts ...clientv3.OpOption) error { + etcdKey = w.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + + kv := clientv3.NewKV(w.Client) + leaderCtx := clientv3.WithRequireLeader(ctx) + _, err := kv.Delete(leaderCtx, etcdKey, opts...) + return err +} + +// TxnPut transaction put operation with if key existed put failed +func (w *EtcdClient) TxnPut(ctxInfo EtcdCtxInfo, etcdKey string, value string) error { + etcdKey = w.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + kv := clientv3.NewKV(w.Client) + leaderCtx := clientv3.WithRequireLeader(ctx) + txnRsp, err := kv.Txn(leaderCtx). + If(clientv3.Compare(clientv3.CreateRevision(etcdKey), "=", 0)). + Then(clientv3.OpPut(etcdKey, value)). + Else(clientv3.OpGet(etcdKey)).Commit() + if err != nil { + return err + } + if !txnRsp.Succeeded { + log.GetLogger().Warnf("the key has already exist: %s", etcdKey) + return fmt.Errorf("duplicated key") + } + return nil +} + +// GetValues return list of object for value +func (w *EtcdClient) GetValues(ctxInfo EtcdCtxInfo, etcdKey string, opts ...clientv3.OpOption) ([]string, error) { + etcdKey = w.AttachAZPrefix(etcdKey) + ctx, cancel := ctxInfo.Ctx, ctxInfo.Cancel + defer cancel() + + kv := clientv3.NewKV(w.Client) + leaderCtx := clientv3.WithRequireLeader(ctx) + response, err := kv.Get(leaderCtx, etcdKey, opts...) + if err != nil { + return nil, err + } + values := make([]string, len(response.Kvs), len(response.Kvs)) + + for index, v := range response.Kvs { + values[index] = string(v.Value) + } + return values, err +} + +// CreateEtcdCtxInfo return context with cancle function +func CreateEtcdCtxInfo(ctx context.Context) EtcdCtxInfo { + ctx, cancel := context.WithCancel(ctx) + leaderCtx := clientv3.WithRequireLeader(ctx) + return EtcdCtxInfo{leaderCtx, cancel} +} + +// CreateEtcdCtxInfoWithTimeout create a context with timeout, default timeout is DurationContextTimeout +func CreateEtcdCtxInfoWithTimeout(ctx context.Context, duration time.Duration) EtcdCtxInfo { + ctx, cancel := context.WithTimeout(ctx, duration) + leaderCtx := clientv3.WithRequireLeader(ctx) + return EtcdCtxInfo{leaderCtx, cancel} +} + +// NewEtcdWatcher new a etcd watcher +func NewEtcdWatcher(config EtcdConfig) (*EtcdClient, error) { + cfg, err := config.getEtcdAuthTypeNoTLS().getEtcdConfig() + if err != nil { + return nil, err + } + + cfg.DialTimeout = dialTimeout + cfg.DialKeepAliveTime = keepaliveTime + cfg.DialKeepAliveTimeout = keepaliveTimeout + + cfg.Endpoints = config.Servers + client, err := clientv3.New(*cfg) + if err != nil { + return nil, err + } + stopCh := make(chan struct{}) + go config.getEtcdAuthTypeNoTLS().renewToken(client, stopCh) + + // fetch registered grpc-proxy endpoints + if config.DisableSync { + return &EtcdClient{Client: client}, nil + } + if err = client.Sync(context.Background()); err != nil { + log.GetLogger().Warnf("Sync endpoints: %s", err.Error()) + } + log.GetLogger().Infof("Etcd discovered endpoints: %v", client.Endpoints()) + return &EtcdClient{Client: client, AZPrefix: config.AZPrefix}, nil +} diff --git a/functionsystem/apps/meta_service/common/etcd3/watcher_test.go b/functionsystem/apps/meta_service/common/etcd3/watcher_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6c09d3d7a93667837ff32045c0d89967e420728c --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcd3/watcher_test.go @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package etcd3 + +import ( + "fmt" + "reflect" + "testing" + "time" + + "meta_service/common/crypto" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + clientv3 "go.etcd.io/etcd/client/v3" + "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/util/wait" +) + +type EtcdTestSuite struct { + suite.Suite + defaultEtcdCtx EtcdCtxInfo + etcdClient *EtcdClient +} + +func (es *EtcdTestSuite) setupEtcdClient() { + var err error + patches := gomonkey.NewPatches() + cli := clientv3.Client{} + patches.ApplyFunc(clientv3.New, func(_ clientv3.Config) (*clientv3.Client, error) { + return &cli, nil + }) + patches.ApplyMethod(reflect.TypeOf(&cli), "Sync", func(_ *clientv3.Client, _ context.Context) error { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(&cli), "Endpoints", func(_ *clientv3.Client) []string { + return []string{"localhost:0"} + }) + defer patches.Reset() + // create a new etcd watcher for the following tests + auth := EtcdConfig{ + Servers: []string{"localhost:0"}, + User: "", + Passwd: "", + SslEnable: false, + } + es.etcdClient, err = NewEtcdWatcher(auth) + if err != nil { + err = fmt.Errorf("failed to create etcd watcher; err: %v", err) + } +} + +func (es *EtcdTestSuite) SetupSuite() { + es.defaultEtcdCtx = CreateEtcdCtxInfo(context.Background()) + es.setupEtcdClient() +} + +func (es *EtcdTestSuite) TearDownSuite() { + es.etcdClient = nil +} + +func (es *EtcdTestSuite) TestNewEtcdWatcher() { + var ( + serverList []string + auth EtcdConfig + err error + ) + + patches := gomonkey.NewPatches() + cli := clientv3.Client{} + patches.ApplyFunc(clientv3.New, func(_ clientv3.Config) (*clientv3.Client, error) { + return &cli, nil + }) + patches.ApplyMethod(reflect.TypeOf(&cli), "Sync", func(_ *clientv3.Client, _ context.Context) error { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(&cli), "Endpoints", func(_ *clientv3.Client) []string { + return []string{"localhost:0"} + }) + patches.ApplyFunc(crypto.Decrypt, func(cipherText []byte, secret []byte) (string, error) { + return "key", nil + }) + patches.ApplyFunc(wait.Until, func(f func(), period time.Duration, stopCh <-chan struct{}) { + return + }) + serverList = []string{"localhost:0"} + auth = EtcdConfig{ + Servers: serverList, + User: "", + Passwd: "", + SslEnable: false, + CaFile: "", + CertFile: "", + KeyFile: "", + } + _, err = NewEtcdWatcher(auth) + assert.Nil(es.T(), err) + + serverList = []string{"localhost:0"} + auth = EtcdConfig{ + Servers: serverList, + User: "", + Passwd: "", + SslEnable: true, + CaFile: "xxx.ca", + CertFile: "xxx.cert", + KeyFile: "xxx.key", + } + _, err = NewEtcdWatcher(auth) + assert.NotNil(es.T(), err) + + serverList = []string{"localhost:0"} + auth = EtcdConfig{ + Servers: serverList, + User: "user", + Passwd: "", + SslEnable: false, + } + _, err = NewEtcdWatcher(auth) + assert.Nil(es.T(), err) + patches.Reset() +} + +func (es *EtcdTestSuite) TestCRUD() { + cli := es.etcdClient + ctxInfo := es.defaultEtcdCtx + etcdKey := "test_key" + etcdValue := "test_value" + + patch := gomonkey.ApplyFunc(clientv3.NewKV, func(*clientv3.Client) clientv3.KV { + return KV{} + }) + err := cli.Put(ctxInfo, etcdKey, etcdValue) + assert.Nil(es.T(), err) + + _, err = cli.GetKeys(ctxInfo, etcdKey) + assert.Nil(es.T(), err) + + _, err = cli.GetValues(ctxInfo, etcdKey) + assert.Nil(es.T(), err) + + _, err = cli.GetResponse(ctxInfo, etcdKey) + assert.Nil(es.T(), err) + + err = cli.Delete(ctxInfo, etcdKey) + assert.Nil(es.T(), err) + + patch.Reset() +} + +func (es *EtcdTestSuite) TestCreateEtcdCtxInfoWithTimeout() { + ctxInfo := es.defaultEtcdCtx.Ctx + etcdCtxInfo := CreateEtcdCtxInfoWithTimeout(ctxInfo, 1) + assert.NotNil(es.T(), etcdCtxInfo) +} + +func TestEtcdTestSuite(t *testing.T) { + suite.Run(t, new(EtcdTestSuite)) +} + +type KV struct{} + +func (k KV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { + return &clientv3.GetResponse{}, nil +} + +func (k KV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { + return &clientv3.PutResponse{}, nil +} + +func (k KV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { + return &clientv3.DeleteResponse{}, nil +} + +func (k KV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { + return &clientv3.CompactResponse{}, nil +} + +func (k KV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { + return clientv3.OpResponse{}, nil +} + +func (k KV) Txn(ctx context.Context) clientv3.Txn { + return nil +} diff --git a/functionsystem/apps/meta_service/common/etcdkey/etcdkey.go b/functionsystem/apps/meta_service/common/etcdkey/etcdkey.go new file mode 100644 index 0000000000000000000000000000000000000000..6e7666d8ae8a0928828ed57220a7dc035d49e4ed --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcdkey/etcdkey.go @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcdkey contains etcd key definition and tools +package etcdkey + +import ( + "fmt" + "strings" + + "meta_service/common/urnutils" +) + +const ( + keyFormat = "/sn/%s/business/%s/tenant/%s/function/%s/version/%s/%s/%s" + keySeparator = "/" + stateWorkerLen = 13 + + cronTriggerTenantIndex = 8 + functionMetadataTenantIndex = 4 +) + +// Index of element in etcd key +const ( + prefixIndex = iota + 1 + typeIndex + businessIDKey + businessIDIndex + tenantIDKey + tenantIDIndex + functionKey + functionIndex + versionKey + versionIndex + zoneIndex + keyIndex +) + +// Index of instance element in etcd key +const ( + instancePrefixIndex = iota + 1 + instanceTypeIndex + instanceBusinessIDKey + instanceBusinessIDIndex + instanceTenantIDKey + instanceTenantIDIndex + instanceZoneIndex + instanceFunctionNameIndex + instanceUrnVersionIndex = 10 + instanceInstanceIDIndex = 13 + instanceKeyLen = 14 +) + +// EtcdKey etcd key interface definition +type EtcdKey interface { + String() string + ParseFrom(key string) error +} + +// StateWorkersKey state workers key +type StateWorkersKey struct { + TypeKey string + BusinessID string + TenantID string + Function string + Version string + Zone string + StateID string +} + +// String serialize state workers key struct to string +func (s *StateWorkersKey) String() string { + return fmt.Sprintf(keyFormat, s.TypeKey, s.BusinessID, s.TenantID, s.Function, s.Version, s.Zone, s.StateID) +} + +// ParseFrom parse string to state workers key struct +func (s *StateWorkersKey) ParseFrom(etcdKey string) error { + elements := strings.Split(etcdKey, keySeparator) + if len(elements) != stateWorkerLen { + return fmt.Errorf("failed to parse etcd key from %s", etcdKey) + } + s.TypeKey = elements[keyIndex] + s.BusinessID = elements[businessIDIndex] + s.TenantID = elements[tenantIDIndex] + s.Function = elements[functionIndex] + s.Version = elements[versionIndex] + s.Zone = elements[zoneIndex] + s.StateID = elements[keyIndex] + + return nil +} + +// WorkerInstanceKey is the etcd key path of worker instance +type WorkerInstanceKey struct { + TypeKey string + BusinessID string + TenantID string + Function string + Version string + Zone string + Instance string +} + +// String serialize worker instance key struct to string +func (w *WorkerInstanceKey) String() string { + return fmt.Sprintf(keyFormat, w.TypeKey, w.BusinessID, w.TenantID, w.Function, w.Version, w.Zone, w.Instance) +} + +// ParseFrom parse string to worker instance key struct +func (w *WorkerInstanceKey) ParseFrom(etcdKey string) error { + elements := strings.Split(etcdKey, keySeparator) + if len(elements) != stateWorkerLen { + return fmt.Errorf("failed to parse etcd key from %s", etcdKey) + } + w.TypeKey = elements[typeIndex] + w.BusinessID = elements[businessIDIndex] + w.TenantID = elements[tenantIDIndex] + w.Function = elements[functionIndex] + w.Version = elements[versionIndex] + w.Zone = elements[zoneIndex] + w.Instance = elements[keyIndex] + return nil +} + +// AnonymizeTenantCommonEtcdKey Anonymize tenant info in common etcd key +// /yr/functions/business/yrk/tenant/8e08d5cc0ad34032bba8d636040a278c/function/0-test1-addone/version/$latest +func AnonymizeTenantCommonEtcdKey(etcdKey string) string { + elements := strings.Split(etcdKey, keySeparator) + if len(elements) <= tenantIDIndex { + return etcdKey + } + elements[tenantIDIndex] = urnutils.Anonymize(elements[tenantIDIndex]) + return strings.Join(elements, keySeparator) +} + +// AnonymizeTenantCronTriggerEtcdKey Anonymize tenant info in cron trigger etcd key +// /sn/triggers/triggerType/CRON/business/yrk/tenant/i1fe539427b24702acc11fbb4e134e17/function/pytzip/version/$latest/398e2ca2-a160-4c22-bd05-94a90a5326e2 +func AnonymizeTenantCronTriggerEtcdKey(etcdKey string) string { + elements := strings.Split(etcdKey, keySeparator) + if len(elements) <= cronTriggerTenantIndex { + return etcdKey + } + elements[cronTriggerTenantIndex] = urnutils.Anonymize(elements[cronTriggerTenantIndex]) + return strings.Join(elements, keySeparator) +} + +// AnonymizeTenantFunctionMetadataEtcdKey Anonymize tenant info in function metadata etcd key +// /repo/FunctionVersion/business/tenant/funcName/version/ +func AnonymizeTenantFunctionMetadataEtcdKey(etcdKey string) string { + elements := strings.Split(etcdKey, keySeparator) + if len(elements) <= functionMetadataTenantIndex { + return etcdKey + } + elements[functionMetadataTenantIndex] = urnutils.Anonymize(elements[functionMetadataTenantIndex]) + return strings.Join(elements, keySeparator) +} + +// FunctionInstanceKey is the etcd key path of function instance +type FunctionInstanceKey struct { + TypeKey string + BusinessID string + TenantID string + FunctionName string + Version string + Zone string + InstanceID string +} + +// ParseFrom parse string to function instance key struct +// /sn/instance/business/yrk/tenant/12345678901234561234567890123456/function/0-defaultservice-py/version/$latest +// /defaultaz/8c9fa45600e5f44f00/10000000-0000-4000-b653-c11128589d17 +func (f *FunctionInstanceKey) ParseFrom(etcdKey string) error { + elements := strings.Split(etcdKey, keySeparator) + if len(elements) != instanceKeyLen { + return fmt.Errorf("failed to parse etcd key from %s: invalid key length", etcdKey) + } + f.TypeKey = elements[instanceTypeIndex] + f.BusinessID = elements[instanceBusinessIDIndex] + f.TenantID = elements[instanceTenantIDIndex] + f.Zone = elements[instanceZoneIndex] + f.InstanceID = elements[instanceInstanceIDIndex] + return nil +} diff --git a/functionsystem/apps/meta_service/common/etcdkey/etcdkey_test.go b/functionsystem/apps/meta_service/common/etcdkey/etcdkey_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e06cd1c2bfc12364dd7cb9bd6db3c8878fa3f569 --- /dev/null +++ b/functionsystem/apps/meta_service/common/etcdkey/etcdkey_test.go @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcdkey contains etcd key definition and tools +package etcdkey + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStateWorkersKey_ParseFrom(t *testing.T) { + type fields struct { + KeyType string + BusinessID string + TenantID string + Function string + Version string + Zone string + StateID string + } + type args struct { + key string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "test001", + fields: fields{ + KeyType: "stateworkers", + BusinessID: "yrk", + TenantID: "tenantID", + Function: "function", + Version: "$latest", + Zone: "defaultaz", + StateID: "stateID", + }, + args: args{ + key: "/sn/stateworkers/business/yrk/tenant/tenantID" + + "/function/function/version/$latest/defaultaz/stateID", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &StateWorkersKey{ + TypeKey: tt.fields.KeyType, + BusinessID: tt.fields.BusinessID, + TenantID: tt.fields.TenantID, + Function: tt.fields.Function, + Version: tt.fields.Version, + Zone: tt.fields.Zone, + StateID: tt.fields.StateID, + } + if err := s.ParseFrom(tt.args.key); (err != nil) != tt.wantErr { + t.Errorf("ParseFrom() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestStateWorkersKey_String(t *testing.T) { + type fields struct { + KeyType string + BusinessID string + TenantID string + Function string + Version string + Zone string + StateID string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "test002", + fields: fields{ + KeyType: "stateworkers", + BusinessID: "yrk", + TenantID: "tenantID", + Function: "function", + Version: "$latest", + Zone: "defaultaz", + StateID: "stateID", + }, + want: "/sn/stateworkers/business/yrk/tenant/tenantID" + + "/function/function/version/$latest/defaultaz/stateID", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &StateWorkersKey{ + TypeKey: tt.fields.KeyType, + BusinessID: tt.fields.BusinessID, + TenantID: tt.fields.TenantID, + Function: tt.fields.Function, + Version: tt.fields.Version, + Zone: tt.fields.Zone, + StateID: tt.fields.StateID, + } + if got := s.String(); got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestWorkerInstanceKey_ParseFrom(t *testing.T) { + tests := []struct { + arg string + want WorkerInstanceKey + err bool + }{ + { + arg: "/sn/workers/business/busid/tenant/abcd/function/" + + "0-counter-addone/version/$latest/defaultaz/defaultaz-#-pool7-500-500-python3.7-58f588848d-smss8", + want: WorkerInstanceKey{ + TypeKey: "workers", + BusinessID: "busid", + TenantID: "abcd", + Function: "0-counter-addone", + Version: "$latest", + Zone: "defaultaz", + Instance: "defaultaz-#-pool7-500-500-python3.7-58f588848d-smss8", + }, + err: false, + }, + { + arg: "/sn/workers/business/busid/tenant/abcd/function/" + + "0-counter-addone/version/$latest/defaultaz", + want: WorkerInstanceKey{}, + err: true, + }, + { + arg: "/sn/workers/business/yrk/tenant/0/function/function-task/version/$latest/defaultaz/dggphis36581", + want: WorkerInstanceKey{ + TypeKey: "workers", + BusinessID: "yrk", + TenantID: "0", + Function: "function-task", + Version: "$latest", + Zone: "defaultaz", + Instance: "dggphis36581", + }, + err: false, + }, + } + for _, tt := range tests { + worker := WorkerInstanceKey{} + err := worker.ParseFrom(tt.arg) + if tt.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, worker) + } + } +} + +func TestWorkerInstanceKey_String(t *testing.T) { + tests := []struct { + arg WorkerInstanceKey + want string + }{ + { + want: "/sn/workers/business/busid/tenant/abcd/function/" + + "0-counter-addone/version/$latest/defaultaz/defaultaz-#-pool7-500-500-python3.7-58f588848d-smss8", + arg: WorkerInstanceKey{ + TypeKey: "workers", + BusinessID: "busid", + TenantID: "abcd", + Function: "0-counter-addone", + Version: "$latest", + Zone: "defaultaz", + Instance: "defaultaz-#-pool7-500-500-python3.7-58f588848d-smss8", + }, + }, + } + for _, tt := range tests { + assert.Equal(t, tt.want, tt.arg.String()) + } +} + +func TestAnonymizeTenantCommonEtcdKey(t *testing.T) { + keyA := "/yr/functions/business/yrk" + anonymizeKeyA := AnonymizeTenantCommonEtcdKey(keyA) + assert.Equal(t, keyA, anonymizeKeyA) + keyB := "/yr/functions/business/yrk/tenant/8e08d5cc0ad34032bba8d636040a278c/function/0-test1-addone/version/$latest" + anonymizeKeyB := AnonymizeTenantCommonEtcdKey(keyB) + assert.NotEqual(t, keyB, anonymizeKeyB) +} +func TestAnonymizeTenantCronTriggerEtcdKey(t *testing.T) { + keyA := "/sn/triggers/triggerType/CRON/business/yrk/tenant" + anonymizeKeyA := AnonymizeTenantCronTriggerEtcdKey(keyA) + assert.Equal(t, keyA, anonymizeKeyA) + keyB := "/sn/triggers/triggerType/CRON/business/yrk/tenant/i1fe539427b2/function/pytzip/version/$latest/398e2ca2" + anonymizeKeyB := AnonymizeTenantCronTriggerEtcdKey(keyB) + assert.NotEqual(t, keyB, anonymizeKeyB) +} +func TestAnonymizeTenantFunctionMetadataEtcdKey(t *testing.T) { + keyA := "/repo/FunctionVersion/business" + anonymizeKeyA := AnonymizeTenantFunctionMetadataEtcdKey(keyA) + assert.Equal(t, keyA, anonymizeKeyA) + keyB := "/repo/FunctionVersion/business/tenant/funcName/version/" + anonymizeKeyB := AnonymizeTenantFunctionMetadataEtcdKey(keyB) + assert.NotEqual(t, keyB, anonymizeKeyB) +} + +func TestFunctionInstanceKey_ParseFrom(t *testing.T) { + tests := []struct { + arg string + want FunctionInstanceKey + err bool + }{ + { + arg: "/sn/instance/business/yrk/tenant/tenantID/defaultaz/instanceID", + want: FunctionInstanceKey{ + TypeKey: "instance", + BusinessID: "yrk", + TenantID: "tenantID", + InstanceID: "instanceID", + Version: "", + Zone: "defaultaz", + }, + err: false, + }, + { + arg: "/sn/instance/business/b1/tenant/t1/function/0-s1-test/version/1/defaultaz", + want: FunctionInstanceKey{}, + err: true, + }, + } + for _, tt := range tests { + instance := FunctionInstanceKey{} + err := instance.ParseFrom(tt.arg) + if tt.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, instance) + } + } +} diff --git a/functionsystem/apps/meta_service/common/functioncapability/capability.go b/functionsystem/apps/meta_service/common/functioncapability/capability.go new file mode 100644 index 0000000000000000000000000000000000000000..498f20b6ad5b5ddbbc846064357510ded40495a8 --- /dev/null +++ b/functionsystem/apps/meta_service/common/functioncapability/capability.go @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package functioncapability - +package functioncapability + +const ( + // Dynamic support request of dynamic resource (multi functions in one pod) + Dynamic = 2 + // Fusion functionGraph + Fusion = 1 + // Full yuanrong + Full = 0 +) diff --git a/functionsystem/apps/meta_service/common/functionhandler/handler.go b/functionsystem/apps/meta_service/common/functionhandler/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..ccb1a4102421aa2862f72835dc5e73ccc6665709 --- /dev/null +++ b/functionsystem/apps/meta_service/common/functionhandler/handler.go @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package functionhandler + +import ( + "fmt" + "strings" +) + +// handler map +const ( + // InitHandler Name of the initHandler in the hookHandler map + InitHandler = "init" + // CallHandler Name of the callHandler in the hookHandler map + CallHandler = "call" + // CheckpointHandler Name of the checkpointHandler in the hookHandler map + CheckpointHandler = "checkpoint" + // RecoverHandler Name of the recoverHandler in the hookHandler map + RecoverHandler = "recover" + // ShutdownHandler Name of the shutdownHandler in the hookHandler map + ShutdownHandler = "shutdown" + // SignalHandler Name of the signalHandler in the hookHandler map + SignalHandler = "signal" + // ExtendedHandler Name of the handler in the extendedHandler map + ExtendedHandler = "handler" + // ExtendedInitializer Name of the initializer in the extendedHandler map + ExtendedInitializer = "initializer" + // ExtendedPreStop Name of the pre_stop in the extendedHandler map + ExtendedPreStop = "pre_stop" + // ExtendedHandlers name of the handlers in the extendedHandler map + ExtendedHandlers = "extendedHandler" + // ExtendedTimeouts name of the initializer's timeout + ExtendedTimeouts = "extendedTimeout" + + // Faas kind of function creation + Faas = "faas" + // Yrlib kind of function creation + Yrlib = "yrlib" + // PosixCustom kind of function creation + PosixCustom = "posix-runtime-custom" + // Custom kind of function creation + Custom = "custom" + // JavaRuntimePrefix java runtime prefix + JavaRuntimePrefix = "java" + // PythonRuntimePrefix python runtime prefix + PythonRuntimePrefix = "python" + // CppRuntimePrefix cpp runtime prefix + CppRuntimePrefix = "cpp" +) + +const ( + yrlibHandler = "fusion_computation_handler.fusion_computation_handler" + customHandler = "" +) + +// FunctionHookHandlerInfo function hook handler info +type FunctionHookHandlerInfo struct { + InitHandler string + CallHandler string + CheckpointHandler string + RecoverHandler string + ShutdownHandler string + SignalHandler string +} + +// BuildHandlerMap function builder interface +type BuildHandlerMap interface { + Handler() string + HookHandler(runtime string, handlerInfo FunctionHookHandlerInfo) map[string]string + ExtendedHandler(initializer, preStop string) map[string]string + ExtendedTimeout(initializerTimeout, preStopTimeout int) map[string]int +} + +// GetBuilder get builder +func GetBuilder(kind, runtime, handler string) BuildHandlerMap { + switch kind { + case Faas: + return FaasBuilder{faasHandler: handler} + case Yrlib: + return YrlibBuilder{runtime: runtime} + case PosixCustom: + return PosixCustomBuilder{} + case Custom: + return CustomBuilder{} + default: + return nil + } +} + +// FaasBuilder builder for faas functions +type FaasBuilder struct { + faasHandler string +} + +// Handler get handler +func (f FaasBuilder) Handler() string { + return f.faasHandler +} + +// HookHandler get hook handler +func (f FaasBuilder) HookHandler(runtime string, handlerInfo FunctionHookHandlerInfo) map[string]string { + hookHandler := map[string]string{} + if strings.HasPrefix(runtime, PythonRuntimePrefix) { + hookHandler = map[string]string{ + InitHandler: "faas_executor.faasInitHandler", + CallHandler: "faas_executor.faasCallHandler", + CheckpointHandler: "faas_executor.faasCheckPointHandler", + RecoverHandler: "faas_executor.faasRecoverHandler", + ShutdownHandler: "faas_executor.faasShutDownHandler", + SignalHandler: "faas_executor.faasSignalHandler", + } + } else if strings.HasPrefix(runtime, JavaRuntimePrefix) { + hookHandler = map[string]string{ + InitHandler: "com.huawei.faas.handler.FaaSExecutor.faasInitHandler", + CallHandler: "com.huawei.faas.handler.FaaSExecutor.faasCallHandler", + CheckpointHandler: "com.huawei.faas.handler.FaaSExecutor.faasCheckPointHandler", + RecoverHandler: "com.huawei.faas.handler.FaaSExecutor.faasRecoverHandler", + ShutdownHandler: "com.huawei.faas.handler.FaaSExecutor.faasShutDownHandler", + SignalHandler: "com.huawei.faas.handler.FaaSExecutor.faasSignalHandler", + } + } else { + fmt.Printf("faas: language matching error") + } + return hookHandler +} + +// ExtendedHandler get extended handler +func (f FaasBuilder) ExtendedHandler(initializer, preStop string) map[string]string { + extendedHandler := map[string]string{ + ExtendedInitializer: initializer, + ExtendedPreStop: preStop, + } + return cleanMap(extendedHandler) +} + +// ExtendedTimeout get extended timeout +func (f FaasBuilder) ExtendedTimeout(initializerTimeout, preStopTimeout int) map[string]int { + if initializerTimeout == 0 { + return map[string]int{} + } + extendedHandler := map[string]int{ + ExtendedInitializer: initializerTimeout, + } + if preStopTimeout != 0 { + extendedHandler[ExtendedPreStop] = preStopTimeout + } + return extendedHandler +} + +// YrlibBuilder yrlib handler +type YrlibBuilder struct { + runtime string +} + +// ExtendedTimeout get extended timeout +func (y YrlibBuilder) ExtendedTimeout(initializerTimeout, preStopTimeout int) map[string]int { + return map[string]int{} +} + +// Handler get handler +func (y YrlibBuilder) Handler() string { + if strings.HasPrefix(y.runtime, CppRuntimePrefix) { + return "" + } + return yrlibHandler +} + +// HookHandler get hook handler +func (y YrlibBuilder) HookHandler(runtime string, handlerInfo FunctionHookHandlerInfo) map[string]string { + hookHandler := map[string]string{} + if strings.HasPrefix(runtime, JavaRuntimePrefix) { + hookHandler = map[string]string{ + InitHandler: "com.huawei.actortask.handler.ActorTaskExecutor.actorTaskInitHandler", + CallHandler: "com.huawei.actortask.handler.ActorTaskExecutor.actorTaskCallHandler", + CheckpointHandler: "com.huawei.actortask.handler.ActorTaskExecutor.actorTaskCheckPointHandler", + RecoverHandler: "com.huawei.actortask.handler.ActorTaskExecutor.actorTaskRecoverHandler", + ShutdownHandler: "com.huawei.actortask.handler.ActorTaskExecutor.actorTaskShutdownHandler", + SignalHandler: "com.huawei.actortask.handler.ActorTaskExecutor.actorTaskSignalHandler", + } + } else { + hookHandler = map[string]string{ + InitHandler: y.getDefaultHandler(handlerInfo.InitHandler, "yrlib_handler.init"), + CallHandler: y.getDefaultHandler(handlerInfo.CallHandler, "yrlib_handler.call"), + CheckpointHandler: y.getDefaultHandler(handlerInfo.CheckpointHandler, "yrlib_handler.checkpoint"), + RecoverHandler: y.getDefaultHandler(handlerInfo.RecoverHandler, "yrlib_handler.recover"), + ShutdownHandler: y.getDefaultHandler(handlerInfo.ShutdownHandler, "yrlib_handler.shutdown"), + SignalHandler: y.getDefaultHandler(handlerInfo.SignalHandler, "yrlib_handler.signal"), + } + } + return cleanMap(hookHandler) +} + +func (y YrlibBuilder) getDefaultHandler(handler, defaultHandler string) string { + if strings.HasPrefix(y.runtime, CppRuntimePrefix) { + return handler + } + if handler == "" { + return defaultHandler + } + return handler +} + +// ExtendedHandler get extended handler +func (y YrlibBuilder) ExtendedHandler(initializer, preStop string) map[string]string { + return map[string]string{} +} + +// PosixCustomBuilder builder for posix custom +type PosixCustomBuilder struct{} + +// ExtendedTimeout get extended timeout +func (p PosixCustomBuilder) ExtendedTimeout(initializerTimeout, preStopTimeout int) map[string]int { + return map[string]int{} +} + +// Handler get handler +func (p PosixCustomBuilder) Handler() string { + return customHandler +} + +// HookHandler get hook handler +func (p PosixCustomBuilder) HookHandler(runtime string, handlerInfo FunctionHookHandlerInfo) map[string]string { + return map[string]string{} +} + +// ExtendedHandler get extended handler +func (p PosixCustomBuilder) ExtendedHandler(initializer, preStop string) map[string]string { + return map[string]string{} +} +func cleanMap(handlerMap map[string]string) map[string]string { + for key, value := range handlerMap { + if value == "" { + delete(handlerMap, key) + } + } + return handlerMap +} + +// CustomBuilder builder for custom function +type CustomBuilder struct{} + +// ExtendedTimeout get extended timeout +func (c CustomBuilder) ExtendedTimeout(initializerTimeout, preStopTimeout int) map[string]int { + return map[string]int{} +} + +// Handler get handler +func (c CustomBuilder) Handler() string { + return "" +} + +// HookHandler get hook handler +func (c CustomBuilder) HookHandler(runtime string, handlerInfo FunctionHookHandlerInfo) map[string]string { + hookHandler := map[string]string{ + InitHandler: handlerInfo.InitHandler, + CallHandler: handlerInfo.CallHandler, + CheckpointHandler: handlerInfo.CheckpointHandler, + RecoverHandler: handlerInfo.RecoverHandler, + ShutdownHandler: handlerInfo.ShutdownHandler, + SignalHandler: handlerInfo.SignalHandler, + } + return cleanMap(hookHandler) +} + +// ExtendedHandler get extended handler +func (c CustomBuilder) ExtendedHandler(initializer, preStop string) map[string]string { + return map[string]string{} +} diff --git a/functionsystem/apps/meta_service/common/functionhandler/handler_test.go b/functionsystem/apps/meta_service/common/functionhandler/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..bfea110e75d12328c6a4d9d41603cb61e536d2e0 --- /dev/null +++ b/functionsystem/apps/meta_service/common/functionhandler/handler_test.go @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package functionhandler + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildHandle(t *testing.T) { + testCases := []struct { + name string + kind string + runtime string + handler string + preStop string + initializer string + initializerTimeout int + preStopTimeout int + info FunctionHookHandlerInfo + handlerWantStd string + hookHandlerWantNum int + extendedHandlerWantNum int + extendedTimeoutWantNum int + }{ + { + name: "faas kind", + runtime: "python3.8", + handler: "handler.myhandler", + preStop: "", + initializer: "init", + initializerTimeout: 900, + kind: "faas", + info: FunctionHookHandlerInfo{}, + handlerWantStd: "handler.myhandler", + hookHandlerWantNum: 6, + extendedHandlerWantNum: 1, + extendedTimeoutWantNum: 1, + }, + { + name: "yrlib kind", + kind: "yrlib", + runtime: "python3.8", + handler: "handler.myhandler", + info: FunctionHookHandlerInfo{}, + handlerWantStd: "fusion_computation_handler.fusion_computation_handler", + hookHandlerWantNum: 6, + extendedHandlerWantNum: 0, + extendedTimeoutWantNum: 0, + }, + { + name: "posix-runtime-custom kind", + kind: "posix-runtime-custom", + runtime: "python3.8", + handler: "handler.myhandler", + info: FunctionHookHandlerInfo{}, + handlerWantStd: "", + hookHandlerWantNum: 0, + extendedHandlerWantNum: 0, + extendedTimeoutWantNum: 0, + }, + { + name: "custom kind", + kind: "custom", + runtime: "", + handler: "handler.myhandler", + info: FunctionHookHandlerInfo{}, + handlerWantStd: "", + hookHandlerWantNum: 0, + extendedHandlerWantNum: 0, + extendedTimeoutWantNum: 0, + }, + { + name: "cpp11 kind", + kind: "yrlib", + runtime: "cpp11", + handler: "handler.myhandler", + info: FunctionHookHandlerInfo{}, + handlerWantStd: "", + hookHandlerWantNum: 0, + extendedHandlerWantNum: 0, + extendedTimeoutWantNum: 0, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + mapBuilder := GetBuilder(tt.kind, tt.runtime, tt.handler) + handler := mapBuilder.Handler() + assert.Equal(t, tt.handlerWantStd, handler) + hookHandler := mapBuilder.HookHandler(tt.runtime, tt.info) + assert.Equal(t, tt.hookHandlerWantNum, len(hookHandler)) + extendedHandler := mapBuilder.ExtendedHandler(tt.initializer, tt.preStop) + assert.Equal(t, tt.extendedHandlerWantNum, len(extendedHandler)) + extendedTimeout := mapBuilder.ExtendedTimeout(tt.initializerTimeout, tt.preStopTimeout) + assert.Equal(t, tt.extendedTimeoutWantNum, len(extendedTimeout)) + }) + } +} diff --git a/functionsystem/apps/meta_service/common/healthcheck/server.go b/functionsystem/apps/meta_service/common/healthcheck/server.go new file mode 100644 index 0000000000000000000000000000000000000000..7c6c4980ff04970c001675a50fb00987a4cbd510 --- /dev/null +++ b/functionsystem/apps/meta_service/common/healthcheck/server.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package healthcheck implements a common health check server +package healthcheck + +import ( + "crypto/tls" + "net/http" + + "meta_service/common/logger/log" + + "github.com/gin-gonic/gin" + + commontls "meta_service/common/tls" +) + +const port = "8090" + +// k8s https livenessProbe + +// StartServe start health check server for k8s +func StartServe(ip string, f func(c *gin.Context)) error { + engine := gin.New() + engine.Use(gin.Recovery()) + + checkFunc := healthCheck + if f != nil { + checkFunc = f + } + + engine.GET("/healthz", checkFunc) + + server := http.Server{ + Addr: ip + ":" + port, + Handler: engine, + } + + err := server.ListenAndServe() + if err != nil { + log.GetLogger().Errorf("failed to start health check server %s", err.Error()) + return err + } + + log.GetLogger().Infof("start health check server at %s", ip) + return nil +} + +// StartServeTLS start tls health check server for k8s +func StartServeTLS(ip, certFile, keyFile, passPhase string, f func(c *gin.Context)) error { + engine := gin.New() + engine.Use(gin.Recovery()) + + checkFunc := healthCheck + if f != nil { + checkFunc = f + } + engine.GET("/healthz", checkFunc) + + healthConf := commontls.NewTLSConfig(commontls.WithClientAuthType(tls.NoClientCert), + commontls.WithCertsByEncryptedKey(certFile, keyFile, passPhase)) + + server := http.Server{ + Addr: ip + ":" + port, + Handler: engine, + TLSConfig: healthConf, + } + + err := server.ListenAndServeTLS("", "") + if err != nil { + log.GetLogger().Errorf("failed to start health check server %s", err.Error()) + return err + } + + log.GetLogger().Infof("start health check tls server at %s", ip) + return nil +} + +func healthCheck(c *gin.Context) { + return +} diff --git a/functionsystem/apps/meta_service/common/healthcheck/server_test.go b/functionsystem/apps/meta_service/common/healthcheck/server_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4ea91ab84a88cf03faa7ad37317193da884b82c9 --- /dev/null +++ b/functionsystem/apps/meta_service/common/healthcheck/server_test.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package healthcheck + +import ( + "github.com/gin-gonic/gin" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const waitTime = 500 + +func TestStartServeCase1(t *testing.T) { + go StartServe("127.0.0.1", nil) + tchan := time.After(waitTime * time.Millisecond) + <-tchan + + rsp, err := http.Get("http://localhost:8090/healthz") + defer rsp.Body.Close() + assert.Nil(t, err, "get error ", err) + assert.Equal(t, http.StatusOK, rsp.StatusCode, "resp status: ", rsp.Status) + + err = StartServeTLS("127.0.0.1", "", "", "", nil) + assert.NotNil(t, err, "err is nil") +} + +func TestStartServeCase2(t *testing.T) { + testFunc := func(c *gin.Context) {} + go StartServe("127.0.0", testFunc) +} diff --git a/functionsystem/apps/meta_service/common/job/config.go b/functionsystem/apps/meta_service/common/job/config.go new file mode 100644 index 0000000000000000000000000000000000000000..0525d968a529bc594fb81472f88604f519ac4047 --- /dev/null +++ b/functionsystem/apps/meta_service/common/job/config.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package job - +package job + +import "meta_service/common/constants" + +// 处理job的对外接口 +const ( + PathParamSubmissionId = "submissionId" + PathGroupJobs = "/api/jobs" + PathGetJobs = constants.DynamicRouterParamPrefix + PathParamSubmissionId + PathDeleteJobs = constants.DynamicRouterParamPrefix + PathParamSubmissionId + PathStopJobs = constants.DynamicRouterParamPrefix + PathParamSubmissionId + "/stop" +) + +const ( + submissionIdPattern = "^[a-z0-9-]{1,64}$" + jobIDPrefix = "app-" + tenantIdKey = "tenantId" +) + +// Response - +type Response struct { + Code int `form:"code" json:"code"` + Message string `form:"message" json:"message"` + Data []byte `form:"data" json:"data"` +} + +// SubmitRequest is SubmitRequest struct +type SubmitRequest struct { + Entrypoint string `form:"entrypoint" json:"entrypoint"` + SubmissionId string `form:"submission_id" json:"submission_id"` + RuntimeEnv *RuntimeEnv `form:"runtime_env" json:"runtime_env" valid:"optional"` + Metadata map[string]string `form:"metadata" json:"metadata" valid:"optional"` + Labels string `form:"labels" json:"labels" valid:"optional"` + CreateOptions map[string]string `form:"createOptions" json:"createOptions" valid:"optional"` + EntrypointResources map[string]float64 `form:"entrypoint_resources" json:"entrypoint_resources" valid:"optional"` + EntrypointNumCpus float64 `form:"entrypoint_num_cpus" json:"entrypoint_num_cpus" valid:"optional"` + EntrypointNumGpus float64 `form:"entrypoint_num_gpus" json:"entrypoint_num_gpus" valid:"optional"` + EntrypointMemory int `form:"entrypoint_memory" json:"entrypoint_memory" valid:"optional"` +} + +// RuntimeEnv args of invoking create_app +type RuntimeEnv struct { + WorkingDir string `form:"working_dir" json:"working_dir" valid:"optional"` + Pip []string `form:"pip" json:"pip" valid:"optional" ` + EnvVars map[string]string `form:"env_vars" json:"env_vars" valid:"optional"` +} diff --git a/functionsystem/apps/meta_service/common/job/handler.go b/functionsystem/apps/meta_service/common/job/handler.go new file mode 100644 index 0000000000000000000000000000000000000000..f297066765daaa9eb13adbbad63cd7c61bb9d6bd --- /dev/null +++ b/functionsystem/apps/meta_service/common/job/handler.go @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package job - +package job + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "net/http" + "regexp" + "strings" + + "meta_service/common/constants" + "meta_service/common/httputil/utils" + "meta_service/common/logger/log" + "meta_service/common/uuid" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +// SubmitJobHandleReq - +func SubmitJobHandleReq(ctx *gin.Context) *SubmitRequest { + traceID := ctx.Request.Header.Get(constants.HeaderTraceID) + logger := log.GetLogger().With(zap.Any("traceID", traceID)) + var req SubmitRequest + if err := ctx.ShouldBind(&req); err != nil { + logger.Errorf("shouldBind SubmitJob request failed, err: %s", err) + ctx.JSON(http.StatusBadRequest, fmt.Sprintf("shouldBind SubmitJob request failed, err: %v", err)) + return nil + } + err := req.CheckField() + if err != nil { + ctx.JSON(http.StatusBadRequest, err.Error()) + return nil + } + req.EntrypointNumCpus = math.Ceil(req.EntrypointNumCpus * constants.CpuUnitConvert) + req.EntrypointMemory = int(math.Ceil(float64(req.EntrypointMemory) / constants.MemoryUnitConvert / constants.MemoryUnitConvert)) + reqHeader := utils.ParseHeader(ctx) + if tenantId, ok := reqHeader[constants.HeaderTenantId]; ok { + req.AddCreateOptions(tenantIdKey, tenantId) + } + if labels, ok := reqHeader[constants.HeaderPoolLabel]; ok { + req.Labels = labels + } + logger.Debugf("SubmitJob createApp start, req:%#v", req) + return &req +} + +// SubmitJobHandleRes - +// SubmitJob godoc +// @Summary submit job +// @Description submit a new job +// @Accept json +// @Produce json +// @Router /api/jobs [POST] +// @Param SubmitRequest body SubmitRequest true "提交job时定义的job信息。" +// @Success 200 {object} map[string]string "提交job成功，返回该job的submission_id" +// @Failure 400 {string} string "用户请求错误，包含错误信息" +// @Failure 404 {string} string "该job已经存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func SubmitJobHandleRes(ctx *gin.Context, resp Response) { + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + var result map[string]string + err := json.Unmarshal(resp.Data, &result) + if err != nil { + ctx.JSON(http.StatusBadRequest, + fmt.Sprintf("unmarshal response data failed, data: %v", resp.Data)) + return + } + ctx.JSON(http.StatusOK, result) + log.GetLogger().Debugf("SubmitJobHandleRes succeed, submission_id: %s", result) +} + +// ListJobsHandleRes - +// ListJobs godoc +// @Summary List Jobs +// @Description list jobs with jobInfo +// @Accept json +// @Produce json +// @Router /api/jobs [GET] +// @Success 200 {array} constant.AppInfo "返回所有jobs的信息" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func ListJobsHandleRes(ctx *gin.Context, resp Response) { + traceID := ctx.Request.Header.Get(constants.HeaderTraceID) + logger := log.GetLogger().With(zap.Any("traceID", traceID)) + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + var result []*constants.AppInfo + err := json.Unmarshal(resp.Data, &result) + if err != nil { + ctx.JSON(http.StatusBadRequest, + fmt.Sprintf("unmarshal response data failed, data: %v", resp.Data)) + return + } + ctx.JSON(http.StatusOK, result) + logger.Debugf("ListJobsHandleRes succeed") +} + +// GetJobInfoHandleRes - +// GetJobInfo godoc +// @Summary Get JobInfo +// @Description get jobInfo by submission_id +// @Accept json +// @Produce json +// @Router /api/jobs/{submissionId} [GET] +// @Param submissionId path string true "job的submission_id，以'app-'开头" +// @Success 200 {object} constant.AppInfo "返回submission_id对应的job信息" +// @Failure 404 {string} string "该job不存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func GetJobInfoHandleRes(ctx *gin.Context, resp Response) { + submissionId := ctx.Param(PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + var result *constants.AppInfo + err := json.Unmarshal(resp.Data, &result) + if err != nil { + ctx.JSON(http.StatusBadRequest, + fmt.Sprintf("unmarshal response data failed, data: %v", resp.Data)) + return + } + ctx.JSON(http.StatusOK, result) + logger.Debugf("GetJobInfoHandleRes succeed") +} + +// DeleteJobHandleRes - +// DeleteJob godoc +// @Summary Delete Job +// @Description delete job by submission_id +// @Accept json +// @Produce json +// @Router /api/jobs/{submissionId} [DELETE] +// @Param submissionId path string true "job的submission_id，以'app-'开头" +// @Success 200 {boolean} bool "返回true则说明可以删除对应的job，返回false则说明无法删除job" +// @Failure 403 {string} string "禁止删除job，包含错误信息和job运行状态" +// @Failure 404 {string} string "该job不存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func DeleteJobHandleRes(ctx *gin.Context, resp Response) { + submissionId := ctx.Param(PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + if resp.Code == http.StatusForbidden { + log.GetLogger().Errorf("forbidden to delete, status: %s", resp.Data) + ctx.JSON(http.StatusOK, false) + return + } + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + ctx.JSON(http.StatusOK, true) + logger.Debugf("DeleteJobHandleRes succeed") +} + +// StopJobHandleRes - +// StopJob godoc +// @Summary Stop Job +// @Description stop job by submission_id +// @Accept json +// @Produce json +// @Router /api/jobs/{submissionId}/stop [POST] +// @Param submissionId path string true "job的submission_id，以'app-'开头" +// @Success 200 {boolean} bool "返回true表示可以停止运行对应的job，返回false表示job当前状态不能被停止" +// @Failure 403 {string} string "禁止删除job，包含错误信息和job运行状态" +// @Failure 404 {string} string "该job不存在" +// @Failure 500 {string} string "服务器处理错误，包含错误信息" +func StopJobHandleRes(ctx *gin.Context, resp Response) { + submissionId := ctx.Param(PathParamSubmissionId) + logger := log.GetLogger().With(zap.Any("SubmissionId", submissionId)) + if resp.Code == http.StatusForbidden { + log.GetLogger().Errorf("forbidden to stop job, status: %s", resp.Data) + ctx.JSON(http.StatusOK, false) + return + } + if resp.Code != http.StatusOK || resp.Message != "" { + ctx.JSON(resp.Code, resp.Message) + return + } + ctx.JSON(http.StatusOK, true) + logger.Debugf("StopJobHandleRes succeed") +} + +// CheckField - +func (req *SubmitRequest) CheckField() error { + if req.Entrypoint == "" { + log.GetLogger().Errorf("entrypoint should not be empty") + return fmt.Errorf("entrypoint should not be empty") + } + if req.RuntimeEnv == nil || req.RuntimeEnv.WorkingDir == "" { + log.GetLogger().Errorf("runtime_env.working_dir should not be empty") + return fmt.Errorf("runtime_env.working_dir should not be empty") + } + if err := req.ValidateResources(); err != nil { + log.GetLogger().Errorf("validateResources error: %s", err.Error()) + return err + } + if err := req.CheckSubmissionId(); err != nil { + log.GetLogger().Errorf("chechk submission_id: %s, error: %s", req.SubmissionId, err.Error()) + return err + } + return nil +} + +// ValidateResources - +func (req *SubmitRequest) ValidateResources() error { + if req.EntrypointNumCpus < 0 { + return errors.New("entrypoint_num_cpus should not be less than 0") + } + if req.EntrypointNumGpus < 0 { + return errors.New("entrypoint_num_gpus should not be less than 0") + } + if req.EntrypointMemory < 0 { + return errors.New("entrypoint_memory should not be less than 0") + } + return nil +} + +// CheckSubmissionId - +func (req *SubmitRequest) CheckSubmissionId() error { + if req.SubmissionId == "" { + return nil + } + if strings.Contains(req.SubmissionId, "driver") { + return errors.New("submission_id should not contain 'driver'") + } + if !strings.HasPrefix(req.SubmissionId, jobIDPrefix) { + req.SubmissionId = jobIDPrefix + req.SubmissionId + } + isMatch, err := regexp.MatchString(submissionIdPattern, req.SubmissionId) + if err != nil || !isMatch { + return fmt.Errorf("regular expression validation error, submissionId: %s, pattern: %s, err: %v", + req.SubmissionId, submissionIdPattern, err) + } + return nil +} + +// NewSubmissionID - +func (req *SubmitRequest) NewSubmissionID() { + if req.SubmissionId == "" { + req.SubmissionId = jobIDPrefix + uuid.New().String() + } +} + +// AddCreateOptions - +func (req *SubmitRequest) AddCreateOptions(key, value string) { + if req.CreateOptions == nil { + req.CreateOptions = map[string]string{} + } + if key != "" { + req.CreateOptions[key] = value + } +} + +// BuildJobResponse - +func BuildJobResponse(data any, code int, err error) Response { + dataBytes, jsonErr := json.Marshal(data) + if jsonErr != nil { + return Response{ + Code: http.StatusInternalServerError, + Message: fmt.Sprintf("marshal job response failed, err: %v", jsonErr), + } + } + var resp Response + resp.Code = code + if data != nil { + resp.Data = dataBytes + } + if err != nil { + resp.Message = err.Error() + } + return resp +} diff --git a/functionsystem/apps/meta_service/common/job/handler_test.go b/functionsystem/apps/meta_service/common/job/handler_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4b74bda264f36460cf863b302322f1f5c02d777f --- /dev/null +++ b/functionsystem/apps/meta_service/common/job/handler_test.go @@ -0,0 +1,587 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package job + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + + "meta_service/common/constants" + "meta_service/common/faas_common/constant" + + "github.com/agiledragon/gomonkey/v2" + "github.com/gin-gonic/gin" + "github.com/smartystreets/goconvey/convey" +) + +func TestSubmitJobHandleReq(t *testing.T) { + convey.Convey("test DeleteJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + bodyBytes, _ := json.Marshal(SubmitRequest{ + Entrypoint: "", + SubmissionId: "", + RuntimeEnv: &RuntimeEnv{ + WorkingDir: "", + Pip: []string{""}, + EnvVars: map[string]string{}, + }, + Metadata: map[string]string{}, + EntrypointResources: map[string]float64{}, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + }) + reader := bytes.NewBuffer(bodyBytes) + c.Request = &http.Request{ + Method: "POST", + URL: &url.URL{Path: PathGroupJobs}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + constants.HeaderTenantID: []string{"123456"}, + constants.HeaderPoolLabel: []string{"abc"}, + }, + Body: io.NopCloser(reader), // 使用 io.NopCloser 包装 reader，使其满足 io.ReadCloser 接口 + } + convey.Convey("when process success", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckField", func() error { + return nil + }).Reset() + expectedResult := &SubmitRequest{ + Entrypoint: "", + SubmissionId: "", + RuntimeEnv: &RuntimeEnv{ + WorkingDir: "", + Pip: []string{""}, + EnvVars: map[string]string{}, + }, + Metadata: map[string]string{}, + Labels: "abc", + CreateOptions: map[string]string{ + "tenantId": "123456", + }, + EntrypointResources: map[string]float64{}, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + } + result := SubmitJobHandleReq(c) + convey.So(result, convey.ShouldResemble, expectedResult) + }) + convey.Convey("when CheckField failed", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckField", func() error { + return errors.New("failed CheckField") + }).Reset() + result := SubmitJobHandleReq(c) + convey.So(result, convey.ShouldBeNil) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed CheckField\"") + }) + }) +} + +func TestSubmitJobHandleRes(t *testing.T) { + convey.Convey("test SubmitJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: []byte("app-123"), + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusNotFound), func() { + resp.Code = http.StatusNotFound + resp.Message = fmt.Sprintf("not found job") + SubmitJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusNotFound) + convey.So(w.Body.String(), convey.ShouldEqual, "\"not found job\"") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + resp.Code = http.StatusInternalServerError + resp.Message = fmt.Sprintf("failed get job") + SubmitJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusInternalServerError) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed get job\"") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + SubmitJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed, data:") + }) + convey.Convey("when process success", func() { + marshal, err := json.Marshal(map[string]string{ + "submission_id": "app-123", + }) + resp.Data = marshal + convey.So(err, convey.ShouldBeNil) + SubmitJobHandleRes(c, resp) + expectedResult, err := json.Marshal(map[string]string{ + "submission_id": "app-123", + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldResemble, string(expectedResult)) + }) + }) +} + +func TestListJobsHandleRes(t *testing.T) { + convey.Convey("test ListJobsHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = &http.Request{ + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + } + dataBytes, err := json.Marshal([]*constant.AppInfo{ + { + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }, + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: dataBytes, + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + resp.Code = http.StatusInternalServerError + resp.Message = fmt.Sprintf("failed get job") + ListJobsHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusInternalServerError) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed get job\"") + }) + convey.Convey("when unmarshal response data failed", func() { + resp.Data = []byte(",aa,") + ListJobsHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed") + }) + convey.Convey("when response data is nil", func() { + resp.Data = []byte("[]") + ListJobsHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "[]") + }) + convey.Convey("when process success", func() { + ListJobsHandleRes(c, resp) + expectedResult, err := json.Marshal([]*constant.AppInfo{ + { + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }, + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldResemble, string(expectedResult)) + }) + }) +} + +func TestGetJobInfoHandleRes(t *testing.T) { + convey.Convey("test GetJobInfoHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + dataBytes, err := json.Marshal(&constant.AppInfo{ + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: dataBytes, + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusNotFound), func() { + resp.Code = http.StatusNotFound + resp.Message = fmt.Sprintf("not found job") + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusNotFound) + convey.So(w.Body.String(), convey.ShouldEqual, "\"not found job\"") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusInternalServerError), func() { + resp.Code = http.StatusInternalServerError + resp.Message = fmt.Sprintf("failed get job") + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusInternalServerError) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed get job\"") + }) + convey.Convey("when unmarshal response data failed", func() { + resp.Data = []byte(",aa,") + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + GetJobInfoHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldStartWith, "\"unmarshal response data failed") + }) + convey.Convey("when process success", func() { + GetJobInfoHandleRes(c, resp) + expectedResult, err := json.Marshal(&constant.AppInfo{ + Type: "SUBMISSION", + Entrypoint: "python script.py", + SubmissionID: "app-123", + }) + if err != nil { + t.Errorf("marshal expected result failed, err: %v", err) + } + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldResemble, string(expectedResult)) + }) + }) +} + +func TestDeleteJobHandleRes(t *testing.T) { + convey.Convey("test DeleteJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: []byte("SUCCEEDED"), + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusForbidden), func() { + resp.Code = http.StatusForbidden + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusBadRequest), func() { + resp.Code = http.StatusBadRequest + resp.Message = fmt.Sprintf("failed delete job") + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed delete job\"") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + convey.Convey("when process success", func() { + DeleteJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + }) +} + +func TestStopJobHandleRes(t *testing.T) { + convey.Convey("test StopJobHandleRes", t, func() { + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + resp := Response{ + Code: http.StatusOK, + Message: "", + Data: []byte(`SUCCEEDED`), + } + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusForbidden), func() { + resp.Code = http.StatusForbidden + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "false") + }) + convey.Convey("when statusCode is "+strconv.Itoa(http.StatusBadRequest), func() { + resp.Code = http.StatusBadRequest + resp.Message = fmt.Sprintf("failed stop job") + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusBadRequest) + convey.So(w.Body.String(), convey.ShouldEqual, "\"failed stop job\"") + }) + convey.Convey("when response data is nil", func() { + resp.Data = nil + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + convey.Convey("when process success", func() { + StopJobHandleRes(c, resp) + convey.So(w.Code, convey.ShouldEqual, http.StatusOK) + convey.So(w.Body.String(), convey.ShouldEqual, "true") + }) + }) +} + +func TestSubmitRequest_CheckField(t *testing.T) { + convey.Convey("test (req *SubmitRequest) CheckField", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "", + RuntimeEnv: &RuntimeEnv{ + WorkingDir: "file:///home/disk/tk/file.zip", + Pip: []string{"numpy==1.24", "scipy==1.11.0"}, + EnvVars: map[string]string{ + "SOURCE_REGION": "suzhou_std", + }, + }, + Metadata: map[string]string{ + "autoscenes_ids": "auto_1-test", + "task_type": "task_1", + "ttl": "1250", + }, + EntrypointResources: map[string]float64{ + "NPU": 0, + }, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + } + convey.Convey("when req.Entrypoint is empty", func() { + req.Entrypoint = "" + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("entrypoint should not be empty")) + }) + convey.Convey("when req.RuntimeEnv is empty", func() { + req.RuntimeEnv = nil + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("runtime_env.working_dir should not be empty")) + }) + convey.Convey("when req.RuntimeEnv.WorkingDir is empty", func() { + req.RuntimeEnv.WorkingDir = "" + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("runtime_env.working_dir should not be empty")) + }) + convey.Convey("when ValidateResources failed", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "ValidateResources", func() error { + return errors.New("failed ValidateResources") + }).Reset() + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckSubmissionId", func() error { + return nil + }).Reset() + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("failed ValidateResources")) + }) + convey.Convey("when CheckSubmissionId failed", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "ValidateResources", func() error { + return nil + }).Reset() + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckSubmissionId", func() error { + return errors.New("failed CheckSubmissionId") + }).Reset() + err := req.CheckField() + convey.So(err, convey.ShouldBeError, errors.New("failed CheckSubmissionId")) + }) + convey.Convey("when process success", func() { + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "ValidateResources", func() error { + return nil + }).Reset() + defer gomonkey.ApplyMethodFunc(&SubmitRequest{}, "CheckSubmissionId", func() error { + return nil + }).Reset() + err := req.CheckField() + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func TestSubmitRequest_ValidateResources(t *testing.T) { + convey.Convey("test (req *SubmitRequest) ValidateResources()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "", + EntrypointResources: map[string]float64{ + "NPU": 0, + }, + EntrypointNumCpus: 0, + EntrypointNumGpus: 0, + EntrypointMemory: 0, + } + convey.Convey("when req.EntrypointNumCpus < 0", func() { + req.EntrypointNumCpus = -0.1 + err := req.ValidateResources() + convey.So(err.Error(), convey.ShouldEqual, "entrypoint_num_cpus should not be less than 0") + }) + convey.Convey("when req.EntrypointNumGpus < 0", func() { + req.EntrypointNumGpus = -0.1 + err := req.ValidateResources() + convey.So(err.Error(), convey.ShouldEqual, "entrypoint_num_gpus should not be less than 0") + }) + convey.Convey("when req.EntrypointMemory < 0", func() { + req.EntrypointMemory = -1 + err := req.ValidateResources() + convey.So(err.Error(), convey.ShouldEqual, "entrypoint_memory should not be less than 0") + }) + convey.Convey("when process success", func() { + err := req.ValidateResources() + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func TestSubmitRequest_CheckSubmissionId(t *testing.T) { + convey.Convey("test (req *SubmitRequest) CheckSubmissionId()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "123", + } + convey.Convey("when req.SubmissionId is empty", func() { + req.SubmissionId = "" + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("when req.SubmissionId start with driver", func() { + req.SubmissionId = "driver-123" + err := req.CheckSubmissionId() + convey.So(err.Error(), convey.ShouldEqual, "submission_id should not contain 'driver'") + }) + convey.Convey("when req.SubmissionId doesn't start with 'app-'", func() { + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is 60 without 'app-'", func() { + req.SubmissionId = "023456781234567822345678323456784234567852345678623456787234" + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is more than 60 without 'app-'", func() { + req.SubmissionId = "0234567812345678223456783234567842345678523456786234567872345" + err := req.CheckSubmissionId() + convey.So(err.Error(), convey.ShouldStartWith, "regular expression validation error,") + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is 64 with 'app-'", func() { + req.SubmissionId = "app-023456781234567822345678323456784234567852345678623456787234" + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when req.SubmissionId length is more than 64 with 'app-'", func() { + req.SubmissionId = "app-0234567812345678223456783234567842345678523456786234567872345" + err := req.CheckSubmissionId() + convey.So(err.Error(), convey.ShouldStartWith, "regular expression validation error,") + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + convey.Convey("when process success", func() { + err := req.CheckSubmissionId() + convey.So(err, convey.ShouldBeNil) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + }) +} + +func TestSubmitRequest_NewSubmissionID(t *testing.T) { + convey.Convey("test (req *SubmitRequest) NewSubmissionID()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "", + } + convey.Convey("when req.SubmissionId is empty", func() { + req.NewSubmissionID() + convey.So(req.SubmissionId, convey.ShouldNotBeEmpty) + convey.So(req.SubmissionId, convey.ShouldStartWith, jobIDPrefix) + }) + }) +} + +func TestSubmitRequest_AddCreateOptions(t *testing.T) { + convey.Convey("test (req *SubmitRequest) AddCreateOptions()", t, func() { + req := &SubmitRequest{ + Entrypoint: "python script.py", + SubmissionId: "123", + } + convey.Convey("when req.CreateOptions is empty", func() { + req.AddCreateOptions("key", "value") + convey.So(len(req.CreateOptions), convey.ShouldEqual, 1) + }) + convey.Convey("when key is empty", func() { + req.AddCreateOptions("", "value") + convey.So(len(req.CreateOptions), convey.ShouldEqual, 0) + }) + convey.Convey("when key is not empty", func() { + req.AddCreateOptions("key", "value") + convey.So(len(req.CreateOptions), convey.ShouldEqual, 1) + }) + }) +} + +func TestBuildJobResponse(t *testing.T) { + convey.Convey("test BuildJobResponse", t, func() { + convey.Convey("when process success", func() { + expectedResult := Response{ + Code: 0, + Message: "", + Data: []byte("test"), + } + result := BuildJobResponse("test", 0, nil) + convey.So(result.Code, convey.ShouldEqual, expectedResult.Code) + convey.So(result.Message, convey.ShouldEqual, expectedResult.Message) + convey.So(string(result.Data), convey.ShouldEqual, "\""+string(expectedResult.Data)+"\"") + }) + convey.Convey("when data is nil", func() { + expectedResult := Response{ + Code: http.StatusOK, + Message: "", + Data: nil, + } + result := BuildJobResponse(nil, http.StatusOK, nil) + convey.So(result, convey.ShouldResemble, expectedResult) + }) + convey.Convey("when response status is "+strconv.Itoa(http.StatusBadRequest), func() { + expectedResult := Response{ + Code: http.StatusBadRequest, + Message: "error request", + Data: nil, + } + result := BuildJobResponse(nil, http.StatusBadRequest, errors.New("error request")) + convey.So(result, convey.ShouldResemble, expectedResult) + }) + convey.Convey("when data marshal failed", func() { + expectedResult := Response{ + Code: http.StatusInternalServerError, + Message: "marshal job response failed, err:", + } + result := BuildJobResponse(func() {}, http.StatusOK, nil) + convey.So(result.Code, convey.ShouldEqual, expectedResult.Code) + convey.So(result.Message, convey.ShouldStartWith, expectedResult.Message) + convey.So(result.Data, convey.ShouldBeNil) + }) + }) +} diff --git a/functionsystem/apps/meta_service/common/logger/config/config.go b/functionsystem/apps/meta_service/common/logger/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..7f184665f6a0e338f538fbc6a4be057b3890f78f --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/config/config.go @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is common logger client +package config + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + + "meta_service/common/utils" + + "github.com/asaskevich/govalidator/v11" +) + +const ( + fileMode = 0o750 +) + +// defaultCoreInfo default logger config +var defaultCoreInfo = CoreInfo{ + FilePath: "/home/sn/log", + Level: "INFO", + Rolling: &RollingInfo{ + MaxSize: 400, // Log file reaches maxSize should be compressed with unit MB + MaxBackups: 1, // Max compressed file nums + MaxAge: 1, // Max Age of old log files with unit days + Compress: true, // Determines if the log files should be compressed + }, + Tick: 10, // Unit: Second + First: 10, // Unit: Number of logs + Thereafter: 5, // Unit: Number of logs + Tracing: false, // tracing log switch + Disable: false, // Disable file logger +} + +// RollingInfo contains log rolling configurations +type RollingInfo struct { + MaxSize int `json:"maxsize" valid:",required"` + MaxBackups int `json:"maxbackups" valid:",required"` + MaxAge int `json:"maxage" valid:",required"` + Compress bool `json:"compress" valid:",required"` +} + +// CoreInfo contains the core info +type CoreInfo struct { + Rolling *RollingInfo `json:"rolling" valid:",optional"` + FilePath string `json:"filepath" valid:",required"` + Level string `json:"level" valid:",required"` + Tick int `json:"tick" valid:"range(0|86400),optional"` + First int `json:"first" valid:"range(0|20000),optional"` + Thereafter int `json:"thereafter" valid:"range(0|1000),optional"` + Tracing bool `json:"tracing" valid:",optional"` + Disable bool `json:"disable" valid:",optional"` +} + +// GetCoreInfo get logger config by read log.json file +func GetCoreInfo(configFile string) (CoreInfo, error) { + var info CoreInfo + data, err := ioutil.ReadFile(configFile) + if os.IsNotExist(err) { + return defaultCoreInfo, nil + } + if err != nil { + return defaultCoreInfo, err + } + err = json.Unmarshal(data, &info) + if err != nil { + return defaultCoreInfo, err + } + // if file path is empty return error + // if log file is not writable + // zap will create a new file with file path and file name + if info.FilePath == "" { + return defaultCoreInfo, errors.New("log file path is empty") + } + if _, err := govalidator.ValidateStruct(info); err != nil { + return defaultCoreInfo, err + } + if err := utils.ValidateFilePath(info.FilePath); err != nil { + return defaultCoreInfo, err + } + if err := os.MkdirAll(info.FilePath, fileMode); err != nil && !os.IsExist(err) { + return defaultCoreInfo, err + } + + return info, nil +} + +// GetCoreInfoByParam get logger config by read log.json file +func GetCoreInfoByParam(configFile string) (CoreInfo, error) { + var coreInfo CoreInfo + data, err := ioutil.ReadFile(configFile) + if os.IsNotExist(err) { + return defaultCoreInfo, nil + } + if err != nil { + return defaultCoreInfo, err + } + err = json.Unmarshal(data, &coreInfo) + if err != nil { + return defaultCoreInfo, err + } + // if file path is empty return error + // if log file is not writable + // zap will create a new file with file path and file name + if coreInfo.FilePath == "" { + return defaultCoreInfo, errors.New("log file path is empty") + } + if _, err := govalidator.ValidateStruct(coreInfo); err != nil { + return defaultCoreInfo, err + } + if err := utils.ValidateFilePath(coreInfo.FilePath); err != nil { + return defaultCoreInfo, err + } + if err := os.MkdirAll(coreInfo.FilePath, fileMode); err != nil && !os.IsExist(err) { + return defaultCoreInfo, err + } + + return coreInfo, nil +} + +// GetDefaultCoreInfo get defaultCoreInfo +func GetDefaultCoreInfo() CoreInfo { + return defaultCoreInfo +} diff --git a/functionsystem/apps/meta_service/common/logger/config/config_test.go b/functionsystem/apps/meta_service/common/logger/config/config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9f25bbb675a29ec0d8fbb5fcd5a0b373cfbc5de3 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/config/config_test.go @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config is common logger client +package config + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/goconvey/convey" +) + +func TestInitConfig(t *testing.T) { + convey.Convey("TestInitConfig", t, func() { + convey.Convey("test 1", func() { + resourcePath := os.Getenv("ResourcePath") + coreInfo, err := GetCoreInfo(filepath.Join(resourcePath, "../log.json")) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldEqual, nil) + }) + }) +} + +func TestInitConfigWithReadFileError(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return nil, errors.New("mock read file error") + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfo("/home/sn/config/log.json") + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestInitConfigWithErrorJson(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + mockErrorJson := "{\n\"filepath\": \"/home/sn/mock\",\n\"level\": \"INFO\",\n\"maxsize\": " + + "500,\n\"maxbackups\": 1,\n\"maxage\": 1,\n\"compress\": true\n" + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return []byte(mockErrorJson), nil + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfo("/home/sn/config/log.json") + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestInitConfigWithEmptyPath(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + mockCfgInfo := "{\n\"filepath\": \"\",\n\"level\": \"INFO\",\n\"maxsize\": " + + "500,\n\"maxbackups\": 1,\n\"maxage\": 1,\n\"compress\": true\n}" + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return []byte(mockCfgInfo), nil + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfo("/home/sn/config/log.json") + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestInitConfigWithValidateError(t *testing.T) { + convey.Convey("TestInitConfigWithEmptyPath", t, func() { + convey.Convey("test 1", func() { + mockErrorJson := "{\n\"filepath\": \"some_relative_path\",\n\"level\": \"INFO\",\n\"maxsize\": " + + "500,\n\"maxbackups\": 1,\n\"maxage\": 1}" + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, + func(filename string) ([]byte, error) { + return []byte(mockErrorJson), nil + }), + } + defer func() { + for _, p := range patches { + p.Reset() + } + }() + coreInfo, err := GetCoreInfo("/home/sn/config/log.json") + fmt.Printf("error:%s\n", err) + fmt.Printf("log config:%+v\n", coreInfo) + convey.So(err, convey.ShouldNotEqual, nil) + }) + }) +} + +func TestGetDefaultCoreInfo(t *testing.T) { + tests := []struct { + name string + want CoreInfo + }{ + { + name: "test001", + want: CoreInfo{ + FilePath: "/home/sn/log", + Level: "INFO", + Rolling: &RollingInfo{ + MaxSize: 400, + MaxBackups: 1, + MaxAge: 1, + Compress: true, + }, + Tick: 10, // Unit: Second + First: 10, // Unit: Number of logs + Thereafter: 5, // Unit: Number of logs + Tracing: false, // tracing log switch + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetDefaultCoreInfo(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetDefaultCoreInfo() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetCoreInfoByParam(t *testing.T) { + GetCoreInfoByParam("/home/sn/config/log.json") + + patch1 := gomonkey.ApplyFunc(ioutil.ReadFile, func(string) ([]byte, error) { + return nil, errors.New("test") + }) + GetCoreInfoByParam("") + patch1.Reset() + + patch2 := gomonkey.ApplyFunc(ioutil.ReadFile, func(string) ([]byte, error) { + return nil, nil + }) + GetCoreInfoByParam("") + patch2.Reset() + + patch3 := gomonkey.ApplyFunc(ioutil.ReadFile, func(string) ([]byte, error) { + return nil, nil + }) + patch3.ApplyFunc(json.Unmarshal, func([]byte, interface{}) error { + return nil + }) + GetCoreInfoByParam("") + patch3.Reset() +} diff --git a/functionsystem/apps/meta_service/common/logger/custom_encoder.go b/functionsystem/apps/meta_service/common/logger/custom_encoder.go new file mode 100644 index 0000000000000000000000000000000000000000..dbd88a0954e81aa7bc6db8a9c9b279f90268fe80 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/custom_encoder.go @@ -0,0 +1,387 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger log +package logger + +import ( + "math" + "os" + "regexp" + "strings" + "sync" + "time" + + "meta_service/common/constants" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +const ( + headerSeparator = ' ' + elementSeparator = " " + customDefaultLineEnding = "\n" + logMsgMaxLen = 1024 +) + +var ( + _customBufferPool = buffer.NewPool() + + _customPool = sync.Pool{New: func() interface{} { + return &customEncoder{} + }} + + replComp = regexp.MustCompile(`\s+`) +) + +// customEncoder represents the encoder for zap logger +// project's interface log +type customEncoder struct { + *zapcore.EncoderConfig + buf *buffer.Buffer + podName string +} + +// NewConsoleEncoder new custom console encoder to zap log module +func NewConsoleEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { + return &customEncoder{ + EncoderConfig: &cfg, + buf: _customBufferPool.Get(), + podName: os.Getenv(constants.HostNameEnvKey), + }, nil +} + +// NewCustomEncoder new custom encoder to zap log module +func NewCustomEncoder(cfg *zapcore.EncoderConfig) zapcore.Encoder { + return &customEncoder{ + EncoderConfig: cfg, + buf: _customBufferPool.Get(), + podName: os.Getenv(constants.HostNameEnvKey), + } +} + +// Clone return zap core Encoder +func (enc *customEncoder) Clone() zapcore.Encoder { + clone := enc.clone() + if enc.buf.Len() > 0 { + _, _ = clone.buf.Write(enc.buf.Bytes()) + } + return clone +} + +// EncodeEntry Encode Entry +func (enc *customEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + final.buf.AppendByte('[') + // add time + final.AppendString(ent.Time.UTC().Format("2006-01-02 15:04:05.000")) + // add level + final.buf.AppendByte(headerSeparator) + final.EncodeLevel(ent.Level, final) + // add caller + if ent.Caller.Defined { + final.buf.AppendByte(headerSeparator) + final.EncodeCaller(ent.Caller, final) + } + final.buf.AppendByte(']') + final.buf.AppendByte(headerSeparator) + final.buf.AppendByte('[') + // add podName + if enc.podName != "" { + final.buf.AppendString(enc.podName) + } + final.buf.AppendByte(']') + final.buf.AppendByte(headerSeparator) + if enc.buf.Len() > 0 { + final.buf.Write(enc.buf.Bytes()) + } + // add msg + if len(ent.Message) > logMsgMaxLen { + final.AppendString(ent.Message[0:logMsgMaxLen]) + } else { + final.AppendString(ent.Message) + } + if ent.Stack != "" && final.StacktraceKey != "" { + final.buf.AppendString(elementSeparator) + final.AddString(final.StacktraceKey, ent.Stack) + } + for _, field := range fields { + field.AddTo(final) + } + final.buf.AppendString(customDefaultLineEnding) + ret := final.buf + putCustomEncoder(final) + return ret, nil +} + +func putCustomEncoder(enc *customEncoder) { + enc.EncoderConfig = nil + enc.buf = nil + _customPool.Put(enc) +} + +func getCustomEncoder() *customEncoder { + return _customPool.Get().(*customEncoder) +} + +func (enc *customEncoder) clone() *customEncoder { + clone := getCustomEncoder() + clone.buf = _customBufferPool.Get() + clone.EncoderConfig = enc.EncoderConfig + clone.podName = enc.podName + return clone +} + +func (enc *customEncoder) writeField(k string, writeVal func()) *customEncoder { + enc.buf.AppendString("(" + k + ":") + writeVal() + enc.buf.AppendString(")") + return enc +} + +// AddArray Add Array +func (enc *customEncoder) AddArray(k string, marshaler zapcore.ArrayMarshaler) error { + return nil +} + +// AddObject Add Object +func (enc *customEncoder) AddObject(k string, marshaler zapcore.ObjectMarshaler) error { + return nil +} + +// AddBinary Add Binary +func (enc *customEncoder) AddBinary(k string, v []byte) { + enc.AddString(k, string(v)) +} + +// AddByteString Add Byte String +func (enc *customEncoder) AddByteString(k string, v []byte) { + enc.AddString(k, string(v)) +} + +// AddBool Add Bool +func (enc *customEncoder) AddBool(k string, v bool) { + enc.writeField(k, func() { + enc.AppendBool(v) + }) +} + +// AddComplex128 Add Complex128 +func (enc *customEncoder) AddComplex128(k string, val complex128) {} + +// AddComplex64 Add Complex64 +func (enc *customEncoder) AddComplex64(k string, v complex64) {} + +// AddDuration Add Duration +func (enc *customEncoder) AddDuration(k string, val time.Duration) { + enc.writeField(k, func() { + enc.AppendString(val.String()) + }) +} + +// AddFloat64 Add Float64 +func (enc *customEncoder) AddFloat64(k string, val float64) { + enc.writeField(k, func() { + enc.AppendFloat64(val) + }) +} + +// AddFloat32 Add Float32 +func (enc *customEncoder) AddFloat32(k string, v float32) { + enc.writeField(k, func() { + enc.AppendFloat64(float64(v)) + }) +} + +// AddInt Add Int +func (enc *customEncoder) AddInt(k string, v int) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddInt64 Add Int64 +func (enc *customEncoder) AddInt64(k string, val int64) { + enc.writeField(k, func() { + enc.AppendInt64(val) + }) +} + +// AddInt32 Add Int32 +func (enc *customEncoder) AddInt32(k string, v int32) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddInt16 Add Int16 +func (enc *customEncoder) AddInt16(k string, v int16) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddInt8 Add Int8 +func (enc *customEncoder) AddInt8(k string, v int8) { + enc.writeField(k, func() { + enc.AppendInt64(int64(v)) + }) +} + +// AddString Append String +func (enc *customEncoder) AddString(k, v string) { + enc.writeField(k, func() { + v = replComp.ReplaceAllString(v, " ") + if strings.Contains(v, " ") { + enc.buf.AppendString("(" + v + ")") + return + } + enc.AppendString(v) + }) +} + +// AddTime Add Time +func (enc *customEncoder) AddTime(k string, v time.Time) { + enc.writeField(k, func() { + enc.AppendString(v.UTC().Format("2006-01-02 15:04:05.000")) + }) +} + +// AddUint Add Uint +func (enc *customEncoder) AddUint(k string, v uint) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUint64 Add Uint64 +func (enc *customEncoder) AddUint64(k string, v uint64) { + enc.writeField(k, func() { + enc.AppendUint64(v) + }) +} + +// AddUint32 Add Uint32 +func (enc *customEncoder) AddUint32(k string, v uint32) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUint16 Add Uint16 +func (enc *customEncoder) AddUint16(k string, v uint16) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUint8 Add Uint8 +func (enc *customEncoder) AddUint8(k string, v uint8) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddUintptr Add Uint ptr +func (enc *customEncoder) AddUintptr(k string, v uintptr) { + enc.writeField(k, func() { + enc.AppendUint64(uint64(v)) + }) +} + +// AddReflected uses reflection to serialize arbitrary objects, so it's slow +// and allocation-heavy. +func (enc *customEncoder) AddReflected(k string, v interface{}) error { + return nil +} + +// OpenNamespace opens an isolated namespace where all subsequent fields will +// be added. Applications can use namespaces to prevent key collisions when +// injecting loggers into sub-components or third-party libraries. +func (enc *customEncoder) OpenNamespace(k string) {} + +// AppendBool Append Bool +func (enc *customEncoder) AppendBool(v bool) { enc.buf.AppendBool(v) } + +// AppendByteString Append Byte String +func (enc *customEncoder) AppendByteString(v []byte) { enc.AppendString(string(v)) } + +// AppendComplex128 Append Complex128 +func (enc *customEncoder) AppendComplex128(v complex128) {} + +// AppendComplex64 Append Complex64 +func (enc *customEncoder) AppendComplex64(v complex64) {} + +// AppendFloat64 Append Float64 +func (enc *customEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) } + +// AppendFloat32 Append Float32 +func (enc *customEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) } + +func (enc *customEncoder) appendFloat(v float64, bitSize int) { + switch { + // If the condition is not met, a string is returned to prevent blankness. + // IsNaN reports whether f is an IEEE 754 ``not-a-number'' value. + case math.IsNaN(v): + enc.buf.AppendString(`"NaN"`) + // IsInf reports whether f is an infinity, according to sign + case math.IsInf(v, 1): + // IsInf reports whether f is positive infinity + enc.buf.AppendString(`"+Inf"`) + // IsInf reports whether f is negative infinity + case math.IsInf(v, -1): + enc.buf.AppendString(`"-Inf"`) + default: + enc.buf.AppendFloat(v, bitSize) + } +} + +// AppendInt Append Int +func (enc *customEncoder) AppendInt(v int) { enc.buf.AppendInt(int64(v)) } + +// AppendInt64 Append Int64 +func (enc *customEncoder) AppendInt64(v int64) { enc.buf.AppendInt(v) } + +// AppendInt32 Append Int32 +func (enc *customEncoder) AppendInt32(v int32) { enc.buf.AppendInt(int64(v)) } + +// AppendInt16 Append Int16 +func (enc *customEncoder) AppendInt16(v int16) { enc.buf.AppendInt(int64(v)) } + +// AppendInt8 Append Int8 +func (enc *customEncoder) AppendInt8(v int8) { enc.buf.AppendInt(int64(v)) } + +// AppendString Append String +func (enc *customEncoder) AppendString(val string) { enc.buf.AppendString(val) } + +// AppendUint Append Uint +func (enc *customEncoder) AppendUint(v uint) { enc.buf.AppendUint(uint64(v)) } + +// AppendUint64 Append Uint64 +func (enc *customEncoder) AppendUint64(v uint64) { enc.buf.AppendUint(v) } + +// AppendUint32 Append Uint32 +func (enc *customEncoder) AppendUint32(v uint32) { enc.buf.AppendUint(uint64(v)) } + +// AppendUint16 Append Uint16 +func (enc *customEncoder) AppendUint16(v uint16) { enc.buf.AppendUint(uint64(v)) } + +// AppendUint8 Append Uint8 +func (enc *customEncoder) AppendUint8(v uint8) { enc.buf.AppendUint(uint64(v)) } + +// AppendUintptr Append Uint ptr +func (enc *customEncoder) AppendUintptr(v uintptr) { enc.buf.AppendUint(uint64(v)) } diff --git a/functionsystem/apps/meta_service/common/logger/custom_encoder_test.go b/functionsystem/apps/meta_service/common/logger/custom_encoder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d07cbcc278182e3dd850e1edbe5120bafca747c1 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/custom_encoder_test.go @@ -0,0 +1,148 @@ +package logger + +import ( + "github.com/stretchr/testify/assert" + "math" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zapcore" +) + +type CustomEncoderTestSuite struct { + suite.Suite + encoder *customEncoder +} + +func (ie *CustomEncoderTestSuite) SetupSuite() { + cfg := &zapcore.EncoderConfig{ + MessageKey: "test", + } + ie.encoder = NewCustomEncoder(cfg).(*customEncoder) +} + +func (ie *CustomEncoderTestSuite) TearDownSuite() { + ie.encoder = nil +} + +func TestCustomEncoderTestSuite(t *testing.T) { + suite.Run(t, new(CustomEncoderTestSuite)) +} + +func (ie *CustomEncoderTestSuite) TestAppendTime() { + encoder := ie.encoder + + encoder.AddTime("test", time.Now()) + encoder.AddDuration("test", time.Second) +} + +func (ie *CustomEncoderTestSuite) TestAppendObject() { + encoder := ie.encoder + encoder.AddArray("test", nil) + encoder.AddBinary("test", nil) + encoder.AddObject("test", nil) + encoder.AddReflected("test", nil) +} + +func (ie *CustomEncoderTestSuite) TestAppendString() { + encoder := ie.encoder + encoder.AppendString("") + encoder.AppendByteString(make([]byte, 0)) + + encoder.AddString("test", " ") + encoder.AddByteString("test", make([]byte, 0)) +} + +func (ie *CustomEncoderTestSuite) TestAppendBool() { + encoder := ie.encoder + encoder.AppendBool(false) + + encoder.AddBool("test", false) +} + +func (ie *CustomEncoderTestSuite) TestAppendInt() { + encoder := ie.encoder + encoder.AppendInt(0) + encoder.AppendInt8(0) + encoder.AppendInt16(0) + encoder.AppendInt32(0) + encoder.AppendInt64(0) + encoder.AppendUint(0) + encoder.AppendUint8(0) + encoder.AppendUint16(0) + encoder.AppendUint32(0) + encoder.AppendUint64(0) + + encoder.AddInt("test", 0) + encoder.AddInt8("test", 0) + encoder.AddInt16("test", 0) + encoder.AddInt32("test", 0) + encoder.AddInt64("test", 0) + encoder.AddUint("test", 0) + encoder.AddUint8("test", 0) + encoder.AddUint16("test", 0) + encoder.AddUint32("test", 0) + encoder.AddUint64("test", 0) +} + +func (ie *CustomEncoderTestSuite) TestAppendFloat() { + encoder := ie.encoder + encoder.buf.AppendByte(' ') + encoder.appendFloat(math.NaN(), 64) + encoder.appendFloat(math.Inf(1), 64) + encoder.appendFloat(math.Inf(-1), 64) + encoder.appendFloat(0, 64) + encoder.AppendFloat32(0) + encoder.AppendFloat64(0) + + encoder.AddFloat32("test", 0) + encoder.AddFloat64("test", 0) +} + +func (ie *CustomEncoderTestSuite) TestAppendComplex() { + encoder := ie.encoder + encoder.AppendComplex64(0) + encoder.AppendComplex128(0) + + encoder.AddComplex64("test", 0) + encoder.AddComplex128("test", 0) +} + +func (ie *CustomEncoderTestSuite) TestAppendPtr() { + encoder := ie.encoder + encoder.AppendUintptr(0) + + encoder.AddUintptr("test", 0) +} + +func (ie *CustomEncoderTestSuite) TestClone() { + encoder := ie.encoder + encoder.buf.AppendByte('1') + encoder.Clone() + assert.NotNil(ie.T(), encoder) +} + +func (ie *CustomEncoderTestSuite) TestEncodeEntry() { + encoder := ie.encoder + encoder.podName = "test" + encoder.buf.AppendByte('1') + encoder.EncodeCaller = func(caller zapcore.EntryCaller, encoder zapcore.PrimitiveArrayEncoder) { + return + } + encoder.EncodeLevel = func(zapcore.Level, zapcore.PrimitiveArrayEncoder) { + return + } + encoder.StacktraceKey = "test" + + ent := zapcore.Entry{} + ent.Caller.Defined = true + ent.Stack = "test" + + fields := make([]zapcore.Field, 0) + field := zapcore.Field{Key: "", Type: zapcore.StringType} + fields = append(fields, field) + + _, err := encoder.EncodeEntry(ent, fields) + assert.Nil(ie.T(), err) +} diff --git a/functionsystem/apps/meta_service/common/logger/interface_encoder.go b/functionsystem/apps/meta_service/common/logger/interface_encoder.go new file mode 100644 index 0000000000000000000000000000000000000000..a0c53b66fad97480db3833a086ead3b7d16c8069 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/interface_encoder.go @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger log +package logger + +import ( + "errors" + "math" + "os" + "sync" + "time" + + "meta_service/common/constants" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +var ( + _bufferPool = buffer.NewPool() + + _interfacePool = sync.Pool{New: func() interface{} { + return &interfaceEncoder{} + }} +) + +// InterfaceEncoderConfig holds interface log encoder config +type InterfaceEncoderConfig struct { + ModuleName string + HTTPMethod string + ModuleFrom string + TenantID string + FuncName string + FuncVer string + EncodeCaller zapcore.CallerEncoder +} + +// interfaceEncoder represents the encoder for interface log +// project's interface log +type interfaceEncoder struct { + *InterfaceEncoderConfig + buf *buffer.Buffer + podName string + spaced bool +} + +func getInterfaceEncoder() *interfaceEncoder { + return _interfacePool.Get().(*interfaceEncoder) +} + +func putInterfaceEncoder(enc *interfaceEncoder) { + enc.InterfaceEncoderConfig = nil + enc.spaced = false + enc.buf = nil + _interfacePool.Put(enc) +} + +// NewInterfaceEncoder create a new interface log encoder +func NewInterfaceEncoder(cfg InterfaceEncoderConfig, spaced bool) zapcore.Encoder { + return newInterfaceEncoder(cfg, spaced) +} + +func newInterfaceEncoder(cfg InterfaceEncoderConfig, spaced bool) *interfaceEncoder { + return &interfaceEncoder{ + InterfaceEncoderConfig: &cfg, + buf: _bufferPool.Get(), + spaced: spaced, + podName: os.Getenv(constants.HostNameEnvKey), + } +} + +// Clone return zap core Encoder +func (enc *interfaceEncoder) Clone() zapcore.Encoder { + return enc.clone() +} + +func (enc *interfaceEncoder) clone() *interfaceEncoder { + clone := getInterfaceEncoder() + clone.InterfaceEncoderConfig = enc.InterfaceEncoderConfig + clone.spaced = enc.spaced + clone.buf = _bufferPool.Get() + return clone +} + +// EncodeEntry Encode Entry +func (enc *interfaceEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + final.buf.AppendByte('[') + // add time + final.AppendString(ent.Time.UTC().Format("2006-01-02 15:04:05.000")) + // add level + // Level of interfaceLog is eternally INFO + final.buf.AppendByte(headerSeparator) + final.AppendString("INFO") + // add caller + if ent.Caller.Defined { + final.buf.AppendByte(headerSeparator) + final.EncodeCaller(ent.Caller, final) + } + final.buf.AppendByte(']') + final.buf.AppendByte(headerSeparator) + final.buf.AppendByte('[') + // add podName + if enc.podName != "" { + final.buf.AppendString(enc.podName) + } + final.buf.AppendByte(']') + final.buf.AppendByte(headerSeparator) + if enc.buf.Len() > 0 { + final.buf.Write(enc.buf.Bytes()) + } + // add msg + final.AppendString(ent.Message) + for _, field := range fields { + field.AddTo(final) + } + final.buf.AppendString(customDefaultLineEnding) + ret := final.buf + putInterfaceEncoder(final) + return ret, nil +} + +// AddString Append String +func (enc *interfaceEncoder) AddString(key, val string) { + enc.buf.AppendString(val) +} + +// AppendString Append String +func (enc *interfaceEncoder) AppendString(val string) { + enc.buf.AppendString(val) +} + +// AddDuration Add Duration +func (enc *interfaceEncoder) AddDuration(key string, val time.Duration) { + enc.AppendDuration(val) +} + +func (enc *interfaceEncoder) addElementSeparator() { + last := enc.buf.Len() - 1 + if last < 0 { + return + } + switch enc.buf.Bytes()[last] { + case headerSeparator: + return + default: + enc.buf.AppendByte(headerSeparator) + if enc.spaced { + enc.buf.AppendByte(' ') + } + } +} + +// AppendTime Append Time +func (enc *interfaceEncoder) AppendTime(val time.Time) { + cur := enc.buf.Len() + interfaceTimeEncode(val, enc) + if cur == enc.buf.Len() { + // User-supplied EncodeTime is a no-op. Fall back to nanos since epoch to keep + // output JSON valid. + enc.AppendInt64(val.UnixNano()) + } +} + +// AddArray Add Array +func (enc *interfaceEncoder) AddArray(key string, marshaler zapcore.ArrayMarshaler) error { + return errors.New("Unsupported method") +} + +// AddObject Add Object +func (enc *interfaceEncoder) AddObject(key string, marshaler zapcore.ObjectMarshaler) error { + return errors.New("Unsupported method") +} + +// AddBinary Add Binary +func (enc *interfaceEncoder) AddBinary(key string, value []byte) {} + +// AddByteString Add Byte String +func (enc *interfaceEncoder) AddByteString(key string, val []byte) { + enc.AppendByteString(val) +} + +// AddBool Add Bool +func (enc *interfaceEncoder) AddBool(key string, value bool) {} + +// AddComplex64 Add Complex64 +func (enc *interfaceEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) } + +// AddFloat32 Add Float32 +func (enc *interfaceEncoder) AddFloat32(k string, v float32) { enc.AddFloat64(k, float64(v)) } + +// AddInt Add Int +func (enc *interfaceEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) } + +// AddInt32 Add Int32 +func (enc *interfaceEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) } + +// AddInt16 Add Int16 +func (enc *interfaceEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) } + +// AddInt8 Add Int8 +func (enc *interfaceEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) } + +// AddUint Add Uint +func (enc *interfaceEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) } + +// AddUint32 Add Uint32 +func (enc *interfaceEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) } + +// AddUint16 Add Uint16 +func (enc *interfaceEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) } + +// AddUint8 Add Uint8 +func (enc *interfaceEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) } + +// AddUintptr Add Uint ptr +func (enc *interfaceEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) } + +// AddComplex128 Add Complex128 +func (enc *interfaceEncoder) AddComplex128(key string, val complex128) { + enc.AppendComplex128(val) +} + +// AddFloat64 Add Float64 +func (enc *interfaceEncoder) AddFloat64(key string, val float64) { + enc.AppendFloat64(val) +} + +// AddInt64 Add Int64 +func (enc *interfaceEncoder) AddInt64(key string, val int64) { + enc.AppendInt64(val) +} + +// AddTime Add Time +func (enc *interfaceEncoder) AddTime(key string, value time.Time) { + enc.AppendTime(value) +} + +// AddUint64 Add Uint64 +func (enc *interfaceEncoder) AddUint64(key string, value uint64) {} + +// AddReflected uses reflection to serialize arbitrary objects, so it's slow +// and allocation-heavy. +func (enc *interfaceEncoder) AddReflected(key string, value interface{}) error { + return nil +} + +// OpenNamespace opens an isolated namespace where all subsequent fields will +// be added. Applications can use namespaces to prevent key collisions when +// injecting loggers into sub-components or third-party libraries. +func (enc *interfaceEncoder) OpenNamespace(key string) {} + +// AppendComplex128 Append Complex128 +func (enc *interfaceEncoder) AppendComplex128(val complex128) {} + +// AppendInt64 Append Int64 +func (enc *interfaceEncoder) AppendInt64(val int64) { + enc.addElementSeparator() + enc.buf.AppendInt(val) +} + +// AppendBool Append Bool +func (enc *interfaceEncoder) AppendBool(val bool) { + enc.addElementSeparator() + enc.buf.AppendBool(val) +} + +func (enc *interfaceEncoder) appendFloat(val float64, bitSize int) { + enc.addElementSeparator() + switch { + case math.IsNaN(val): + enc.buf.AppendString(`"NaN"`) + case math.IsInf(val, 1): + enc.buf.AppendString(`"+Inf"`) + case math.IsInf(val, -1): + enc.buf.AppendString(`"-Inf"`) + default: + enc.buf.AppendFloat(val, bitSize) + } +} + +// AppendUint64 Append Uint64 +func (enc *interfaceEncoder) AppendUint64(val uint64) { + enc.addElementSeparator() + enc.buf.AppendUint(val) +} + +// AppendByteString Append Byte String +func (enc *interfaceEncoder) AppendByteString(val []byte) {} + +// AppendDuration Append Duration +func (enc *interfaceEncoder) AppendDuration(val time.Duration) {} + +// AppendComplex64 Append Complex64 +func (enc *interfaceEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) } + +// AppendFloat64 Append Float64 +func (enc *interfaceEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) } + +// AppendFloat32 Append Float32 +func (enc *interfaceEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) } + +// AppendInt Append Int +func (enc *interfaceEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) } + +// AppendInt32 Append Int32 +func (enc *interfaceEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) } + +// AppendInt16 Append Int16 +func (enc *interfaceEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) } + +// AppendInt8 Append Int8 +func (enc *interfaceEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) } + +// AppendUint Append Uint +func (enc *interfaceEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) } + +// AppendUint32 Append Uint32 +func (enc *interfaceEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) } + +// AppendUint16 Append Uint16 +func (enc *interfaceEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) } + +// AppendUint8 Append Uint8 +func (enc *interfaceEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) } + +// AppendUintptr Append Uint ptr +func (enc *interfaceEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) } + +func interfaceTimeEncode(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + t = t.UTC() + enc.AppendString(t.Format("2006-01-02 15:04:05.000")) +} diff --git a/functionsystem/apps/meta_service/common/logger/interface_encoder_test.go b/functionsystem/apps/meta_service/common/logger/interface_encoder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3b01084b6dfaac09fca7162a07dade2ac4860b03 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/interface_encoder_test.go @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logger + +import ( + "math" + "os" + "testing" + "time" + + "meta_service/common/logger/config" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// TestNewInterfaceEncoder Test New Interface Encoder +func TestNewInterfaceEncoder(t *testing.T) { + cfg := InterfaceEncoderConfig{ + ModuleName: "FunctionWorker", + HTTPMethod: "POST", + ModuleFrom: "FrontendInvoke", + TenantID: "tenant2", + FuncName: "myFunction", + FuncVer: "latest", + } + + encoder := NewInterfaceEncoder(cfg, false) + + priority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl >= zapcore.InfoLevel + }) + + sink := zapcore.Lock(os.Stdout) + core := zapcore.NewCore(encoder, sink, priority) + logger := zap.New(core) + + logger.Info("e1b71add-cb24-4ef8-93eb-af8d3ceb74e8|0|success|1") +} + +func Test_newCore(t *testing.T) { + coreInfo := config.CoreInfo{} + cfg := InterfaceEncoderConfig{} + _, err := newCore(coreInfo, cfg) + assert.NotNil(t, err) +} + +type InterfaceEncoderTestSuite struct { + suite.Suite + encoder *interfaceEncoder +} + +func (ie *InterfaceEncoderTestSuite) SetupSuite() { + cfg := InterfaceEncoderConfig{ + ModuleName: "FunctionWorker", + HTTPMethod: "POST", + ModuleFrom: "FrontendInvoke", + TenantID: "tenant2", + FuncName: "myFunction", + FuncVer: "latest", + } + ie.encoder = newInterfaceEncoder(cfg, false) +} + +func (ie *InterfaceEncoderTestSuite) TearDownSuite() { + ie.encoder = nil +} + +func TestInterfaceEncoderTestSuite(t *testing.T) { + suite.Run(t, new(InterfaceEncoderTestSuite)) +} + +func (ie *InterfaceEncoderTestSuite) TestClone() { + encoder := ie.encoder.Clone() + assert.NotNil(ie.T(), encoder) +} + +func (ie *InterfaceEncoderTestSuite) TestOpenNameSpace() { + encoder := ie.encoder + encoder.OpenNamespace("test") + assert.NotNil(ie.T(), encoder) +} + +func (ie *InterfaceEncoderTestSuite) TestEncodeEntry() { + encoder := ie.encoder + encoder.podName = "test" + encoder.buf.AppendByte('1') + encoder.EncodeCaller = func(caller zapcore.EntryCaller, encoder zapcore.PrimitiveArrayEncoder) { + return + } + + ent := zapcore.Entry{} + ent.Caller.Defined = true + + fields := make([]zapcore.Field, 0) + field := zapcore.Field{Key: "", Type: zapcore.StringType} + fields = append(fields, field) + + _, err := encoder.EncodeEntry(ent, fields) + assert.Nil(ie.T(), err) +} + +func (ie *InterfaceEncoderTestSuite) TestAddElementSeparator() { + encoder := ie.encoder + encoder.addElementSeparator() + + encoder.buf.AppendByte(' ') + encoder.addElementSeparator() + + encoder.buf.AppendByte('a') + encoder.spaced = true + encoder.addElementSeparator() +} + +func (ie *InterfaceEncoderTestSuite) TestInterfaceTimeEncode() { + encoder := ie.encoder + interfaceTimeEncode(time.Now(), encoder) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendTime() { + encoder := ie.encoder + encoder.AppendTime(time.Now()) + encoder.AppendDuration(time.Second) + + encoder.AddTime("test", time.Now()) + encoder.AddDuration("test", time.Second) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendObject() { + encoder := ie.encoder + encoder.AddArray("test", nil) + encoder.AddBinary("test", nil) + encoder.AddObject("test", nil) + encoder.AddReflected("test", nil) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendString() { + encoder := ie.encoder + encoder.AppendString("") + encoder.AppendByteString(make([]byte, 0)) + + encoder.AddString("test", "") + encoder.AddByteString("test", make([]byte, 0)) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendBool() { + encoder := ie.encoder + encoder.AppendBool(false) + + encoder.AddBool("test", false) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendInt() { + encoder := ie.encoder + encoder.AppendInt(0) + encoder.AppendInt8(0) + encoder.AppendInt16(0) + encoder.AppendInt32(0) + encoder.AppendInt64(0) + encoder.AppendUint(0) + encoder.AppendUint8(0) + encoder.AppendUint16(0) + encoder.AppendUint32(0) + encoder.AppendUint64(0) + + encoder.AddInt("test", 0) + encoder.AddInt8("test", 0) + encoder.AddInt16("test", 0) + encoder.AddInt32("test", 0) + encoder.AddInt64("test", 0) + encoder.AddUint("test", 0) + encoder.AddUint8("test", 0) + encoder.AddUint16("test", 0) + encoder.AddUint32("test", 0) + encoder.AddUint64("test", 0) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendFloat() { + encoder := ie.encoder + encoder.buf.AppendByte(' ') + encoder.appendFloat(math.NaN(), 64) + encoder.appendFloat(math.Inf(1), 64) + encoder.appendFloat(math.Inf(-1), 64) + encoder.appendFloat(0, 64) + encoder.AppendFloat32(0) + encoder.AppendFloat64(0) + + encoder.AddFloat32("test", 0) + encoder.AddFloat64("test", 0) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendComplex() { + encoder := ie.encoder + encoder.AppendComplex64(0) + encoder.AppendComplex128(0) + + encoder.AddComplex64("test", 0) + encoder.AddComplex128("test", 0) +} + +func (ie *InterfaceEncoderTestSuite) TestAppendPtr() { + encoder := ie.encoder + encoder.AppendUintptr(0) + + encoder.AddUintptr("test", 0) +} + +/* +func (ie *InterfaceEncoderTestSuite) Test() { +encoder := ie.encoder +} +*/ diff --git a/functionsystem/apps/meta_service/common/logger/interfacelogger.go b/functionsystem/apps/meta_service/common/logger/interfacelogger.go new file mode 100644 index 0000000000000000000000000000000000000000..f525f11fef761fb50cc49a71792fc1aeb1067ba0 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/interfacelogger.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger log +package logger + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "meta_service/common/logger/config" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const defaultPerm = 0o666 + +// NewInterfaceLogger returns a new interface logger +func NewInterfaceLogger(logPath, fileName string, cfg InterfaceEncoderConfig) (*InterfaceLogger, error) { + coreInfo, err := config.GetCoreInfo(logPath) + if err != nil { + coreInfo = config.GetDefaultCoreInfo() + } + filePath := filepath.Join(coreInfo.FilePath, fileName+".log") + + coreInfo.FilePath = filePath + cfg.EncodeCaller = zapcore.ShortCallerEncoder + // skip level to print caller line of origin log + const skipLevel = 5 + core, err := newCore(coreInfo, cfg) + if err != nil { + return nil, err + } + logger := zap.New(core, zap.AddCaller(), zap.AddCallerSkip(skipLevel)) + + return &InterfaceLogger{log: logger}, nil +} + +// InterfaceLogger interface logger which implements by zap logger +type InterfaceLogger struct { + log *zap.Logger +} + +// Write writes message information +func (logger *InterfaceLogger) Write(msg string) { + logger.log.Debug(msg) +} + +func newCore(coreInfo config.CoreInfo, cfg InterfaceEncoderConfig) (zapcore.Core, error) { + w, err := CreateSink(coreInfo) + if err != nil { + return nil, err + } + syncer := zapcore.AddSync(w) + + encoder := NewInterfaceEncoder(cfg, false) + + priority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + var customLevel zapcore.Level + if err := customLevel.UnmarshalText([]byte(coreInfo.Level)); err != nil { + customLevel = zapcore.InfoLevel + } + return lvl >= customLevel + }) + + return zapcore.NewCore(encoder, syncer, priority), nil +} + +// CreateSink creates a new zap log sink +func CreateSink(coreInfo config.CoreInfo) (io.Writer, error) { + // create directory if not already exist + dir := filepath.Dir(coreInfo.FilePath) + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + fmt.Printf("failed to mkdir: %s", dir) + return nil, err + } + w, err := initRollingLog(coreInfo, os.O_WRONLY|os.O_APPEND|os.O_CREATE, defaultPerm) + if err != nil { + fmt.Printf("failed to open log file: %s, err: %s\n", coreInfo.FilePath, err.Error()) + return nil, err + } + return w, nil +} diff --git a/functionsystem/apps/meta_service/common/logger/interfacelogger_test.go b/functionsystem/apps/meta_service/common/logger/interfacelogger_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ecd58015560f6fa2dd161c95d100b503da1697a6 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/interfacelogger_test.go @@ -0,0 +1,38 @@ +package logger + +import ( + "errors" + "testing" + + "meta_service/common/logger/config" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" + "go.uber.org/zap/zapcore" +) + +func TestNewInterfaceLogger(t *testing.T) { + cfg := InterfaceEncoderConfig{} + _, err := NewInterfaceLogger("", "file", cfg) + assert.Nil(t, err) + + patch1 := gomonkey.ApplyFunc(config.GetCoreInfo, func(string) (config.CoreInfo, error) { + return config.CoreInfo{}, errors.New("GetDefaultCoreInfo fail") + }) + _, err = NewInterfaceLogger("", "file", cfg) + assert.Nil(t, err) + patch1.Reset() + + patch2 := gomonkey.ApplyFunc(newCore, func(config.CoreInfo, InterfaceEncoderConfig) (zapcore.Core, error) { + return nil, errors.New("newCore fail") + }) + _, err = NewInterfaceLogger("", "file", cfg) + assert.NotNil(t, err) + patch2.Reset() +} + +func TestWrite(*testing.T) { + cfg := InterfaceEncoderConfig{} + logger, _ := NewInterfaceLogger("", "file", cfg) + logger.Write("test") +} diff --git a/functionsystem/apps/meta_service/common/logger/log/log.go b/functionsystem/apps/meta_service/common/logger/log/log.go new file mode 100644 index 0000000000000000000000000000000000000000..da8c3558852177ad48a672ef564fa6b95d27b963 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/log/log.go @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package log is common logger client +package log + +import ( + "fmt" + "meta_service/common/logger/config" + "meta_service/common/logger/zap" + "path/filepath" + "sync" + + "github.com/asaskevich/govalidator/v11" + uberZap "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var ( + once sync.Once + formatLogger FormatLogger + defaultLogger, _ = uberZap.NewProduction() +) + +// InitRunLog init run log with log.json file +func InitRunLog(configFile, fileName string) error { + coreInfo, err := config.GetCoreInfo(configFile) + if err != nil { + return err + } + if coreInfo.Disable { + return nil + } + formatLogger, err = newFormatLogger(fileName, coreInfo) + return err +} + +// InitRunLogByParam init run log with log.json file +func InitRunLogByParam(configFile string, fileName string) error { + coreInfo, err := config.GetCoreInfoByParam(configFile) + if err != nil { + return err + } + if coreInfo.Disable { + return nil + } + formatLogger, err = newFormatLogger(fileName, coreInfo) + return err +} + +// InitRunLogWithConfig init run log with config +func InitRunLogWithConfig(fileName string, coreInfo config.CoreInfo) (FormatLogger, error) { + if _, err := govalidator.ValidateStruct(coreInfo); err != nil { + return nil, err + } + return newFormatLogger(fileName, coreInfo) +} + +// FormatLogger format logger interface +type FormatLogger interface { + With(fields ...zapcore.Field) FormatLogger + + Infof(format string, paras ...interface{}) + Errorf(format string, paras ...interface{}) + Warnf(format string, paras ...interface{}) + Debugf(format string, paras ...interface{}) + Fatalf(format string, paras ...interface{}) + + Info(msg string, fields ...uberZap.Field) + Error(msg string, fields ...uberZap.Field) + Warn(msg string, fields ...uberZap.Field) + Debug(msg string, fields ...uberZap.Field) + Fatal(msg string, fields ...uberZap.Field) + + Sync() +} + +// zapLoggerWithFormat define logger +type zapLoggerWithFormat struct { + Logger *uberZap.Logger + SLogger *uberZap.SugaredLogger +} + +// newFormatLogger new formatLogger with log config info +func newFormatLogger(fileName string, coreInfo config.CoreInfo) (FormatLogger, error) { + coreInfo.FilePath = filepath.Join(coreInfo.FilePath, fileName+"-run.log") + logger, err := zap.NewWithLevel(coreInfo) + if err != nil { + return nil, err + } + + return &zapLoggerWithFormat{ + Logger: logger, + SLogger: logger.Sugar(), + }, nil +} + +// NewConsoleLogger returns a console logger +func NewConsoleLogger() FormatLogger { + logger, err := zap.NewConsoleLog() + if err != nil { + fmt.Println("new console log error", err) + logger = defaultLogger + } + return &zapLoggerWithFormat{ + Logger: logger, + SLogger: logger.Sugar(), + } +} + +// GetLogger get logger directly +func GetLogger() FormatLogger { + if formatLogger == nil { + once.Do(func() { + formatLogger = NewConsoleLogger() + }) + } + return formatLogger +} + +// With add fields to log header +func (z *zapLoggerWithFormat) With(fields ...zapcore.Field) FormatLogger { + logger := z.Logger.With(fields...) + return &zapLoggerWithFormat{ + Logger: logger, + SLogger: logger.Sugar(), + } +} + +// Infof stdout format and paras +func (z *zapLoggerWithFormat) Infof(format string, paras ...interface{}) { + z.SLogger.Infof(format, paras...) +} + +// Errorf stdout format and paras +func (z *zapLoggerWithFormat) Errorf(format string, paras ...interface{}) { + z.SLogger.Errorf(format, paras...) +} + +// Warnf stdout format and paras +func (z *zapLoggerWithFormat) Warnf(format string, paras ...interface{}) { + z.SLogger.Warnf(format, paras...) +} + +// Debugf stdout format and paras +func (z *zapLoggerWithFormat) Debugf(format string, paras ...interface{}) { + z.SLogger.Debugf(format, paras...) +} + +// Fatalf stdout format and paras +func (z *zapLoggerWithFormat) Fatalf(format string, paras ...interface{}) { + z.SLogger.Fatalf(format, paras...) +} + +// Info stdout format and paras +func (z *zapLoggerWithFormat) Info(msg string, fields ...uberZap.Field) { + z.Logger.Info(msg, fields...) +} + +// Error stdout format and paras +func (z *zapLoggerWithFormat) Error(msg string, fields ...uberZap.Field) { + z.Logger.Error(msg, fields...) +} + +// Warn stdout format and paras +func (z *zapLoggerWithFormat) Warn(msg string, fields ...uberZap.Field) { + z.Logger.Warn(msg, fields...) +} + +// Debug stdout format and paras +func (z *zapLoggerWithFormat) Debug(msg string, fields ...uberZap.Field) { + z.Logger.Debug(msg, fields...) +} + +// Fatal stdout format and paras +func (z *zapLoggerWithFormat) Fatal(msg string, fields ...uberZap.Field) { + z.Logger.Fatal(msg, fields...) +} + +// Sync calls the underlying Core's Sync method, flushing any buffered log +// entries. Applications should take care to call Sync before exiting. +func (z *zapLoggerWithFormat) Sync() { + z.Logger.Sync() +} diff --git a/functionsystem/apps/meta_service/common/logger/log/log_test.go b/functionsystem/apps/meta_service/common/logger/log/log_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6e27f01e870a655502341750ff0863f77c8bb490 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/log/log_test.go @@ -0,0 +1,129 @@ +package log + +import ( + "errors" + "testing" + + "meta_service/common/logger/config" + "meta_service/common/logger/zap" + + "github.com/agiledragon/gomonkey" + "github.com/asaskevich/govalidator/v11" + . "github.com/smartystreets/goconvey/convey" + uberZap "go.uber.org/zap" +) + +func TestNewConsoleLogger(t *testing.T) { + patch := gomonkey.ApplyFunc(zap.NewConsoleLog, func() (*uberZap.Logger, error) { + return nil, errors.New("NewConsoleLog fail") + }) + NewConsoleLogger() + patch.Reset() +} + +func TestInitRunLogByParam(t *testing.T) { + Convey("test InitRunLogByParam GetCoreInfoByParam fail", t, func() { + patch := gomonkey.ApplyFunc(config.GetCoreInfoByParam, func(string) (config.CoreInfo, error) { + return config.CoreInfo{}, errors.New("GetCoreInfoByParam fail") + }) + err := InitRunLogByParam("", "") + So(err, ShouldNotBeNil) + patch.Reset() + }) + Convey("test InitRunLogByParam GetCoreInfoByParam success", t, func() { + patch := gomonkey.ApplyFunc(config.GetCoreInfoByParam, func(string) (config.CoreInfo, error) { + return config.CoreInfo{Disable: true}, nil + }) + err := InitRunLogByParam("", "") + So(err, ShouldBeNil) + patch.Reset() + }) + Convey("test InitRunLogByParam", t, func() { + patches := gomonkey.NewPatches() + patches.ApplyFunc(config.GetCoreInfoByParam, func(string) (config.CoreInfo, error) { + return config.CoreInfo{Disable: false}, nil + }) + patches.ApplyFunc(newFormatLogger, func(string, config.CoreInfo) (FormatLogger, error) { + return nil, nil + }) + err := InitRunLogByParam("", "") + So(err, ShouldBeNil) + patches.Reset() + }) +} + +func TestInitRunLog(t *testing.T) { + Convey("test InitRunLogByParam GetCoreInfo fail", t, func() { + patch := gomonkey.ApplyFunc(config.GetCoreInfo, func(string) (config.CoreInfo, error) { + return config.CoreInfo{}, errors.New("GetCoreInfo fail") + }) + err := InitRunLog("", "") + So(err, ShouldNotBeNil) + patch.Reset() + }) + Convey("test InitRunLogByParam GetCoreInfo success", t, func() { + patch := gomonkey.ApplyFunc(config.GetCoreInfo, func(string) (config.CoreInfo, error) { + return config.CoreInfo{Disable: true}, nil + }) + err := InitRunLog("", "") + So(err, ShouldBeNil) + patch.Reset() + }) + Convey("test InitRunLog", t, func() { + patches := gomonkey.NewPatches() + patches.ApplyFunc(config.GetCoreInfo, func(string) (config.CoreInfo, error) { + return config.CoreInfo{Disable: false}, nil + }) + patches.ApplyFunc(newFormatLogger, func(string, config.CoreInfo) (FormatLogger, error) { + return nil, nil + }) + err := InitRunLog("", "") + So(err, ShouldBeNil) + patches.Reset() + }) +} + +func TestInitRunLogWithConfig(t *testing.T) { + Convey("test InitRunLogWithConfig ValidateStruct fail", t, func() { + patches := gomonkey.NewPatches() + patches.ApplyFunc(govalidator.ValidateStruct, func(interface{}) (bool, error) { + return false, errors.New("ValidateStruct fail") + }) + _, err := InitRunLogWithConfig("", config.CoreInfo{}) + So(err, ShouldNotBeNil) + patches.Reset() + }) + Convey("test InitRunLogWithConfig ValidateStruct success", t, func() { + patches := gomonkey.NewPatches() + patches.ApplyFunc(govalidator.ValidateStruct, func(interface{}) (bool, error) { + return false, nil + }) + patches.ApplyFunc(newFormatLogger, func(string, config.CoreInfo) (FormatLogger, error) { + return nil, nil + }) + _, err := InitRunLogWithConfig("", config.CoreInfo{}) + So(err, ShouldBeNil) + patches.Reset() + }) +} + +func TestNewFormatLogger(t *testing.T) { + Convey("test NewFormatLogger NewWithLevel success", t, func() { + patches := gomonkey.NewPatches() + patches.ApplyFunc(zap.NewWithLevel, func(config.CoreInfo) (*uberZap.Logger, error) { + return &uberZap.Logger{}, nil + }) + _, err := newFormatLogger("", config.CoreInfo{FilePath: "/test"}) + So(err, ShouldBeNil) + patches.Reset() + }) + Convey("test NewFormatLogger NewWithLevel fail", t, func() { + patches := gomonkey.NewPatches() + patches.ApplyFunc(zap.NewWithLevel, func(config.CoreInfo) (*uberZap.Logger, error) { + return &uberZap.Logger{}, errors.New("NewWithLevel fail") + }) + _, err := newFormatLogger("", config.CoreInfo{FilePath: "/test"}) + So(err, ShouldNotBeNil) + patches.Reset() + }) +} diff --git a/functionsystem/apps/meta_service/common/logger/logservice/logtankservice.go b/functionsystem/apps/meta_service/common/logger/logservice/logtankservice.go new file mode 100644 index 0000000000000000000000000000000000000000..a1f598452c7def2f20262ecd9a57489548433316 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/logservice/logtankservice.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logservice is common logger service +package logservice + +// LogTankService - +type LogTankService struct { + GroupID string `json:"logGroupId" valid:",optional"` + StreamID string `json:"logStreamId" valid:",optional"` +} diff --git a/functionsystem/apps/meta_service/common/logger/logwrap/logwrap.go b/functionsystem/apps/meta_service/common/logger/logwrap/logwrap.go new file mode 100644 index 0000000000000000000000000000000000000000..b85392e6a93edbc1c78e3d5959cc4304be7d9d91 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/logwrap/logwrap.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logwrap print logs by frequency. +package logwrap + +import ( + "sync" + + "meta_service/common/logger/log" +) + +const ( + // Frequency print frequency + Frequency = 10 + // INFO info log level + INFO = "INFO" + // ERROR error log level + ERROR = "ERROR" + // DEBUG debug log level + DEBUG = "DEBUG" + // WARN warn log level + WARN = "WARN" +) + +var logWrap = &logWrapper{} + +type logWrapper struct { + // sync map key is runtimeID string + // sync map value is nums + logWrap sync.Map +} + +// Store sets the value for the logString +func Store(logString string) { + logWrapTime, exist := logWrap.logWrap.Load(logString) + if !exist || logWrapTime == nil { + logWrapTime = 0 + } else { + logWrapTime = (logWrapTime.(int) + 1) % Frequency + } + logWrap.logWrap.Store(logString, logWrapTime) +} + +// Delete deletes logWrap for the logString +func Delete(logString string) { + logWrap.logWrap.Delete(logString) +} + +// Print print logs by frequency and level +func Print(id string, level, format string, paras ...interface{}) { + logWrapTime, exist := logWrap.logWrap.Load(id) + if !exist || logWrapTime == nil { + logWrapTime = 0 + } + if logWrapTime.(int)%Frequency == 0 { + switch level { + case ERROR: + log.GetLogger().Errorf(format, paras...) + case INFO: + log.GetLogger().Infof(format, paras...) + case DEBUG: + log.GetLogger().Debugf(format, paras...) + case WARN: + log.GetLogger().Warnf(format, paras...) + default: + log.GetLogger().Infof(format, paras...) + } + } +} diff --git a/functionsystem/apps/meta_service/common/logger/logwrap/logwrap_test.go b/functionsystem/apps/meta_service/common/logger/logwrap/logwrap_test.go new file mode 100644 index 0000000000000000000000000000000000000000..868a916b9413a658b2f8ab55c80555b6ec3e98a1 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/logwrap/logwrap_test.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logwrap print logs by frequency. +package logwrap + +import "testing" + +func TestLogWrap(t *testing.T) { + Store("runtime1") + Print("runtime1", "", "TestLogWrap: %s") + Print("runtime1", INFO, "TestLogWrap: %s") + Print("runtime1", ERROR, "TestLogWrap: %s") + Print("runtime1", DEBUG, "TestLogWrap: %s") + Print("runtime1", WARN, "TestLogWrap: %s") + Delete("runtime1") +} \ No newline at end of file diff --git a/functionsystem/apps/meta_service/common/logger/rollinglog.go b/functionsystem/apps/meta_service/common/logger/rollinglog.go new file mode 100644 index 0000000000000000000000000000000000000000..9337c43aedce8ff757ab7ba279c993f6da421497 --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/rollinglog.go @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logger rollingLog +package logger + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "meta_service/common/logger/config" +) + +const ( + megabyte = 1024 * 1024 + defaultFileSize = 100 + defaultBackups = 20 + defaultMaxSize = 400 + defaultMaxBacks = 1 +) + +var logNameCache = struct { + m map[string]string + sync.Mutex +}{ + m: make(map[string]string, 1), + Mutex: sync.Mutex{}, +} + +type rollingLog struct { + file *os.File + reg *regexp.Regexp + mu sync.RWMutex + sinks []string + dir string + nameTemplate string + maxSize int64 + size int64 + maxBackups int + flag int + perm os.FileMode + isUserLog bool +} + +func initRollingLog(coreInfo config.CoreInfo, flag int, perm os.FileMode) (*rollingLog, error) { + if coreInfo.FilePath == "" { + return nil, errors.New("empty log file path") + } + if coreInfo.Rolling == nil { + coreInfo.Rolling = &config.RollingInfo{ + MaxSize: defaultMaxSize, + MaxBackups: defaultMaxBacks, + } + } + log := &rollingLog{ + dir: filepath.Dir(coreInfo.FilePath), + nameTemplate: filepath.Base(coreInfo.FilePath), + flag: flag, + perm: perm, + maxSize: int64(coreInfo.Rolling.MaxSize * megabyte), + maxBackups: coreInfo.Rolling.MaxBackups, + isUserLog: false, + } + if log.maxBackups < 1 { + log.maxBackups = defaultBackups + } + if log.maxSize < megabyte { + log.maxSize = defaultFileSize * megabyte + } + if log.isUserLog { + return log, log.tidySinks() + } + extension := filepath.Ext(log.nameTemplate) + regExp := fmt.Sprintf(`^%s(?:(?:-|\.)\d*)?\%s$`, + log.nameTemplate[:len(log.nameTemplate)-len(extension)], extension) + reg, err := regexp.Compile(regExp) + if err != nil { + return nil, err + } + log.reg = reg + return log, log.tidySinks() +} + +func (r *rollingLog) tidySinks() error { + if r.isUserLog || r.file != nil { + return r.newSink() + } + // scan and reuse past log file when service restarted + r.scanLogFiles() + if len(r.sinks) > 0 { + fullName := r.sinks[len(r.sinks)-1] + info, err := os.Stat(fullName) + if err != nil || info.Size() >= r.maxSize { + return r.newSink() + } + file, err := os.OpenFile(fullName, r.flag, r.perm) + if err == nil { + r.file = file + r.size = info.Size() + return nil + } + } + return r.newSink() +} + +func (r *rollingLog) scanLogFiles() { + dirEntrys, err := os.ReadDir(r.dir) + if err != nil { + fmt.Printf("failed to read dir: %s\n", r.dir) + return + } + infos := make([]os.FileInfo, 0, r.maxBackups) + for _, entry := range dirEntrys { + if r.reg.MatchString(entry.Name()) { + info, err := entry.Info() + if err == nil { + infos = append(infos, info) + } + } + } + if len(infos) > 0 { + sort.Slice(infos, func(i, j int) bool { + return infos[i].ModTime().Before(infos[j].ModTime()) + }) + for i := range infos { + r.sinks = append(r.sinks, filepath.Join(r.dir, infos[i].Name())) + } + r.cleanRedundantSinks() + } +} + +func (r *rollingLog) cleanRedundantSinks() { + if len(r.sinks) < r.maxBackups { + return + } + curSinks := make([]string, 0, len(r.sinks)) + for _, name := range r.sinks { + if isAvailable(name) { + curSinks = append(curSinks, name) + } + } + r.sinks = curSinks + sinkNum := len(r.sinks) + if sinkNum > r.maxBackups { + removes := r.sinks[:sinkNum-r.maxBackups] + go removeFiles(removes) + r.sinks = r.sinks[sinkNum-r.maxBackups:] + } + return +} + +func removeFiles(paths []string) { + for _, path := range paths { + err := os.Remove(path) + if err != nil && !os.IsNotExist(err) { + fmt.Printf("failed remove file %s\n", path) + } + } +} + +func isAvailable(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func (r *rollingLog) newSink() error { + fullName := filepath.Join(r.dir, r.newName()) + if isAvailable(fullName) && r.file != nil && r.file.Name() == filepath.Base(fullName) { + return errors.New("log file already opened: " + fullName) + } + file, err := os.OpenFile(fullName, r.flag, r.perm) + if err != nil { + return err + } + if r.file != nil { + err = r.file.Close() + } + if err != nil { + fmt.Printf("failed to close file: %s\n", err.Error()) + } + r.file = file + info, err := file.Stat() + if err != nil { + r.size = 0 + } else { + r.size = info.Size() + } + r.sinks = append(r.sinks, fullName) + r.cleanRedundantSinks() + if r.isUserLog { + logNameCache.Lock() + logNameCache.m[r.nameTemplate] = fullName + logNameCache.Unlock() + } + return nil +} + +func (r *rollingLog) newName() string { + if !r.isUserLog { + timeNow := time.Now().Format("2006010215040506") + ext := filepath.Ext(r.nameTemplate) + return fmt.Sprintf("%s.%s%s", r.nameTemplate[:len(r.nameTemplate)-len(ext)], timeNow, ext) + } + if r.file == nil { + return r.nameTemplate + } + timeNow := time.Now().Format("2006010215040506") + var prefix, suffix string + if index := strings.LastIndex(r.nameTemplate, "@") + 1; index <= len(r.nameTemplate) { + prefix = r.nameTemplate[:index] + } + if index := strings.Index(r.nameTemplate, "#"); index >= 0 { + suffix = r.nameTemplate[index:] + } + if prefix == "" || suffix == "" { + return "" + } + return fmt.Sprintf("%s%s%s", prefix, timeNow, suffix) +} + +// Write data to file and check whether to rotate log +func (r *rollingLog) Write(data []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r == nil || r.file == nil { + return 0, errors.New("log file is nil") + } + n, err := r.file.Write(data) + r.size += int64(n) + if r.size > r.maxSize { + r.tryRotate() + } + if syncErr := r.file.Sync(); syncErr != nil { + fmt.Printf("failed to sync log err: %s\n", syncErr.Error()) + } + return n, err +} + +func (r *rollingLog) tryRotate() { + if info, err := r.file.Stat(); err == nil && info.Size() < r.maxSize { + return + } + err := r.tidySinks() + if err != nil { + fmt.Printf("failed to rotate log err: %s\n", err.Error()) + } + return +} + +// GetLogName get current log name when refreshing user log mod time +func GetLogName(nameTemplate string) string { + logNameCache.Lock() + name := logNameCache.m[nameTemplate] + logNameCache.Unlock() + return name +} diff --git a/functionsystem/apps/meta_service/common/logger/rollinglog_test.go b/functionsystem/apps/meta_service/common/logger/rollinglog_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c70b8efb79a128d51cf4284906a4a33a6e8c5d9a --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/rollinglog_test.go @@ -0,0 +1,134 @@ +package logger + +import ( + "errors" + "io/fs" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey" + + "functioncore/pkg/common/logger/config" + + "github.com/smartystreets/goconvey/convey" +) + +type mockInfo struct { + name string + isDir bool + size int64 +} + +func (m mockInfo) Name() string { + return m.name +} + +func (m mockInfo) IsDir() bool { + return m.isDir +} + +func (m mockInfo) Type() fs.FileMode { + return 0 +} + +func (m mockInfo) Info() (fs.FileInfo, error) { + return m, nil +} + +func (m mockInfo) Size() int64 { + return m.size +} + +func (m mockInfo) Mode() fs.FileMode { + return 0 +} + +func (m mockInfo) ModTime() time.Time { + return time.Now() +} + +func (m mockInfo) Sys() interface{} { + return nil +} + +func Test_initRollingLog(t *testing.T) { + coreInfo := config.CoreInfo{ + FilePath: "./test-run.log", + Rolling: &config.RollingInfo{ + MaxSize: 400, // Log file reaches maxSize should be compressed with unit MB + MaxBackups: 1, // Max compressed file nums + MaxAge: 1, // Max Age of old log files with unit days + Compress: true, // Determines if the log files should be compressed + }, + } + defer gomonkey.ApplyFunc(os.ReadDir, func(string) ([]os.DirEntry, error) { + return []os.DirEntry{ + mockInfo{name: "test-run.2006010215040507.log"}, + mockInfo{name: "test-run.2006010215040508.log"}, + mockInfo{name: "{funcName}@ABCabc@latest@pool22-300-128-fusion-85c55c66d7-zzj9x@{timeNow}#{logGroupID}#{logStreamID}#cff-log.log"}, + }, nil + }).ApplyFunc(os.OpenFile, func(string, int, os.FileMode) (*os.File, error) { + return nil, nil + }).Reset() + convey.Convey("init service log", t, func() { + defer gomonkey.ApplyFunc(os.Stat, func(name string) (os.FileInfo, error) { + return mockInfo{name: strings.TrimPrefix(name, "./")}, nil + }).Reset() + log, err := initRollingLog(coreInfo, os.O_WRONLY|os.O_APPEND|os.O_CREATE, defaultPerm) + convey.So(log, convey.ShouldNotBeNil) + convey.So(err, convey.ShouldBeNil) + }) +} + +func Test_rollingLog_Write(t *testing.T) { + log := &rollingLog{} + log.maxSize = 0 + log.isUserLog = true + log.file = &os.File{} + log.nameTemplate = "{funcName}@ABCabc@latest@pool22-300-128-fusion-85c55c66d7-zzj9x@{timeNow}#{logGroupID}#{logStreamID}#cff-log.log" + convey.Convey("write rolling log", t, func() { + convey.Convey("case1: failed to write rolling log", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(log.file), "Write", func(f *os.File, b []byte) (n int, err error) { + return len(b), nil + }).ApplyMethod(reflect.TypeOf(log.file), "Stat", func(f *os.File) (info os.FileInfo, err error) { + return mockInfo{size: 3}, nil + }).ApplyMethod(reflect.TypeOf(log.file), "Sync", func(f *os.File) error { + return nil + }).Reset() + n, err := log.Write([]byte("abc")) + convey.So(n, convey.ShouldEqual, 3) + convey.So(err, convey.ShouldBeNil) + }) + + convey.Convey("case2: failed to write rolling log", func() { + defer gomonkey.ApplyMethod(reflect.TypeOf(log.file), "Write", func(f *os.File, b []byte) (n int, err error) { + return len(b), nil + }).ApplyMethod(reflect.TypeOf(log.file), "Stat", func(f *os.File) (info os.FileInfo, err error) { + return mockInfo{size: 3}, nil + }).ApplyMethod(reflect.TypeOf(log.file), "Sync", func(f *os.File) error { + return errors.New("test") + }).Reset() + n, err := log.Write([]byte("abc")) + convey.So(n, convey.ShouldEqual, 3) + convey.So(err, convey.ShouldBeNil) + }) + }) +} + +func Test_rollingLog_cleanRedundantSinks(t *testing.T) { + log := &rollingLog{} + log.maxBackups = 0 + tn := time.Now().String() + os.Create("test_log_1#" + tn) + os.Create("test_log_2#" + tn) + log.sinks = []string{"test_log_1#" + tn, "test_log_2#" + tn} + convey.Convey("rollingLog_cleanRedundantSinks", t, func() { + log.cleanRedundantSinks() + time.Sleep(50 * time.Millisecond) + convey.So(isAvailable("test_log_1#"+tn), convey.ShouldEqual, false) + convey.So(isAvailable("test_log_2#"+tn), convey.ShouldEqual, false) + }) +} diff --git a/functionsystem/apps/meta_service/common/logger/zap/zaplog.go b/functionsystem/apps/meta_service/common/logger/zap/zaplog.go new file mode 100644 index 0000000000000000000000000000000000000000..3a1a0503ae4a414e280a6f079a1a5158315acf3d --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/zap/zaplog.go @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package zap zapper log +package zap + +import ( + "fmt" + "path/filepath" + "time" + + "meta_service/common/logger" + "meta_service/common/logger/config" + + uberZap "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const ( + skipLevel = 1 +) + +// DefaultEncodeConfig is the default EncoderConfig used for zaplog +var DefaultEncodeConfig = zapcore.EncoderConfig{ + TimeKey: "T", + LevelKey: "L", + NameKey: "Logger", + MessageKey: "M", + CallerKey: "C", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, +} + +func init() { + uberZap.RegisterEncoder("custom_console", logger.NewConsoleEncoder) +} + +// NewDevelopmentLog returns a development logger based on uber zap and it output entry to stdout and stderr +func NewDevelopmentLog() (*uberZap.Logger, error) { + cfg := uberZap.NewDevelopmentConfig() + cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + return cfg.Build() +} + +// NewProductionLog returns a product logger based on uber zap +func NewProductionLog(outputPath string) (*uberZap.Logger, error) { + const pathLen = 1 + outputPaths := make([]string, pathLen, pathLen) + if outputPath == "" { + outputPaths = []string{filepath.Join("/", "log", "run.log"), "stdout"} + } else { + outputPaths = []string{outputPath, "stdout"} + } + + cfg := uberZap.Config{ + Level: uberZap.NewAtomicLevelAt(uberZap.InfoLevel), + Development: false, + DisableCaller: false, + DisableStacktrace: true, + Encoding: "json", + OutputPaths: outputPaths, + ErrorOutputPaths: outputPaths, + EncoderConfig: DefaultEncodeConfig, + } + + return cfg.Build() +} + +// NewConsoleLog returns a console logger based on uber zap +func NewConsoleLog() (*uberZap.Logger, error) { + outputPaths := []string{"stdout"} + cfg := uberZap.Config{ + Level: uberZap.NewAtomicLevelAt(uberZap.InfoLevel), + Development: false, + DisableCaller: false, + DisableStacktrace: true, + Encoding: "custom_console", + OutputPaths: outputPaths, + ErrorOutputPaths: outputPaths, + EncoderConfig: DefaultEncodeConfig, + } + consoleLogger, err := cfg.Build() + if err != nil { + return nil, err + } + return consoleLogger.WithOptions(uberZap.AddCaller(), uberZap.AddCallerSkip(skipLevel)), nil +} + +// NewWithLevel returns a log based on zap with Level +func NewWithLevel(coreInfo config.CoreInfo) (*uberZap.Logger, error) { + core, err := newCore(coreInfo) + if err != nil { + return nil, err + } + + return uberZap.New(core, uberZap.AddCaller(), uberZap.AddCallerSkip(skipLevel)), nil +} + +func newCore(coreInfo config.CoreInfo) (zapcore.Core, error) { + w, err := logger.CreateSink(coreInfo) + if err != nil { + return nil, err + } + syncer := zapcore.AddSync(w) + + encoderConfig := DefaultEncodeConfig + + fileEncoder := logger.NewCustomEncoder(&encoderConfig) + priority := uberZap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + var customLevel zapcore.Level + if err := customLevel.UnmarshalText([]byte(coreInfo.Level)); err != nil { + customLevel = zapcore.InfoLevel + } + return lvl >= customLevel + }) + + if coreInfo.Tick == 0 || coreInfo.First == 0 || coreInfo.Thereafter == 0 { + return zapcore.NewCore(fileEncoder, syncer, priority), nil + } + return zapcore.NewSamplerWithOptions(zapcore.NewCore(fileEncoder, syncer, priority), + time.Duration(coreInfo.Tick)*time.Second, coreInfo.First, coreInfo.Thereafter), nil +} + +// LoggerWithFormat zap logger +type LoggerWithFormat struct { + *uberZap.Logger +} + +// Infof stdout format and paras +func (z *LoggerWithFormat) Infof(format string, paras ...interface{}) { + z.Logger.Info(fmt.Sprintf(format, paras...)) +} + +// Errorf stdout format and paras +func (z *LoggerWithFormat) Errorf(format string, paras ...interface{}) { + z.Logger.Error(fmt.Sprintf(format, paras...)) +} + +// Warnf stdout format and paras +func (z *LoggerWithFormat) Warnf(format string, paras ...interface{}) { + z.Logger.Warn(fmt.Sprintf(format, paras...)) +} + +// Debugf stdout format and paras +func (z *LoggerWithFormat) Debugf(format string, paras ...interface{}) { + z.Logger.Debug(fmt.Sprintf(format, paras...)) +} + +// Fatalf stdout format and paras +func (z *LoggerWithFormat) Fatalf(format string, paras ...interface{}) { + z.Logger.Fatal(fmt.Sprintf(format, paras...)) +} diff --git a/functionsystem/apps/meta_service/common/logger/zap/zaplog_test.go b/functionsystem/apps/meta_service/common/logger/zap/zaplog_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3daf580ccaad1e773a10a22cd128bdeeca12efcb --- /dev/null +++ b/functionsystem/apps/meta_service/common/logger/zap/zaplog_test.go @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package zap + +import ( + "testing" + + "meta_service/common/logger/config" + + "github.com/stretchr/testify/assert" +) + +// TestNewDevelopmentLog Test New Development Log +func TestNewDevelopmentLog(t *testing.T) { + if _, err := NewDevelopmentLog(); err != nil { + t.Errorf("NewDevelopmentLog() = %q, wants *logger", err) + } +} + +func TestNewProductionLog(t *testing.T) { + _, err := NewProductionLog("") + assert.NotNil(t, err) + + _, err = NewProductionLog("test") + assert.Nil(t, err) +} + +func TestNewConsoleLog(t *testing.T) { + _, err := NewConsoleLog() + assert.Nil(t, err) +} + +func TestNewWithLevel(t *testing.T) { + _, err := NewWithLevel(config.CoreInfo{}) + assert.NotNil(t, err) +} + +func TestNewCore(t *testing.T) { + coreInfo := config.CoreInfo{} + newCore(coreInfo) +} diff --git a/functionsystem/apps/meta_service/common/metadata/code_metadata.go b/functionsystem/apps/meta_service/common/metadata/code_metadata.go new file mode 100644 index 0000000000000000000000000000000000000000..a4397f170156523c1ab055eeb5aceef06a0874b9 --- /dev/null +++ b/functionsystem/apps/meta_service/common/metadata/code_metadata.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metadata + +// CodeMetaData define code meta info +type CodeMetaData struct { + // for repo manage package if local means package not controlled by repo + CodeUploadType string `json:"codeUploadType" valid:",optional"` + Sha512 string `json:"sha512" valid:"optional"` + LocalMetaData + S3MetaData +} + +// S3MetaData define meta function info for OBS +type S3MetaData struct { + AppID string `json:"appId" valid:"stringlength(1|128),optional"` + BucketID string `json:"bucketId" valid:"stringlength(1|255),optional"` + ObjectID string `json:"objectId" valid:"stringlength(1|255),optional"` + BucketURL string `json:"bucketUrl" valid:"url,optional"` + CodeType string `json:"code_type" valid:",optional"` + CodeURL string `json:"code_url" valid:",optional"` + CodeFileName string `json:"code_filename" valid:",optional"` + FuncCode FuncCode `json:"func_code" valid:",optional"` +} + +// LocalMetaData define meta function info for local +type LocalMetaData struct { + StorageType string `json:"storage_type" valid:",optional"` + CodePath string `json:"code_path" valid:"optional"` +} diff --git a/functionsystem/apps/meta_service/common/metadata/faasfunction.go b/functionsystem/apps/meta_service/common/metadata/faasfunction.go new file mode 100644 index 0000000000000000000000000000000000000000..b970e84358170efc5766d4ae8ecd0243d2579535 --- /dev/null +++ b/functionsystem/apps/meta_service/common/metadata/faasfunction.go @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metadata define struct of metadata stored in storage like etcd +package metadata + +// HTTPResponse is general http response +type HTTPResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// LogTankService - +type LogTankService struct { + GroupID string `json:"logGroupId" valid:",optional"` + StreamID string `json:"logStreamId" valid:",optional"` +} + +// TraceService - +type TraceService struct { + TraceAK string `json:"tracing_ak" valid:",optional"` + TraceSK string `json:"tracing_sk" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` +} + +// Initializer include initializer handler and timeout +type Initializer struct { + Handler string `json:"initializer_handler" valid:",optional"` + Timeout int64 `json:"initializer_timeout" valid:",optional"` +} + +// PreStop include pre_stop handler and timeout +type PreStop struct { + Handler string `json:"pre_stop_handler" valid:",optional"` + Timeout int64 `json:"pre_stop_timeout" valid:",optional"` +} + +// FuncMountConfig function mount config +type FuncMountConfig struct { + FuncMountUser FuncMountUser `json:"mount_user" valid:",optional"` + FuncMounts []FuncMount `json:"func_mounts" valid:",optional"` +} + +// FuncMountUser function mount user +type FuncMountUser struct { + UserID int `json:"user_id" valid:",optional"` + GroupID int `json:"user_group_id" valid:",optional"` +} + +// FuncMount function mount +type FuncMount struct { + MountType string `json:"mount_type" valid:",optional"` + MountResource string `json:"mount_resource" valid:",optional"` + MountSharePath string `json:"mount_share_path" valid:",optional"` + LocalMountPath string `json:"local_mount_path" valid:",optional"` + Status string `json:"status" valid:",optional"` +} + +// Role include x_role and app_x_role +type Role struct { + XRole string `json:"xrole" valid:",optional"` + AppXRole string `json:"app_xrole" valid:",optional"` +} + +// FunctionDeploymentSpec define function deployment spec +type FunctionDeploymentSpec struct { + BucketID string `json:"bucket_id"` + ObjectID string `json:"object_id"` + Layers string `json:"layers"` + DeployDir string `json:"deploydir"` +} + +// InstanceResource describes the cpu and memory info of an instance +type InstanceResource struct { + CPU string `json:"cpu"` + Memory string `json:"memory"` + CustomResources map[string]int64 `json:"customresources"` +} + +// Worker define a worker +type Worker struct { + Instances []*Instance `json:"instances"` + FunctionName string `json:"functionname"` + FunctionVersion string `json:"functionversion"` + Tenant string `json:"tenant"` + Business string `json:"business"` +} + +// Instance define a instance +type Instance struct { + IP string `json:"ip"` + Port string `json:"port"` + GrpcPort string `json:"grpcPort"` + InstanceID string `json:"instanceID,omitempty"` + DeployedIP string `json:"deployed_ip"` + DeployedNode string `json:"deployed_node"` + DeployedNodeID string `json:"deployed_node_id"` + TenantID string `json:"tenant_id"` +} + +// ResourceStack stores properties of resource stack +type ResourceStack struct { + StackID string `json:"id" valid:"required"` + CPU int64 `json:"cpu" valid:"required"` + Mem int64 `json:"mem" valid:"required"` + CustomResources map[string]int64 `json:"customResources,omitempty" valid:"optional"` +} + +// ResourceGroup stores properties of resource group +type ResourceGroup struct { + GroupID string `json:"id" valid:"required"` + DeployOption string `json:"deployOption" valid:"required"` + GroupState string `json:"groupState" valid:"required"` + ResourceStacks []ResourceStack `json:"resourceStacks" valid:"required"` + ScheduledStacks map[string][]ResourceStack `json:"scheduledStacks,omitempty" valid:"optional"` +} + +// AffinityInfo is data affinity information +type AffinityInfo struct { + AffinityRequest AffinityRequest + AffinityNode string // if AffinityNode is not empty, the affinity node has been calculated + NeedToForward bool +} + +// AffinityRequest is affinity request parameter +type AffinityRequest struct { + Strategy string `json:"strategy"` + ObjectIDs []string `json:"object_ids"` +} + +// GroupInfo stores groupID and stackID +type GroupInfo struct { + GroupID string `json:"groupID"` + StackID string `json:"stackID"` +} + +// MetricsData shows the quantities of a specific resource +type MetricsData struct { + TotalResource float32 `json:"totalResource"` + InUseResource float32 `json:"inUseResource"` +} + +// ResourceMetrics contains several resources' MetricsData +type ResourceMetrics map[string]MetricsData + +// WorkerMetrics stores metrics used for scheduler +type WorkerMetrics struct { + SystemResources ResourceMetrics + // key levels: functionUrn instanceID + FunctionResources map[string]map[string]ResourceMetrics +} + +// UserAgency define AK/SK of user's agency +type UserAgency struct { + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + Token string `json:"token"` +} + +// FaaSFuncCode include function code file and link info for FaaS +type FaaSFuncCode struct { + File string `json:"file" valid:",optional"` + Link string `json:"link" valid:",optional"` +} + +// FaaSStrategyConfig - +type FaaSStrategyConfig struct { + Concurrency int `json:"concurrency" valid:",optional"` +} + +// FaaSFuncMeta define function meta info for FaaS +type FaaSFuncMeta struct { + FuncMetaData FaaSFuncMetaData `json:"funcMetaData" valid:",optional"` + CodeMetaData CodeMetaData `json:"codeMetaData" valid:",optional"` + EnvMetaData EnvMetaData `json:"envMetaData" valid:",optional"` + ResourceMetaData FaaSResourceMetaData `json:"resourceMetaData" valid:",optional"` + InstanceMetaData FaaSInstanceMetaData `json:"instanceMetaData" valid:",optional"` + ExtendedMetaData FaaSExtendedMetaData `json:"extendedMetaData" valid:",optional"` +} + +// FaaSFuncMetaData define meta data of functions for FaaS +type FaaSFuncMetaData struct { + Layers []*FaaSLayer `json:"layers" valid:",optional"` + Name string `json:"name"` + FunctionDescription string `json:"description" valid:"stringlength(1|1024)"` + FunctionURN string `json:"functionUrn"` + ReversedConcurrency int `json:"reversedConcurrency" valid:"int"` + TenantID string `json:"tenantId"` + Tags map[string]string `json:"tags" valid:",optional"` + HostAliasConfig map[string]string `json:"hostAliasConfig,omitempty"` + FunctionUpdateTime string `json:"functionUpdateTime" valid:",optional"` + FunctionVersionURN string `json:"functionVersionUrn"` + RevisionID string `json:"revisionId" valid:"stringlength(1|20),optional"` + CodeSize int `json:"codeSize" valid:"int"` + CodeSha512 string `json:"codeSha512" valid:"stringlength(1|128),optional"` + Handler string `json:"handler" valid:"stringlength(1|255)"` + Runtime string `json:"runtime" valid:"stringlength(1|63)"` + Timeout int64 `json:"timeout" valid:"required"` + Version string `json:"version" valid:"stringlength(1|32)"` + VersionDescription string `json:"versionDescription" valid:"stringlength(1|1024)"` + DeadLetterConfig string `json:"deadLetterConfig" valid:"stringlength(1|255)"` + BusinessID string `json:"businessId" valid:"stringlength(1|32)"` + FunctionType string `json:"functionType" valid:",optional"` + FuncID string `json:"func_id" valid:",optional"` + FuncName string `json:"func_name" valid:",optional"` + DomainID string `json:"domain_id" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` + Service string `json:"service" valid:",optional"` + Dependencies string `json:"dependencies" valid:",optional"` + EnableCloudDebug string `json:"enable_cloud_debug" valid:",optional"` + IsStatefulFunction bool `json:"isStatefulFunction" valid:"optional"` + IsBridgeFunction bool `json:"isBridgeFunction" valid:"optional"` + IsStreamEnable bool `json:"isStreamEnable" valid:"optional"` + Type string `json:"type" valid:"optional"` + CreationTime string `json:"created" valid:",optional"` + EnableAuthInHeader bool `json:"enable_auth_in_header" valid:"optional"` + DNSDomainCfg []DNSDomainInfo `json:"dns_domain_cfg" valid:",optional"` + VPCTriggerImage string `json:"vpcTriggerImage" valid:",optional"` +} + +// FaaSResourceMetaData include resource data such as cpu and memory +type FaaSResourceMetaData struct { + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + GpuMemory int64 `json:"gpu_memory"` + EnableDynamicMemory bool `json:"enable_dynamic_memory" valid:",optional"` + CustomResources string `json:"customResources" valid:",optional"` + EnableTmpExpansion bool `json:"enable_tmp_expansion" valid:",optional"` + EphemeralStorage int `json:"ephemeral_storage" valid:"int,optional"` +} + +// FaaSInstanceMetaData define instance meta data of FG functions +type FaaSInstanceMetaData struct { + MaxInstance int64 `json:"maxInstance" valid:",optional"` + MinInstance int64 `json:"minInstance" valid:",optional"` + ConcurrentNum int `json:"concurrentNum" valid:",optional"` + InstanceType string `json:"instanceType" valid:",optional"` + IdleMode bool `json:"idleMode" valid:",optional"` + PoolLabel string `json:"poolLabel" valid:",optional"` + PoolID string `json:"poolId" valid:",optional"` +} + +// ReserveInstanceMetaData meta data for reserved instance +type ReserveInstanceMetaData struct { + InstanceMetaData FaaSInstanceMetaData `json:"instanceMetaData"` +} + +// FaaSExtendedMetaData define external meta data of functions +type FaaSExtendedMetaData struct { + ImageName string `json:"image_name" valid:",optional"` + Role Role `json:"role" valid:",optional"` + VpcConfig *VpcConfig `json:"func_vpc" valid:",optional"` + EndpointTenantVpc *VpcConfig `json:"endpoint_tenant_vpc" valid:",optional"` + FuncMountConfig *FuncMountConfig `json:"mount_config" valid:",optional"` + StrategyConfig FaaSStrategyConfig `json:"strategy_config" valid:",optional"` + ExtendConfig string `json:"extend_config" valid:",optional"` + Initializer Initializer `json:"initializer" valid:",optional"` + PreStop PreStop `json:"pre_stop" valid:",optional"` + Heartbeat Heartbeat `json:"heartbeat" valid:",optional"` + EnterpriseProjectID string `json:"enterprise_project_id" valid:",optional"` + LogTankService LogTankService `json:"log_tank_service" valid:",optional"` + TraceService TraceService `json:"tracing_config" valid:",optional"` + CustomContainerConfig CustomContainerConfig `json:"custom_container_config" valid:",optional"` + AsyncConfigLoaded bool `json:"async_config_loaded" valid:",optional"` + RestoreHook RestoreHook `json:"restore_hook,omitempty" valid:",optional"` + NetworkController NetworkController `json:"network_controller" valid:",optional"` + UserAgency UserAgency `json:"user_agency" valid:",optional"` +} + +// Heartbeat define user custom heartbeat function config +type Heartbeat struct { + // Handler define heartbeat function entry + Handler string `json:"heartbeat_handler" valid:",optional"` +} + +// CustomContainerConfig contains the metadata for custom container +type CustomContainerConfig struct { + ControlPath string `json:"control_path" valid:",optional"` + Image string `json:"image" valid:",optional"` + Command []string `json:"command" valid:",optional"` + Args []string `json:"args" valid:",optional"` + WorkingDir string `json:"working_dir" valid:",optional"` + UID int `json:"uid" valid:",optional"` + GID int `json:"gid" valid:",optional"` +} + +// RestoreHook include restorehook handler and timeout +type RestoreHook struct { + Handler string `json:"restore_hook_handler,omitempty" valid:",optional"` + Timeout int64 `json:"restore_hook_timeout,omitempty" valid:",optional"` +} + +// NetworkController contains some special network settings +type NetworkController struct { + DisablePublicNetwork bool `json:"disable_public_network" valid:",optional"` + TriggerAccessVpcs []VpcInfo `json:"trigger_access_vpcs" valid:",optional"` +} + +// VpcInfo contains the information of VPC access restriction +type VpcInfo struct { + VpcName string `json:"vpc_name,omitempty"` + VpcID string `json:"vpc_id,omitempty"` +} + +// VpcConfig include info of function vpc +type VpcConfig struct { + ID string `json:"id,omitempty"` + DomainID string `json:"domain_id,omitempty"` + Namespace string `json:"namespace,omitempty"` + VpcName string `json:"vpc_name,omitempty"` + VpcID string `json:"vpc_id,omitempty"` + SubnetName string `json:"subnet_name,omitempty"` + SubnetID string `json:"subnet_id,omitempty"` + TenantCidr string `json:"tenant_cidr,omitempty"` + HostVMCidr string `json:"host_vm_cidr,omitempty"` + Gateway string `json:"gateway,omitempty"` + Xrole string `json:"xrole,omitempty"` +} + +// FaaSLayer define layer info for FaaS +type FaaSLayer struct { + BucketURL string `json:"bucketUrl" valid:"url,optional"` + ObjectID string `json:"objectId" valid:"stringlength(1|255),optional"` + BucketID string `json:"bucketId" valid:"stringlength(1|255),optional"` + AppID string `json:"appId" valid:"stringlength(1|128),optional"` + ETag string `json:"etag" valid:"optional"` + Link string `json:"link" valid:"optional"` + Name string `json:"name" valid:",optional"` + Sha256 string `json:"sha256" valid:"optional"` + DependencyType string `json:"dependencyType" valid:",optional"` +} + +// DNSDomainInfo dns domain info +type DNSDomainInfo struct { + ID string `json:"id"` + DomainName string `json:"domain_name"` + Type string `json:"type" valid:",optional"` + ZoneType string `json:"zone_type" valid:",optional"` +} diff --git a/functionsystem/apps/meta_service/common/metadata/function.go b/functionsystem/apps/meta_service/common/metadata/function.go new file mode 100644 index 0000000000000000000000000000000000000000..434910ce6383d136bb25d424a646539862c9bd87 --- /dev/null +++ b/functionsystem/apps/meta_service/common/metadata/function.go @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metadata define struct of metadata stored in storage like etcd +package metadata + +import ( + "fmt" + + "meta_service/common/logger/logservice" + "meta_service/common/trace" + "meta_service/common/types" + "meta_service/common/urnutils" +) + +// Function define function meta info +type Function struct { + FuncMetaData FuncMetaData `json:"funcMetaData" valid:",optional"` + CodeMetaData CodeMetaData `json:"codeMetaData" valid:",optional"` + EnvMetaData EnvMetaData `json:"envMetaData" valid:",optional"` + ResourceMetaData ResourceMetaData `json:"resourceMetaData" valid:",optional"` + ExtendedMetaData ExtendedMetaData `json:"extendedMetaData" valid:",optional"` +} + +// FuncMetaData define meta data of functions +type FuncMetaData struct { + Layers []CodeMetaData `json:"layers" valid:",optional"` + Name string `json:"name"` + FunctionDescription string `json:"description" valid:"stringlength(1|1024)"` + FunctionURN string `json:"functionUrn"` + ReversedConcurrency int `json:"reversedConcurrency" valid:"int"` + Tags map[string]string `json:"tags" valid:",optional"` + FunctionUpdateTime string `json:"functionUpdateTime" valid:",optional"` + FunctionVersionURN string `json:"functionVersionUrn"` + CodeSize int64 `json:"codeSize" valid:"int"` + CodeSha256 string `json:"codeSha256" valid:"stringlength(1|64),optional"` + CodeSha512 string `json:"codeSha512" valid:"stringlength(1|128),optional"` + Handler string `json:"handler" valid:"stringlength(1|255)"` + Runtime string `json:"runtime" valid:"stringlength(1|63)"` + Timeout int64 `json:"timeout" valid:"required"` + Version string `json:"version" valid:"stringlength(1|16)"` + VersionDescription string `json:"versionDescription" valid:"stringlength(1|1024)"` + DeadLetterConfig string `json:"deadLetterConfig" valid:"stringlength(1|255)"` + LatestVersionUpdateTime string `json:"latestVersionUpdateTime" valid:",optional"` + PublishTime string `json:"publishTime" valid:",optional"` + BusinessID string `json:"businessId" valid:"stringlength(1|32)"` + TenantID string `json:"tenantId"` + DomainID string `json:"domain_id" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` + RevisionID string `json:"revisionId" valid:"stringlength(1|20),optional"` + CreationTime string `json:"created" valid:",optional"` + StatefulFlag bool `json:"statefulFlag"` + HookHandler map[string]string `json:"hookHandler" valid:",optional"` +} + +// Layer define layer info +type Layer struct { + AppID string `json:"appId" valid:"stringlength(1|128),optional"` + BucketID string `json:"bucketId" valid:"stringlength(1|255),optional"` + ObjectID string `json:"objectId" valid:"stringlength(1|255),optional"` + BucketURL string `json:"bucketUrl" valid:"url,optional"` + Sha256 string `json:"sha256" valid:"optional"` +} + +// EnvMetaData define env info of functions +type EnvMetaData struct { + EnvKey string `json:"envKey,omitempty" valid:",optional"` + Environment string `json:"environment" valid:",optional"` + EncryptedUserData string `json:"encrypted_user_data" valid:",optional"` + CryptoAlgorithm string `json:"cryptoAlgorithm" valid:",optional"` +} + +// ResourceMetaData include resource data such as cpu and memory +type ResourceMetaData struct { + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + CustomResources string `json:"customResources" valid:",optional"` +} + +// InstanceMetaData define instance meta data of functions +type InstanceMetaData struct { + MaxInstance int64 `json:"maxInstance" valid:",optional"` + MinInstance int64 `json:"minInstance" valid:",optional"` + ConcurrentNum int `json:"concurrentNum" valid:",optional"` + CacheInstance int `json:"cacheInstance" valid:",optional"` + PoolLabel string `json:"poolLabel" valid:",optional"` + PoolID string `json:"poolId" valid:",optional"` +} + +// ExtendedMetaData define external meta data of functions +type ExtendedMetaData struct { + ImageName string `json:"image_name" valid:",optional"` + Role types.Role `json:"role" valid:",optional"` + FuncMountConfig types.FuncMountConfig `json:"mount_config" valid:",optional"` + StrategyConfig StrategyConfig `json:"strategy_config" valid:",optional"` + ExtendConfig string `json:"extend_config" valid:",optional"` + Initializer types.Initializer `json:"initializer" valid:",optional"` + PreStop types.PreStop `json:"pre_stop" valid:",optional"` + EnterpriseProjectID string `json:"enterprise_project_id" valid:",optional"` + LogTankService logservice.LogTankService `json:"log_tank_service" valid:",optional"` + TraceService trace.Service `json:"tracing_config" valid:",optional"` + UserType string `json:"user_type" valid:",optional"` + InstanceMetaData InstanceMetaData `json:"instance_meta_data" valid:",optional"` + ExtendedHandler map[string]string `json:"extended_handler" valid:",optional"` + ExtendedTimeout map[string]int `json:"extended_timeout" valid:",optional"` + Device types.Device `json:"device,omitempty" valid:",optional"` +} + +// FuncCode include function code file and link info +type FuncCode struct { + File string `json:"file" valid:",optional"` + Link string `json:"link" valid:",optional"` +} + +// StrategyConfig include concurrency of function +type StrategyConfig struct { + Concurrency int `json:"concurrency" valid:",optional"` +} + +// FunctionKey contains key info of a function +type FunctionKey struct { + BusinessID string `json:"businessID" valid:"optional"` + TenantID string `json:"tenantID" valid:"optional"` + ServiceID string `json:"serviceID" valid:"optional"` + FuncName string `json:"functionName" valid:"optional"` + FuncVersion string `json:"functionVersion" valid:"optional"` +} + +// ToAnonymousString return the anonymous string of funcMeta +func (f *FunctionKey) ToAnonymousString() string { + return fmt.Sprintf("%s-%s-%s-%s-%s", urnutils.Anonymize(f.TenantID), f.BusinessID, f.ServiceID, f.FuncName, + f.FuncVersion) +} + +// FullName return full function name(0-$serviceID-$funcName) of function +func (f *FunctionKey) FullName() string { + return fmt.Sprintf("0-%s-%s", f.ServiceID, f.FuncName) +} diff --git a/functionsystem/apps/meta_service/common/metadata/function_test.go b/functionsystem/apps/meta_service/common/metadata/function_test.go new file mode 100644 index 0000000000000000000000000000000000000000..50b8b94b76c46c594672fe39372dad847b54a030 --- /dev/null +++ b/functionsystem/apps/meta_service/common/metadata/function_test.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metadata define struct of metadata stored in storage like etcd +package metadata + +import "testing" + +func TestFunctionKey_FullName(t *testing.T) { + funcKey := &FunctionKey{ + ServiceID: "serviceA", + FuncName: "helloWorld", + } + + res := funcKey.FullName() + if res != "0-serviceA-helloWorld" { + t.Errorf("failed to get full name") + } + +} +func TestFunctionKey_ToAnonymousString(t *testing.T) { + funcKey := &FunctionKey{ + BusinessID: "1", + TenantID: "1234", + ServiceID: "serviceA", + FuncName: "helloWorld", + FuncVersion: "$latest", + } + res := funcKey.ToAnonymousString() + if res != "****-1-serviceA-helloWorld-$latest" { + t.Errorf("failed to AnonymousString") + } +} diff --git a/functionsystem/apps/meta_service/common/metadata/trigger.go b/functionsystem/apps/meta_service/common/metadata/trigger.go new file mode 100644 index 0000000000000000000000000000000000000000..2a39c2efc4af77398a5047003aeb7643c035c386 --- /dev/null +++ b/functionsystem/apps/meta_service/common/metadata/trigger.go @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metadata + +// CronTrigger use for crontrigger metainfo +type CronTrigger struct { + FunctionID string `json:"funcId" valid:"stringlength(1|255)"` + TriggerID string `json:"triggerId" valid:"stringlength(1|63)"` + + TriggerSwitch bool `json:"triggerSwitch" valid:",optional"` + TriggerMode string `json:"triggerMode" valid:"stringlength(1|16)"` + TriggerType string `json:"triggerType" valid:" stringlength(1|16)"` + + Payload string `json:"payload,omitempty" valid:",optional"` + Schedule string `json:"schedule,omitempty" valid:"stringlength(1|255)"` + StateKey string `json:"stateKey" valid:" stringlength(1|255)"` +} diff --git a/functionsystem/apps/meta_service/common/obs/api.go b/functionsystem/apps/meta_service/common/obs/api.go new file mode 100644 index 0000000000000000000000000000000000000000..2a4cb93f9c3ae7a073d4c474344f05223e8abf8c --- /dev/null +++ b/functionsystem/apps/meta_service/common/obs/api.go @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package obs is the obs client +package obs + +import ( + "crypto/tls" + "net/http" + "strings" + "sync" + + "meta_service/common/utils" + + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + + commonTLS "meta_service/common/tls" +) + +const ( + // caFile default Certificate authority file path + caFile = "/home/sn/module/ca.crt" + httpPrefix = "http://" + httpsPrefix = "https://" + uint64Base = 10 + dirMode = 0o700 + fileMode = 0o600 + readSize = 32 * 1024 +) + +var tlsConfig = struct { + sync.Once + config *tls.Config +}{} + +var ( + secretKeyRecord []byte + accessKeyRecord []byte +) + +// Option contains the options to connect the OBS +type Option struct { + Secure bool + AccessKey string + SecretKey string + Endpoint string + CaFile string + TrustedCA bool + MaxRetryCount int + Timeout int +} + +// NewObsClient instantiate obs client, adds automatic verification of signature. +func NewObsClient(o Option) (*obs.ObsClient, error) { + var endpoint string + var urlPrefix string + var err error + + var transport *http.Transport = nil + + endpoint, urlPrefix = getEndpoint(o.Endpoint) + if o.Secure { + // load ca certificate + if o.CaFile == "" { + o.CaFile = caFile + } + tlsConfig := getTLSConfig(o.CaFile, o.TrustedCA) + transport = &http.Transport{ + TLSClientConfig: tlsConfig, + DisableCompression: true, + } + urlPrefix = httpsPrefix + } else { + endpoint, err = utils.Domain2IP(endpoint) + if err != nil { + return nil, err + } + } + + obsClient, err := obs.New(o.AccessKey, o.SecretKey, urlPrefix+endpoint, + obs.WithSslVerify(o.Secure), obs.WithHttpTransport(transport), obs.WithMaxRetryCount(o.MaxRetryCount), + obs.WithConnectTimeout(o.Timeout), obs.WithSocketTimeout(o.Timeout), obs.WithPathStyle(true)) + if err != nil { + return nil, err + } + secretKeyRecord = []byte(o.SecretKey) + accessKeyRecord = []byte(o.AccessKey) + return obsClient, nil +} + +// getTLSConfig get tls config +func getTLSConfig(ca string, trustedCA bool) *tls.Config { + tlsConfig.Do(func() { + if trustedCA { + // No ca is required. + tlsConfig.config = commonTLS.NewTLSConfig() + tlsConfig.config.InsecureSkipVerify = true + } else { + tlsConfig.config = commonTLS.NewTLSConfig(commonTLS.WithRootCAs(ca)) + } + }) + return tlsConfig.config +} + +func getEndpoint(endpoint string) (string, string) { + var urlPrefix string + urlPrefix = httpPrefix + if strings.HasPrefix(endpoint, httpPrefix) { + endpoint = strings.TrimPrefix(endpoint, httpPrefix) + urlPrefix = httpPrefix + } + if strings.HasPrefix(endpoint, httpsPrefix) { + endpoint = strings.TrimPrefix(endpoint, httpsPrefix) + urlPrefix = httpsPrefix + } + return endpoint, urlPrefix +} + +// ClearSecretKeyMemory clear secretKey memory +func ClearSecretKeyMemory() { + utils.ClearStringMemory(string(secretKeyRecord)) + utils.ClearStringMemory(string(accessKeyRecord)) +} diff --git a/functionsystem/apps/meta_service/common/obs/api_test.go b/functionsystem/apps/meta_service/common/obs/api_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7439ee8a0811519ba458f9dc72a46f4ad6ffa1a9 --- /dev/null +++ b/functionsystem/apps/meta_service/common/obs/api_test.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package obs is the obs client +package obs + +import ( + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + "github.com/stretchr/testify/assert" +) + +func TestNewObsClient(t *testing.T) { + conf := Option{} + conf.Secure = true + _, err := NewObsClient(conf) + assert.Equal(t, nil, err) + + conf.Secure = false + _, err = NewObsClient(conf) + assert.NotEqual(t, nil, err) +} + +func Test_getEndpoint(t *testing.T) { + endpoint := httpPrefix + endpoint, urlPrefix := getEndpoint(endpoint) + assert.Equal(t, httpPrefix, urlPrefix) + + endpoint = httpsPrefix + endpoint, urlPrefix = getEndpoint(endpoint) + assert.Equal(t, httpsPrefix, urlPrefix) +} + +func TestClearSecretKeyMemory(t *testing.T) { + ClearSecretKeyMemory() +} + +func TestGetTLSConfig(t *testing.T) { + getTLSConfig("", true) +} diff --git a/functionsystem/apps/meta_service/common/reader/reader.go b/functionsystem/apps/meta_service/common/reader/reader.go new file mode 100644 index 0000000000000000000000000000000000000000..0be3f5286c409116f31c76b32e576444f4d441a0 --- /dev/null +++ b/functionsystem/apps/meta_service/common/reader/reader.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package reader provides ReadFile with timeConsumption +package reader + +import ( + "fmt" + "io/ioutil" + "os" + "time" +) + +// MaxReadFileTime elapsed time allowed to read config file from disk +const MaxReadFileTime = 10 + +// ReadFileWithTimeout is to ReadFile and count timeConsumption at same time +func ReadFileWithTimeout(configFile string) ([]byte, error) { + stopCh := make(chan struct{}) + go printTimeOut(stopCh) + data, err := ioutil.ReadFile(configFile) + close(stopCh) + return data, err +} + +// ReadFileInfoWithTimeout is to Read FileInfo and count timeConsumption at same time +func ReadFileInfoWithTimeout(filePath string) (os.FileInfo, error) { + stopCh := make(chan struct{}) + go printTimeOut(stopCh) + fileInfo, err := os.Stat(filePath) + close(stopCh) + return fileInfo, err +} + +// printTimeOut print error info every 10s after timeout +func printTimeOut(stopCh <-chan struct{}) { + if stopCh == nil { + os.Exit(0) + return + } + timer := time.NewTicker(time.Second * MaxReadFileTime) + count := 0 + for { + <-timer.C + select { + case _, ok := <-stopCh: + if !ok { + timer.Stop() + return + } + default: + count += MaxReadFileTime + fmt.Printf("ReadFile Timeout: elapsed time %ds\n", count) + } + } +} diff --git a/functionsystem/apps/meta_service/common/reader/reader_test.go b/functionsystem/apps/meta_service/common/reader/reader_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7664068a5411d8a0a0180c87523ec5f7f5f2aa51 --- /dev/null +++ b/functionsystem/apps/meta_service/common/reader/reader_test.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package reader provides ReadFile with timeConsumption +package reader + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" +) + +func TestReadFileWithTimeout(t *testing.T) { + patch := gomonkey.ApplyFunc(ioutil.ReadFile, func(string) ([]byte, error) { + return nil, nil + }) + data, _ := ReadFileWithTimeout("/sn/home") + assert.Nil(t, data) + patch.Reset() +} + +func TestReadFileInfoWithTimeout(t *testing.T) { + patch := gomonkey.ApplyFunc(os.Stat, func(string) (os.FileInfo, error) { + return nil, nil + }) + fileInfo, _ := ReadFileInfoWithTimeout("/sn/home") + assert.Nil(t, fileInfo) + patch.Reset() +} + +func TestPrintTimeout(t *testing.T) { + stopCh := make(chan struct{}) + go printTimeOut(stopCh) + time.Sleep(time.Second * 15) + close(stopCh) +} + +func TestPrintTimeoutErr(t *testing.T) { + test := 0 + patch := gomonkey.ApplyFunc(os.Exit, func(code int) { + test++ + }) + printTimeOut(nil) + assert.EqualValues(t, test, 1) + patch.Reset() +} diff --git a/functionsystem/apps/meta_service/common/signals/signal.go b/functionsystem/apps/meta_service/common/signals/signal.go new file mode 100644 index 0000000000000000000000000000000000000000..6f3751208972f7019056ea0c61bc873f4f13fe27 --- /dev/null +++ b/functionsystem/apps/meta_service/common/signals/signal.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package signals is for shutdownHandler +package signals + +import ( + "os" + "os/signal" + "syscall" +) + +var ( + shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGKILL} + onlyOneSignalHandler = make(chan struct{}) + shutdownHandler chan os.Signal +) + +const channelCount = 2 + +// WaitForSignal defines signal handler process. +func WaitForSignal() <-chan struct{} { + close(onlyOneSignalHandler) // panic when called twice + + // 2 is the length of shutdown Handler channel + shutdownHandler = make(chan os.Signal, channelCount) + + stopCh := make(chan struct{}) + signal.Notify(shutdownHandler, shutdownSignals...) + + go func() { + <-shutdownHandler + close(stopCh) + <-shutdownHandler + os.Exit(1) + }() + + return stopCh +} diff --git a/functionsystem/apps/meta_service/common/signals/signal_test.go b/functionsystem/apps/meta_service/common/signals/signal_test.go new file mode 100644 index 0000000000000000000000000000000000000000..19a23bbc15243811391d00a85e83cdf3a6a24d8e --- /dev/null +++ b/functionsystem/apps/meta_service/common/signals/signal_test.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package signals + +import ( + "syscall" + "testing" + "time" + + "gotest.tools/assert" +) + +func TestWaitForSignal(t *testing.T) { + stopCh := WaitForSignal() + + go func() { + time.Sleep(200 * time.Millisecond) + shutdownHandler <- syscall.SIGTERM + }() + select { + case <-stopCh: + t.Log("received termination signal") + case <-time.After(time.Second): + t.Fatal("failed to signal in 1s") + } + + _, ok := <-stopCh + assert.Equal(t, ok, false) +} diff --git a/functionsystem/apps/meta_service/common/snerror/badresponse.go b/functionsystem/apps/meta_service/common/snerror/badresponse.go new file mode 100644 index 0000000000000000000000000000000000000000..53812cf025a5caa8d89b6ecb770ff5644aca91b3 --- /dev/null +++ b/functionsystem/apps/meta_service/common/snerror/badresponse.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package snerror is basic information contained in the SN error. +package snerror + +import ( + "encoding/json" + "fmt" +) + +// BadResponse HTTP request message that does not return 200 +type BadResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// ConvertError Convert SNError to BadResponse +func ConvertError(error SNError) BadResponse { + return BadResponse{ + Code: error.Code(), + Message: error.Error(), + } +} + +// Marshal Marshal SNError to byte +func Marshal(error SNError) []byte { + b, e := json.Marshal(ConvertError(error)) + if e != nil { + return []byte(fmt.Sprintf("marshal snError failed %s.snerror/badresponse:33", e)) + } + return b +} + +// ConvertBadResponse Convert BadResponse body to error +func ConvertBadResponse(badResponseBody []byte) error { + badResponse := &BadResponse{} + if err := json.Unmarshal(badResponseBody, badResponse); err != nil { + return err + } + return New(badResponse.Code, badResponse.Message) +} diff --git a/functionsystem/apps/meta_service/common/snerror/errorcode.go b/functionsystem/apps/meta_service/common/snerror/errorcode.go new file mode 100644 index 0000000000000000000000000000000000000000..883afbd688874b5b42510cddfa6d901adc8168d9 --- /dev/null +++ b/functionsystem/apps/meta_service/common/snerror/errorcode.go @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package snerror is basic information contained in the SN error. +package snerror + +// The value of commonError must be a 6-digit integer starting with 33. +const ( + // Internal error ,if the value is 331404, try again. + InternalError = 330404 + InternalErrorRetry = 331404 + InternalErrorMsg = "internal system error" +) + +// ClientInsExceptionCode used in functionbus and worker +const ( + // ClientInsExceptionCode function instance exception, for example OOM + ClientInsExceptionCode = 4020 + // ClientInsException - + ClientInsException = "function instance exception" + // MaxRequestBodySizeErr maxRequestBodySizeErr - + MaxRequestBodySizeErr = 4040 + MaxRequestBodySizeMsg = "the size of request body is beyond maximum:%d, body size:%d" + + // UnableSpecifyResourceCode UnableSpecifyResource unable to specify resource in a scene where no resource specified + UnableSpecifyResourceCode = 4026 + // UnableSpecifyResourceMsg - + UnableSpecifyResourceMsg = "unable to specify resource" +) + +// user error code +const ( + // FunctionEntryNotFound function entry not found + FunctionEntryNotFound = 4001 + // FunctionExceptionCode code of function exception + FunctionExceptionCode = 4002 + // FunctionNotStateful - + FunctionNotStateful = 4006 + // IllegalArgumentErrorCode code of illegal argument + IllegalArgumentErrorCode = 4008 + // StreamExceedLimitErrorCode code of illegal argument + StreamExceedLimitErrorCode = 4009 + // RuntimeInvokeTimeoutCode code of invoking runtime timed out + RuntimeInvokeTimeoutCode = 4010 + // RuntimeLoadTimeoutCode code of load function timed out + RuntimeLoadTimeoutCode = 4012 + // RequestAuthCheckFail - + RequestAuthCheckFail = 4013 + // NoSuchFunction error code of function is not found + NoSuchFunction = 4024 + // CreateStateByStatelessFunction error code of creating state using stateless functions + CreateStateByStatelessFunction = 4025 + // NoSuchInstanceName error code of querying instanceID by using instanceName is not exist + NoSuchInstanceName = 4026 + // InstanceNameExist error code of create instanceID by using instanceName but the instanceName is already used + InstanceNameExist = 4027 + // TooManyDependencyFiles function + TooManyDependencyFiles = 4028 + // InvokeStatefulFunctionWithoutStateID error code when user invoke a stateful function without stateID + InvokeStatefulFunctionWithoutStateID = 4029 + + // TenantStateNumExceedLimit - + TenantStateNumExceedLimit = 4143 + + // FunctionInitFailCode code of function init fail + FunctionInitFailCode = 4201 + // RuntimeBootstrapNotFound code of runtime bootstrap not found + RuntimeBootstrapNotFound = 4202 + // RuntimeProcessExit code of runtime exiting + RuntimeProcessExit = 4204 + // RuntimeMemoryExceedLimit runtime has consumed too much memory + RuntimeMemoryExceedLimit = 4205 + // RuntimeMountErrorCode error code of mount failure + RuntimeMountErrorCode = 4206 + // DiskUsageExceed disk usage exceed code + DiskUsageExceed = 4207 + // RequestCanceled - + RequestCanceled = 4208 + // RuntimeUnHealthy Runtime is UnHealthy + RuntimeUnHealthy = 4209 + // RuntimeInitTimeoutCode code of initialing runtime timed out + RuntimeInitTimeoutCode = 4211 + // VpcNoOperationalPermissions vpc has no operational permissions + VpcNoOperationalPermissions = 4212 + // FunctionRestoreHookFailCode - + FunctionRestoreHookFailCode = 4213 + // RuntimeRestoreHookTimeoutCode - + RuntimeRestoreHookTimeoutCode = 4214 + // BridgeManagerFailCode error code of bridge manager plugin failure + BridgeManagerFailCode = 4215 + // ExtensionShellNotFound error code of extension shell not found + ExtensionShellNotFound = 4216 + // ExtensionExecFailed error code of extension exec failed + ExtensionExecFailed = 4217 + // TooManySpecializedFails marked that there are too many specialized fails + TooManySpecializedFails = 4218 + // VPCNotFound error code of VPC not found + VPCNotFound = 4219 + // RuntimeDirForbidden runtime dir is forbidden + RuntimeDirForbidden = 4220 + // INodeUsageExceed - inode usage exceed code + INodeUsageExceed = 4221 + // VPCXRoleNotFound vcp xrole not func + VPCXRoleNotFound = 4222 + // StatefulFunctionReloading - stateful function instance is reloading + StatefulFunctionReloading = 4223 + // WebSocketDownStreamConnError - cannot connect websocket server of user + WebSocketDownStreamConnError = 4224 + // ZipFormatInvalid zip code package format invalid + ZipFormatInvalid = 4225 +) + +// instance manager error code +const ( + // ClusterOverloadCode cluster is overload and unavailable now + ClusterOverloadCode = 150430 + // GettingPodErrorCode getting pod error code + GettingPodErrorCode = 150431 + // VIPClusterOverloadCode cluster has no available resource + VIPClusterOverloadCode = 150510 + // FuncMetaNotFound function meta not found, this error occurs only when the internal service is abnormal. + FuncMetaNotFound = 150424 + // ReachMaxInstancesCode reach function max instances + ReachMaxInstancesCode = 150429 + // ReachMaxOnDemandInstancesPerTenant reach tenant max on-demand instances + ReachMaxOnDemandInstancesPerTenant = 150432 + // ReachMaxReversedInstancesPerTenant reach tenant max reversed instances + ReachMaxReversedInstancesPerTenant = 150433 + // FunctionIsDisabled function is disabled + FunctionIsDisabled = 150434 + // RefreshSilentFunc waiting for silent function to refresh, retry required + RefreshSilentFunc = 150435 + // NotEnoughNIC marked that there were not enough network cards + NotEnoughNIC = 150436 + // InternalVPCError internal vpc error + InternalVPCError = 150437 + // InsufficientEphemeralStorage marked that ephemeral storage is insufficient + InsufficientEphemeralStorage = 150438 + // CancelGeneralizePod user update function metadata to cancel generalize pod while generalizing is not finished + CancelGeneralizePod = 150439 + // CancelCheckpoint cancel checkpoint when the format of snapshot data is abnormal + CancelCheckpoint = 150440 + // StreamConnException stream connection exception + StreamConnException = 150450 +) + +// worker error code +const ( + // WorkerInternalErrorCode code of unexpected error in worker + WorkerInternalErrorCode = 161900 + // ReadingCodeTimeoutCode reading code package timed out + ReadingCodeTimeoutCode = 161901 + // CallFunctionErrorCode code of calling other function error + CallFunctionErrorCode = 161902 + // FuncInsExceptionCode function instance exception + FuncInsExceptionCode = 161903 + // CheckSumErrorCode code of check sum error + CheckSumErrorCode = 161904 + // DownLoadCodeErrorCode code of download code error + DownLoadCodeErrorCode = 161905 + // RPCClientEmptyErrorCode code of when rpc client is nil + RPCClientEmptyErrorCode = 161906 + // RuntimeManagerProcessExited runtime-manager process exited code + RuntimeManagerProcessExited = 161907 + // WorkerPingVpcGatewayError code of worker ping vpc gateway error + WorkerPingVpcGatewayError = 161908 + // UploadSnapshotErrorCode code of worker upload snapshot error + UploadSnapshotErrorCode = 161909 + // RestoreDeadErrorCode code of restore is dead + RestoreDeadErrorCode = 161910 + // ContentInconsistentErrorCode code of worker content inconsistent error + ContentInconsistentErrorCode = 161911 + // WebSocketUpStreamConnError cannot connect websocket server of control plane + WebSocketUpStreamConnError = 161912 + // WebSocketRequestInternalError internal error when processing websocket related request + WebSocketRequestInternalError = 161913 +) + +const ( + // MinUserCode min user code + MinUserCode = 4000 + // MinSysCode min system code + MinSysCode = 10000 +) + +var ( + userErrorMsg = map[int]string{ + FunctionEntryNotFound: "function entry exception", // 4001 + FunctionExceptionCode: "function invocation exception", // 4002 + FunctionNotStateful: "function name has no service ID", // 4006 + IllegalArgumentErrorCode: "response body size %d exceeds the limit of %d", // 4008 + StreamExceedLimitErrorCode: "send stream exceed limit", // 4009 + RuntimeInvokeTimeoutCode: "call invoke timeout %s", // 4010 + RuntimeInitTimeoutCode: "runtime initialization timed out after %s", // 4011 + RuntimeLoadTimeoutCode: "load function timed out after %s", // 4012 + RequestAuthCheckFail: "failed to check auth of the request", // 4013 + NoSuchFunction: "function metadata is not found", // 4024 + CreateStateByStatelessFunction: "can not create a state using the stateless function", // 4025 + NoSuchInstanceName: "the instance name (%s) or instance id (%s) does not exist", // 4026 + InstanceNameExist: "instanceID cannot be created repeatedly", // 4027 + TooManyDependencyFiles: "amount of files in package is over %d ,exceeding the limit", // 4028 + FunctionInitFailCode: "function initialization failed", // 4201 + RuntimeBootstrapNotFound: "runtime bootstrap file is not found", // 4202 + RuntimeProcessExit: "runtime process is exited", // 4204 + RuntimeMemoryExceedLimit: "runtime memory limit exceeded", // 4205 + RuntimeMountErrorCode: "failed to mount volumes for function", // 4206 + DiskUsageExceed: "disk usage exceed limit", // 4207 + RequestCanceled: "function invocation canceled", // 4208 + RuntimeUnHealthy: "runtime is unHealthy", // 4209 + MaxRequestBodySizeErr: "the size of request body is beyond maximum:%d, body size:%d", // 4140 + TenantStateNumExceedLimit: "the number of states exceeds the limit: %d", // 4143 + FunctionRestoreHookFailCode: "function restore failed", // 4213 + RuntimeRestoreHookTimeoutCode: "runtime restore timed out after %s", // 4214 + BridgeManagerFailCode: "bridge manager internal error: %s", // 4215 + ExtensionShellNotFound: "extension shell file is not found", // 4216 + ExtensionExecFailed: "extension exec failed, error: %s", // 4217 + TooManySpecializedFails: "the function fails to be started for multiple times", // 4218 + VPCNotFound: "VPC item not found", // 4219 + RuntimeDirForbidden: "runtime dir /opt/function/runtime and /home/snuser is forbidden", // 4220 + INodeUsageExceed: "Inode usage exceed limit", // 4221 + VPCXRoleNotFound: "VPC can't find xrole", // 4222 + StatefulFunctionReloading: "Stateful function instance is reloading when scaleUp", // 4223 + WebSocketDownStreamConnError: "cannot connect user's websocket server", // 4224 + } + systemErrorMsg = map[int]string{ + FuncMetaNotFound: "function metadata not found", // 150424 + ClusterOverloadCode: "the cluster is overload and unavailable now", // 150430 + GettingPodErrorCode: "getting pod from pool error", // 150431 + RefreshSilentFunc: "waiting for refreshing the silent function", // 150434 + NotEnoughNIC: "not enough network cards", // 150436 + InsufficientEphemeralStorage: "insufficient ephemeral storage", // 150438 + VIPClusterOverloadCode: "the VIP node is overload and unavailable now", // 150510 + StreamConnException: "streaming data exception", // 150450 + WorkerInternalErrorCode: "worker internal error: %s", // 161900 + ReadingCodeTimeoutCode: "reading the function code package timed out", // 161901 + CallFunctionErrorCode: "call other function error", // 161902 + FuncInsExceptionCode: "function instance exception", // 161903 + CheckSumErrorCode: "check file sum error: %s", // 161904 + DownLoadCodeErrorCode: "download code from obs error: %s, bucket: %s, object: %s, layer: %s", // 161905 + RPCClientEmptyErrorCode: "rpc client is nil", // 161906 + RuntimeManagerProcessExited: "runtime-manager process exited", // 161907 + WorkerPingVpcGatewayError: "ping vpc gateway error", // 161908 + UploadSnapshotErrorCode: "upload snapshot to obs err: %s, bucket: %s, object: %s, fileName: %s", // 161909 + RestoreDeadErrorCode: "function snapshot restore is dead", // 161910 + WebSocketUpStreamConnError: "cannot connect websocket server of rdispatcher", // 161912 + WebSocketRequestInternalError: "websocket processing internal error", // 161913 + } +) + +// ErrText error text +func ErrText(code int) string { + if code > MinUserCode && code < MinSysCode { + return userErrorMsg[code] + } + return systemErrorMsg[code] +} diff --git a/functionsystem/apps/meta_service/common/snerror/snerror.go b/functionsystem/apps/meta_service/common/snerror/snerror.go new file mode 100644 index 0000000000000000000000000000000000000000..d4f456ddf2709fbc805a802da20d727582ef7148 --- /dev/null +++ b/functionsystem/apps/meta_service/common/snerror/snerror.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package snerror is basic information contained in the SN error. +package snerror + +import ( + "fmt" +) + +const ( + // UserErrorMax is maximum value of user error + UserErrorMax = 10000 +) + +// SNError defines the action contained in the SN error information. +type SNError interface { + // Code Returned error code + Code() int + + Error() string +} + +type snError struct { + code int + message string +} + +// New returns an error. +// message is a complete English sentence with punctuation. +func New(code int, message string) SNError { + return &snError{ + code: code, + message: message, + } +} + +// NewWithFmtMsg The message can contain placeholders. +func NewWithFmtMsg(code int, fmtMessage string, paras ...interface{}) SNError { + return &snError{ + code: code, + message: fmt.Sprintf(fmtMessage, paras...), + } +} + +// NewWithError err not nil. +func NewWithError(code int, err error) SNError { + var message = "" + if err != nil { + message = err.Error() + } + return &snError{ + code: code, + message: message, + } +} + +// Code Returned error code +func (s *snError) Code() int { + return s.code +} + +// Error Implement the native error interface. +func (s *snError) Error() string { + return s.message +} + +// IsUserError true if a user error occurs +func IsUserError(s SNError) bool { + // The user error is a four-digit integer. + if s.Code() < UserErrorMax { + return true + } + return false +} diff --git a/functionsystem/apps/meta_service/common/snerror/snerror_test.go b/functionsystem/apps/meta_service/common/snerror/snerror_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f23eaef89fcfd420dd952823526a0c16475dded2 --- /dev/null +++ b/functionsystem/apps/meta_service/common/snerror/snerror_test.go @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package snerror is basic information contained in the SN error. +package snerror + +import ( + "encoding/json" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestNewError +func TestNewError(t *testing.T) { + + e := New(InternalError, InternalErrorMsg) + fmt.Println(e.Code()) + assert.Equal(t, InternalError, e.Code()) + + e = New(InternalErrorRetry, InternalErrorMsg) + fmt.Println(e.Code()) + assert.Equal(t, InternalErrorRetry, e.Code()) + + e = NewWithFmtMsg(InternalError, "test %s,%d", "test", 1) + fmt.Println(e.Code()) + assert.Equal(t, InternalError, e.Code()) + + r := ConvertError(e) + assert.Equal(t, InternalError, r.Code) + + e = NewWithError(10086, errors.New("aaa")) + fmt.Println(e.Code()) + assert.Equal(t, 10086, e.Code()) + assert.Equal(t, false, IsUserError(e)) +} + +func TestNewErrorAgain(t *testing.T) { + e := New(ClientInsExceptionCode, ClientInsException) + fmt.Println(e.Code()) + assert.Equal(t, ClientInsExceptionCode, e.Code()) + + e = New(MaxRequestBodySizeErr, MaxRequestBodySizeMsg) + fmt.Println(e.Code()) + assert.Equal(t, MaxRequestBodySizeErr, e.Code()) + + e = New(UnableSpecifyResourceCode, UnableSpecifyResourceMsg) + fmt.Println(e.Code()) + assert.Equal(t, UnableSpecifyResourceCode, e.Code()) + + e = NewWithFmtMsg(ClientInsExceptionCode, "test %s,%d", "test", 1) + fmt.Println(e.Code()) + assert.Equal(t, ClientInsExceptionCode, e.Code()) + r := ConvertError(e) + assert.Equal(t, ClientInsExceptionCode, r.Code) + + e = NewWithFmtMsg(MaxRequestBodySizeErr, "test %s,%d", "test", 1) + fmt.Println(e.Code()) + assert.Equal(t, MaxRequestBodySizeErr, e.Code()) + r = ConvertError(e) + assert.Equal(t, MaxRequestBodySizeErr, r.Code) + + e = NewWithFmtMsg(UnableSpecifyResourceCode, "test %s,%d", "test", 1) + fmt.Println(e.Code()) + assert.Equal(t, UnableSpecifyResourceCode, e.Code()) + r = ConvertError(e) + assert.Equal(t, UnableSpecifyResourceCode, r.Code) + + e = NewWithError(10000, errors.New("aaa")) + fmt.Println(e.Code()) + assert.Equal(t, 10000, e.Code()) + assert.Equal(t, false, IsUserError(e)) +} + +// TestError +func TestError(t *testing.T) { + err := errors.New("aaa") + fmt.Println(err.Error()) + e := NewWithError(10086, errors.New("aaa")) + fmt.Println(e.Code()) + fmt.Println(e.Error()) + assert.Equal(t, 10086, e.Code()) + + e1 := getErr() + fmt.Println(e1.Error()) + + e2 := getErr().(SNError) + fmt.Println(e2.Error()) + fmt.Println(e2.Code()) + assert.Equal(t, 10086, e.Code()) +} + +func getErr() error { + return NewWithError(10086, errors.New("aaa")) +} + +// TestMarshal +func TestMarshal(t *testing.T) { + e := NewWithError(10086, errors.New("aaa")) + b := Marshal(e) + var re BadResponse + json.Unmarshal(b, &re) + assert.Equal(t, re.Code, 10086) +} + +func TestConvertBadResponse(t *testing.T) { + err := ConvertBadResponse([]byte("bad")) + assert.NotNil(t, err) +} \ No newline at end of file diff --git a/functionsystem/apps/meta_service/common/timer/timer.go b/functionsystem/apps/meta_service/common/timer/timer.go new file mode 100644 index 0000000000000000000000000000000000000000..3a76368724b9cd4bfc09f0bcb6b9187b69efaf16 --- /dev/null +++ b/functionsystem/apps/meta_service/common/timer/timer.go @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package timer delayed execution function +package timer + +import ( + "container/heap" + "errors" + "reflect" + "sync" + "time" + + "meta_service/common/logger/log" +) + +type timerInfo struct { + function interface{} + params []interface{} + target int64 +} + +// FuncHeap delay function information top heap +var FuncHeap *delayFuncHeap + +type delayFuncHeap []timerInfo + +var once sync.Once + +// InitTimer Creating a global timer +func InitTimer() { + once.Do(func() { + FuncHeap = &delayFuncHeap{} + heap.Init(FuncHeap) + go FuncHeap.startTimer() + }) +} + +func (d *delayFuncHeap) startTimer() { + timer := time.NewTicker(time.Second) + for { + <-timer.C + for len(*d) > 0 { + x := (*d)[0] + if x.target > time.Now().Unix() { + break + } + funcInfoIf := heap.Pop(d) + if funcInfoIf == nil { + log.GetLogger().Warn("function info is nil.") + continue + } + funcInfo, ok := funcInfoIf.(timerInfo) + if !ok { + log.GetLogger().Warn("failed to executing function ,err: type error") + continue + } + _, err := call(funcInfo.function, funcInfo.params...) + if err != nil { + log.GetLogger().Warnf("failed to executing function ,err: %s", err.Error()) + } + } + } +} + +// AddFunc add delay function +func (d *delayFuncHeap) AddFunc(delayTime int, function interface{}, params ...interface{}) { + heap.Push(d, timerInfo{ + function: function, + params: params, + target: time.Now().Unix() + int64(delayTime), + }) +} + +func call(function interface{}, params ...interface{}) ([]reflect.Value, error) { + if function == nil { + return nil, errors.New("function is nil") + } + defer func() { + if err := recover(); err != nil { + log.GetLogger().Errorf("function call failed, err: %s", err) + } + }() + functionValueOf := reflect.ValueOf(function) + if len(params) != functionValueOf.Type().NumIn() { + return nil, errors.New("the number of params is not adapted") + } + + in := make([]reflect.Value, len(params)) + for k, param := range params { + in[k] = reflect.ValueOf(param) + } + result := functionValueOf.Call(in) + return result, nil +} + +// Len returns the size +func (d delayFuncHeap) Len() int { + return len(d) +} + +// Less is used to compare two objects in the heap. +func (d delayFuncHeap) Less(i, j int) bool { + if i >= 0 && j >= 0 && i < d.Len() && j < d.Len() { + return d[i].target < d[j].target + } + log.GetLogger().Errorf("Index out of bound") + return false +} + +// Swap implements swapping of two elements in the heap. +func (d delayFuncHeap) Swap(i, j int) { + if i >= 0 && j >= 0 && i < d.Len() && j < d.Len() { + d[i], d[j] = d[j], d[i] + } else { + log.GetLogger().Errorf("Index out of bound") + } +} + +// Push push an item to heap +func (d *delayFuncHeap) Push(x interface{}) { + *d = append(*d, x.(timerInfo)) +} + +// Pop pop an item from heap +func (d *delayFuncHeap) Pop() interface{} { + old := *d + n := len(old) + x := old[n-1] + *d = old[0 : n-1] + return x +} diff --git a/functionsystem/apps/meta_service/common/timer/timer_test.go b/functionsystem/apps/meta_service/common/timer/timer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..12de6a578eeb59d4d2f23c9a439d0d4d223e1e05 --- /dev/null +++ b/functionsystem/apps/meta_service/common/timer/timer_test.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package timer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCall(t *testing.T) { + InitTimer() + testCh := make(chan string, 1) + FuncHeap.AddFunc(1, calledFunc, testCh) + res := <-testCh + assert.Equal(t, res, "test") +} + +func calledFunc(testCh chan string) { + testCh <- "test" +} + +func TestLess(t *testing.T) { + testHeap := &delayFuncHeap{} + *testHeap = append(*testHeap, timerInfo{}) + *testHeap = append(*testHeap, timerInfo{}) + testHeap.Less(0, 1) +} diff --git a/functionsystem/apps/meta_service/common/timeutil/conv.go b/functionsystem/apps/meta_service/common/timeutil/conv.go new file mode 100644 index 0000000000000000000000000000000000000000..a7c7d8c551e8203cf6e8c2e9b376ddcecfaa41bf --- /dev/null +++ b/functionsystem/apps/meta_service/common/timeutil/conv.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package timeutil provides some utils for time +package timeutil + +import "time" + +const ( + // NanosecondToMillisecond is the factor to convert nanosecond to millisecond, + // it should be equal to time.Millisecond/time.Nanosecond + NanosecondToMillisecond = 1000000 +) + +// UnixMillisecond convert time.Time to unix timestamp in millisecond +func UnixMillisecond(t time.Time) int64 { + return t.UnixNano() / NanosecondToMillisecond +} + +// NowUnixMillisecond get current unix timestamp in millisecond +func NowUnixMillisecond() int64 { + return UnixMillisecond(time.Now()) +} + +// NowUnixNanoseconds get current unix timestamp in nanoseconds +func NowUnixNanoseconds() int64 { + return time.Now().UnixNano() +} diff --git a/functionsystem/apps/meta_service/common/timeutil/conv_test.go b/functionsystem/apps/meta_service/common/timeutil/conv_test.go new file mode 100644 index 0000000000000000000000000000000000000000..968859c22a60336630d02c5fbfd8c8f330d34672 --- /dev/null +++ b/functionsystem/apps/meta_service/common/timeutil/conv_test.go @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package timeutil + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestUnixMillisecond(t *testing.T) { + assert.Equal(t, time.Second.Nanoseconds()/time.Second.Milliseconds(), int64(NanosecondToMillisecond)) + assert.Equal(t, int64(time.Millisecond/time.Nanosecond), int64(NanosecondToMillisecond)) +} + +func TestTimeutil(t *testing.T) { + UnixMillisecond(time.Now()) + NowUnixMillisecond() + NowUnixNanoseconds() +} diff --git a/functionsystem/apps/meta_service/common/tls/https.go b/functionsystem/apps/meta_service/common/tls/https.go new file mode 100644 index 0000000000000000000000000000000000000000..155fb9fcd1afa5fcda039bde5cce8e74f9da7024 --- /dev/null +++ b/functionsystem/apps/meta_service/common/tls/https.go @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tls + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + + "meta_service/common/logger/log" + "meta_service/common/reader" + "meta_service/common/utils" +) + +// HTTPSConfig is for needed HTTPS config +type HTTPSConfig struct { + CipherSuite []uint16 + MinVers uint16 + MaxVers uint16 + CACertFile string + CertFile string + SecretKeyFile string + PwdFilePath string + KeyPassPhase string +} + +// InternalHTTPSConfig is for input config +type InternalHTTPSConfig struct { + HTTPSEnable bool `json:"httpsEnable" yaml:"httpsEnable" valid:"optional"` + TLSProtocol string `json:"tlsProtocol" yaml:"tlsProtocol" valid:"optional"` + TLSCiphers string `json:"tlsCiphers" yaml:"tlsCiphers" valid:"optional"` +} + +var ( + // HTTPSConfigs is a global variable of HTTPS config + HTTPSConfigs = &HTTPSConfig{} + // tlsConfig is a global variable of TLS config + tlsConfig *tls.Config + once sync.Once + + // tlsVersionMap is a set of TLS versions + tlsVersionMap = map[string]uint16{ + "TLSv1.2": tls.VersionTLS12, + } +) + +// tlsCipherSuiteMap is a set of supported TLS algorithms +var tlsCipherSuiteMap = map[string]uint16{ + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +} + +// GetURLScheme returns "http" or "https" +func GetURLScheme(https bool) string { + if https { + return "https" + } + return "http" +} + +// HTTPListenAndServeTLS listens and serves by TLS in HTTP +func HTTPListenAndServeTLS(addr string, server *http.Server) error { + listener, err := net.Listen("tcp4", addr) + if err != nil { + return err + } + + tlsListener := tls.NewListener(listener, tlsConfig) + + if err = server.Serve(tlsListener); err != nil { + return err + } + return nil +} + +// GetClientTLSConfig returns the config of TLS +func GetClientTLSConfig() *tls.Config { + return tlsConfig +} + +// GetHTTPTransport get http transport +func GetHTTPTransport() *http.Transport { + tr, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil + } + tr.TLSClientConfig = GetClientTLSConfig() + + return tr +} + +func loadCerts(path string) string { + env := os.Getenv("SSL_ROOT") + if len(env) == 0 { + log.GetLogger().Errorf("failed to get SSL_ROOT") + return "" + } + certPath, err := filepath.Abs(filepath.Join(env, path)) + if err != nil { + log.GetLogger().Errorf("failed to return an absolute representation of path: %s", path) + return "" + } + ok := utils.FileExists(certPath) + if !ok { + log.GetLogger().Errorf("failed to load the cert file: %s", certPath) + return "" + } + return certPath +} + +func loadTLSConfig() (err error) { + clientAuthMode := tls.NoClientCert + var pool *x509.CertPool + + pool, err = GetX509CACertPool(HTTPSConfigs.CACertFile) + if err != nil { + log.GetLogger().Errorf("failed to GetX509CACertPool: %s", err.Error()) + return err + } + + var certs []tls.Certificate + certs, err = loadServerTLSCertificate() + if err != nil { + log.GetLogger().Errorf("failed to loadServerTLSCertificate: %s", err.Error()) + return err + } + + tlsConfig = &tls.Config{ + ClientCAs: pool, + Certificates: certs, + CipherSuites: HTTPSConfigs.CipherSuite, + PreferServerCipherSuites: true, + ClientAuth: clientAuthMode, + InsecureSkipVerify: true, + MinVersion: HTTPSConfigs.MinVers, + MaxVersion: HTTPSConfigs.MaxVers, + Renegotiation: tls.RenegotiateNever, + } + + return nil +} + +// loadHTTPSConfig loads the protocol and ciphers of TLS +func loadHTTPSConfig(tlsProtocols string, tlsCiphers []byte) error { + HTTPSConfigs = &HTTPSConfig{ + MinVers: tls.VersionTLS12, + MaxVers: tls.VersionTLS12, + CipherSuite: nil, + CACertFile: loadCerts("trust.cer"), + CertFile: loadCerts("server.cer"), + SecretKeyFile: loadCerts("server_key.pem"), + PwdFilePath: loadCerts("cert_pwd"), + KeyPassPhase: "", + } + + minVersion := parseSSLProtocol(tlsProtocols) + if HTTPSConfigs.MinVers == 0 { + return errors.New("invalid TLS protocol") + } + HTTPSConfigs.MinVers = minVersion + cipherSuites := parseSSLCipherSuites(tlsCiphers) + if len(cipherSuites) == 0 { + return errors.New("invalid TLS ciphers") + } + HTTPSConfigs.CipherSuite = cipherSuites + + keyPassPhase, err := reader.ReadFileWithTimeout(HTTPSConfigs.PwdFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read file cert_pwd: %s", err.Error()) + return err + } + HTTPSConfigs.KeyPassPhase = string(keyPassPhase) + + return nil +} + +// InitTLSConfig inits config of HTTPS +func InitTLSConfig(tlsProtocols string, tlsCiphers []byte) (err error) { + once.Do(func() { + err = loadHTTPSConfig(tlsProtocols, tlsCiphers) + if err != nil { + err = errors.New("failed to load HTTPS config") + return + } + err = loadTLSConfig() + if err != nil { + return + } + }) + return err +} + +// GetX509CACertPool get ca cert pool +func GetX509CACertPool(caCertFilePath string) (caCertPool *x509.CertPool, err error) { + pool := x509.NewCertPool() + caCertContent, err := LoadCACertBytes(caCertFilePath) + if err != nil { + return nil, err + } + + pool.AppendCertsFromPEM(caCertContent) + return pool, nil +} + +func loadServerTLSCertificate() (tlsCert []tls.Certificate, err error) { + certContent, keyContent, err := LoadCertAndKeyBytes(HTTPSConfigs.CertFile, HTTPSConfigs.SecretKeyFile, + HTTPSConfigs.KeyPassPhase) + if err != nil { + return nil, err + } + + cert, err := tls.X509KeyPair(certContent, keyContent) + if err != nil { + log.GetLogger().Errorf("failed to load the X509 key pair from cert file %s with key file %s: %s", + HTTPSConfigs.CertFile, HTTPSConfigs.SecretKeyFile, err.Error()) + return nil, err + } + + var certs []tls.Certificate + certs = append(certs, cert) + + return certs, nil +} + +// LoadServerTLSCertificate generates tls certificate by certfile and keyfile +func LoadServerTLSCertificate(cerfile, keyfile string) (tlsCert []tls.Certificate, err error) { + certContent, keyContent, err := LoadCertAndKeyBytes(cerfile, keyfile, "") + if err != nil { + return nil, err + } + cert, err := tls.X509KeyPair(certContent, keyContent) + if err != nil { + log.GetLogger().Errorf("failed to load the X509 key pair from cert file %s with key file %s: %s", + cerfile, keyfile, err.Error()) + return nil, err + } + var certs []tls.Certificate + certs = append(certs, cert) + return certs, nil +} + +// LoadCertAndKeyBytes load cert and key bytes +func LoadCertAndKeyBytes(certFilePath, keyFilePath, passPhase string) (certPEMBlock, keyPEMBlock []byte, err error) { + certContent, err := reader.ReadFileWithTimeout(certFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read cert file %s", err.Error()) + return nil, nil, err + } + + keyContent, err := reader.ReadFileWithTimeout(keyFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read key file %s", err.Error()) + return nil, nil, err + } + return certContent, keyContent, nil +} + +func clearByteMemory(src []byte) { + for idx := 0; idx < len(src)&32; idx++ { + src[idx] = 0 + } +} + +// LoadCACertBytes Load CA Cert Content +func LoadCACertBytes(caCertFilePath string) ([]byte, error) { + caCertContent, err := reader.ReadFileWithTimeout(caCertFilePath) + if err != nil { + log.GetLogger().Errorf("failed to read ca cert file %s: %s", caCertFilePath, err.Error()) + return nil, err + } + + return caCertContent, nil +} + +func parseSSLProtocol(rawProtocol string) uint16 { + if protocol, ok := tlsVersionMap[rawProtocol]; ok { + return protocol + } + log.GetLogger().Errorf("invalid SSL version %s, use the default protocol version", rawProtocol) + return 0 +} + +func parseSSLCipherSuites(ciphers []byte) []uint16 { + cipherSuiteNameList := strings.Split(string(ciphers), ",") + if len(cipherSuiteNameList) == 0 { + log.GetLogger().Errorf("no input cipher suite") + return nil + } + cipherSuiteList := make([]uint16, 0, len(cipherSuiteNameList)) + for _, cipherSuiteItem := range cipherSuiteNameList { + cipherSuiteItem = strings.TrimSpace(cipherSuiteItem) + if len(cipherSuiteItem) == 0 { + continue + } + + if cipherSuite, ok := tlsCipherSuiteMap[cipherSuiteItem]; ok { + cipherSuiteList = append(cipherSuiteList, cipherSuite) + } else { + log.GetLogger().Errorf("cipher %s does not exist", cipherSuiteItem) + } + } + + return cipherSuiteList +} diff --git a/functionsystem/apps/meta_service/common/tls/https_test.go b/functionsystem/apps/meta_service/common/tls/https_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8cc69d1a5dc76f2e628cd2ea9bc55a71556deb44 --- /dev/null +++ b/functionsystem/apps/meta_service/common/tls/https_test.go @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tls + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "io/ioutil" + "net" + "net/http" + "os" + "testing" + + "meta_service/common/crypto" + "meta_service/common/localauth" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" +) + +func TestGetURLScheme(t *testing.T) { + if "https" != GetURLScheme(true) { + t.Error("GetURLScheme failed") + } + if "http" != GetURLScheme(false) { + t.Error("GetURLScheme failed") + } +} + +func TestInitTLSConfig1(t *testing.T) { + os.Setenv("SSL_ROOT", "/home/sn/resource/https") + tlsCiphers := []byte("TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_TEST") + err := InitTLSConfig("TLSv1.2", tlsCiphers) + assert.NotEqual(t, nil, err) + + patch := gomonkey.ApplyFunc(loadHTTPSConfig, func(string, []byte) error { + return nil + }) + InitTLSConfig("TLSv1.2", tlsCiphers) + patch.Reset() +} + +func TestInitTLSConfig2(t *testing.T) { + os.Setenv("SSL_ROOT", "/home/sn/resource/https") + tlsCiphers := []byte("TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_TEST") + patch := gomonkey.ApplyFunc(loadHTTPSConfig, func(string, []byte) error { + return nil + }) + InitTLSConfig("TLSv1.2", tlsCiphers) + patch.Reset() +} + +func Test_loadServerTLSCertificate(t *testing.T) { + HTTPSConfigs.CertFile = "/home/snuser" + _, err := loadServerTLSCertificate() + assert.NotNil(t, err) + + convey.Convey("test loadServerTLSCertificate", t, func() { + convey.Convey("LoadCertAndKeyBytes success", func() { + patch1 := gomonkey.ApplyFunc(LoadCertAndKeyBytes, func(certFilePath, keyFilePath, passPhase string) (certPEMBlock, + keyPEMBlock []byte, err error, + ) { + return nil, nil, nil + }) + convey.Convey("X509KeyPair success", func() { + patch2 := gomonkey.ApplyFunc(tls.X509KeyPair, func(certPEMBlock, keyPEMBlock []byte) (tls.Certificate, error) { + return tls.Certificate{}, nil + }) + _, err := loadServerTLSCertificate() + convey.So(err, convey.ShouldBeNil) + _, err = LoadServerTLSCertificate("", "") + convey.So(err, convey.ShouldBeNil) + defer patch2.Reset() + }) + convey.Convey("X509KeyPair fail", func() { + patch3 := gomonkey.ApplyFunc(tls.X509KeyPair, func(certPEMBlock, keyPEMBlock []byte) (tls.Certificate, error) { + return tls.Certificate{}, errors.New("fail to load X509KeyPair") + }) + _, err := loadServerTLSCertificate() + convey.So(err, convey.ShouldNotBeNil) + _, err = LoadServerTLSCertificate("", "") + convey.So(err, convey.ShouldNotBeNil) + defer patch3.Reset() + }) + defer patch1.Reset() + }) + convey.Convey("LoadCertAndKeyBytes fail", func() { + patch4 := gomonkey.ApplyFunc(LoadCertAndKeyBytes, func(certFilePath, keyFilePath, passPhase string) (certPEMBlock, + keyPEMBlock []byte, err error, + ) { + return nil, nil, errors.New("fail to LoadCertAndKeyBytes") + }) + _, err := loadServerTLSCertificate() + convey.So(err, convey.ShouldNotBeNil) + _, err = LoadServerTLSCertificate("", "") + convey.So(err, convey.ShouldNotBeNil) + defer patch4.Reset() + }) + }) +} + +func TestHTTPListenAndServeTLS(t *testing.T) { + server := &http.Server{} + err := HTTPListenAndServeTLS("127.0.0.1", server) + assert.NotNil(t, err) +} + +func Test_loadTLSConfig(t *testing.T) { + HTTPSConfigs = &HTTPSConfig{} + err := loadTLSConfig() + assert.NotNil(t, err) + + convey.Convey("test loadTLSConfig", t, func() { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(GetX509CACertPool, func(caCertFilePath string) (caCertPool *x509.CertPool, err error) { + return x509.NewCertPool(), nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + err = loadTLSConfig() + assert.NotNil(t, err) + }) +} + +func Test_LoadCertAndKeyBytes(t *testing.T) { + convey.Convey("test LoadCertAndKeyBytes", t, func() { + convey.Convey("ReadFile fail", func() { + _, _, err := LoadCertAndKeyBytes("certPath", "keyPath", "pass") + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("ReadFile success", func() { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { + return nil, nil + }), + gomonkey.ApplyFunc(crypto.GetRootKey, func() []byte { + return nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + + convey.Convey("DecryptByte fail", func() { + patch := gomonkey.ApplyFunc(crypto.DecryptByte, func(cipherText []byte, secret []byte) ([]byte, error) { + return []byte{}, errors.New("DecryptByte fail") + }) + defer patch.Reset() + + _, _, err := LoadCertAndKeyBytes("certPath", "keyPath", "pass") + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("DecryptByte success", func() { + patch := gomonkey.ApplyFunc(crypto.DecryptByte, func(cipherText []byte, secret []byte) ([]byte, error) { + return []byte{}, nil + }) + defer patch.Reset() + + convey.Convey("Decode fail", func() { + patch := gomonkey.ApplyFunc(pem.Decode, func(data []byte) (p *pem.Block, rest []byte) { + return nil, nil + }) + defer patch.Reset() + + _, _, err := LoadCertAndKeyBytes("certPath", "keyPath", "pass") + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("Decode success", func() { + patch := gomonkey.ApplyFunc(pem.Decode, func(data []byte) (p *pem.Block, rest []byte) { + return &pem.Block{}, nil + }) + defer patch.Reset() + + convey.Convey("crypto.IsEncryptedPEMBlock fail", func() { + patch := gomonkey.ApplyFunc(crypto.IsEncryptedPEMBlock, func(b *pem.Block) bool { + return false + }) + defer patch.Reset() + + _, _, err := LoadCertAndKeyBytes("certPath", "keyPath", "pass") + convey.So(err, convey.ShouldBeNil) + }) + convey.Convey("crypto.IsEncryptedPEMBlock success", func() { + patch := gomonkey.ApplyFunc(crypto.IsEncryptedPEMBlock, func(b *pem.Block) bool { + return true + }) + defer patch.Reset() + + convey.Convey("localauth.Decrypt fail", func() { + patch := gomonkey.ApplyFunc(localauth.Decrypt, func(src string) ([]byte, error) { + return nil, errors.New("localauth.Decrypt fail") + }) + defer patch.Reset() + + _, _, err := LoadCertAndKeyBytes("certPath", "keyPath", "pass") + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("localauth.Decrypt success", func() { + patch := gomonkey.ApplyFunc(localauth.Decrypt, func(src string) ([]byte, error) { + return []byte{}, nil + }) + defer patch.Reset() + + convey.Convey("crypto.DecryptPEMBlock fail", func() { + patch := gomonkey.ApplyFunc(crypto.DecryptPEMBlock, + func(b *pem.Block, password []byte) ([]byte, error) { + return nil, errors.New("crypto.DecryptPEMBlock fail") + }) + defer patch.Reset() + + _, _, err := LoadCertAndKeyBytes("certPath", "keyPath", "pass") + convey.So(err, convey.ShouldNotBeNil) + }) + }) + }) + }) + }) + }) + }) +} + +func Test_loadCerts(t *testing.T) { + convey.Convey("env length 0", t, func() { + patch := gomonkey.ApplyFunc(os.Getenv, func(key string) string { + return "" + }) + defer patch.Reset() + result := loadCerts("") + convey.So(result, convey.ShouldEqual, "") + }) +} + +func Test_GetClientTLSConfig(t *testing.T) { + convey.Convey("test GetClientTLSConfig", t, func() { + convey.So(GetClientTLSConfig(), convey.ShouldBeNil) + }) +} + +func Test_GetHTTPTransport(t *testing.T) { + convey.Convey("test GetHTTPTransport", t, func() { + convey.So(GetHTTPTransport(), convey.ShouldNotBeNil) + }) +} + +func Test_BuildClientTLSConfOpts(t *testing.T) { + BuildClientTLSConfOpts(MutualTLSConfig{}) +} + +func Test_BuildServerTLSConfOpts(t *testing.T) { + BuildServerTLSConfOpts(MutualTLSConfig{}) +} + +func Test_ClearByteMemory(t *testing.T) { + convey.Convey("test clearByteMemory", t, func() { + s := make([]byte, 33) + s = append(s, 'A') + clearByteMemory() + convey.So(s[0], convey.ShouldEqual, 0) + }) +} + +func Test_parseSSLProtocol(t *testing.T) { + parseSSLProtocol("") +} + +func Test_HTTPListenAndServeTLS(t *testing.T) { + patch := gomonkey.ApplyFunc(net.Listen, func(string, string) (net.Listener, error) { + return nil, nil + }) + defer patch.Reset() + + patch1 := gomonkey.ApplyFunc((*http.Server).Serve, func(*http.Server, net.Listener) error { + return nil + }) + HTTPListenAndServeTLS("", &http.Server{}) + patch1.Reset() + + patch2 := gomonkey.ApplyFunc((*http.Server).Serve, func(*http.Server, net.Listener) error { + return errors.New("test") + }) + HTTPListenAndServeTLS("", &http.Server{}) + patch2.Reset() +} + +func Test_GetX509CACertPool(t *testing.T) { + patch := gomonkey.ApplyFunc(LoadCACertBytes, func(string) ([]byte, error) { + return []byte{'a'}, nil + }) + GetX509CACertPool("") + patch.Reset() +} diff --git a/functionsystem/apps/meta_service/common/tls/option.go b/functionsystem/apps/meta_service/common/tls/option.go new file mode 100644 index 0000000000000000000000000000000000000000..e9051d5efe047390c5f4eccb43fe362c547cd73f --- /dev/null +++ b/functionsystem/apps/meta_service/common/tls/option.go @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tls + +import ( + "crypto/tls" + "crypto/x509" + + "meta_service/common/logger/log" + "meta_service/common/reader" + "meta_service/common/utils" +) + +// NewTLSConfig returns tls.Config with given options +func NewTLSConfig(opts ...Option) *tls.Config { + config := &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + // for TLS1.2 + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + // for TLS1.3 + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + }, + PreferServerCipherSuites: true, + Renegotiation: tls.RenegotiateNever, + } + for _, opt := range opts { + opt.apply(config) + } + return config +} + +// Option is optional argument for tls.Config +type Option interface { + apply(*tls.Config) +} + +type rootCAOption struct { + cas *x509.CertPool +} + +func (r *rootCAOption) apply(config *tls.Config) { + config.RootCAs = r.cas +} + +// WithRootCAs returns Option that applies root CAs to tls.Config +func WithRootCAs(caFiles ...string) Option { + rootCAs, err := LoadRootCAs(caFiles...) + if err != nil { + log.GetLogger().Warnf("failed to load root ca, err: %s", err.Error()) + rootCAs = nil + } + return &rootCAOption{ + cas: rootCAs, + } +} + +type certsOption struct { + certs []tls.Certificate +} + +func (c *certsOption) apply(config *tls.Config) { + config.Certificates = c.certs +} + +// WithCerts returns Option that applies cert file and key file to tls.Config +func WithCerts(certFile, keyFile string) Option { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + log.GetLogger().Warnf("load cert.pem and key.pem error: %s", err) + cert = tls.Certificate{} + } + return &certsOption{ + certs: []tls.Certificate{cert}, + } +} + +// WithCertsByEncryptedKey returns Option that applies cert file and encrypted key file to tls.Config +func WithCertsByEncryptedKey(certFile, keyFile, passPhase string) Option { + cert := tls.Certificate{} + certPEM, err := reader.ReadFileWithTimeout(certFile) + if err != nil { + return &certsOption{certs: []tls.Certificate{cert}} + } + keyPEMBlock, err := getKeyContent(keyFile) + if err != nil { + return &certsOption{certs: []tls.Certificate{cert}} + } + + cert, err = tls.X509KeyPair(certPEM, keyPEMBlock) + if err != nil { + cert = tls.Certificate{} + } + utils.ClearByteMemory(keyPEMBlock) + return &certsOption{certs: []tls.Certificate{cert}} +} + +func getKeyContent(keyFile string) ([]byte, error) { + var err error + keyPEMBlock, err := reader.ReadFileWithTimeout(keyFile) + if err != nil { + log.GetLogger().Errorf("fialed to SCCDecrypt, err: %s", err.Error()) + return nil, err + } + return keyPEMBlock, nil +} + +// WithCertsContent returns Option that applies cert content and key content to tls.Config +func WithCertsContent(certContent, keyContent []byte) Option { + cert, err := tls.X509KeyPair(certContent, keyContent) + utils.ClearByteMemory(keyContent) + if err != nil { + log.GetLogger().Warnf("load cert.pem and key.pem error: %s", err) + cert = tls.Certificate{} + } + return &certsOption{ + certs: []tls.Certificate{cert}, + } +} + +type skipVerifyOption struct{} + +func (s *skipVerifyOption) apply(config *tls.Config) { + config.InsecureSkipVerify = true +} + +// WithSkipVerify returns Option that skips to verify certificates +func WithSkipVerify() Option { + return &skipVerifyOption{} +} + +type clientAuthOption struct { + clientAuthType tls.ClientAuthType +} + +func (a *clientAuthOption) apply(config *tls.Config) { + config.ClientAuth = a.clientAuthType +} + +// WithClientAuthType returns Option with client auth strategy +func WithClientAuthType(t tls.ClientAuthType) Option { + return &clientAuthOption{ + clientAuthType: t, + } +} + +type clientCAOption struct { + clientCAs *x509.CertPool +} + +func (r *clientCAOption) apply(config *tls.Config) { + config.ClientCAs = r.clientCAs +} + +// WithClientCAs returns Option that applies client CAs to tls.Config +func WithClientCAs(caFiles ...string) Option { + clientCAs, err := LoadRootCAs(caFiles...) + if err != nil { + log.GetLogger().Warnf("failed to load client ca, err: %s", err.Error()) + clientCAs = nil + } + return &clientCAOption{ + clientCAs: clientCAs, + } +} + +type serverNameOption struct { + serverName string +} + +func (sn *serverNameOption) apply(config *tls.Config) { + config.ServerName = sn.serverName +} + +// WithServerName returns Option that applies server name to tls.Config +func WithServerName(name string) Option { + return &serverNameOption{ + serverName: name, + } +} diff --git a/functionsystem/apps/meta_service/common/tls/option_test.go b/functionsystem/apps/meta_service/common/tls/option_test.go new file mode 100644 index 0000000000000000000000000000000000000000..63d4529bfea30b1f5494516dfc358a0632d887d8 --- /dev/null +++ b/functionsystem/apps/meta_service/common/tls/option_test.go @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package tls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite + server http.Server + rootKEY string + rootPEM string + rootSRL string + serverKEY string + serverPEM string + serverCSR string +} + +func (s *TestSuite) SetupSuite() { + certificatePath, err := os.Getwd() + if err != nil { + s.T().Errorf("failed to get current working dictionary: %s", err.Error()) + return + } + + certificatePath += "/../../../test/" + s.rootKEY = certificatePath + "ca.key" + s.rootPEM = certificatePath + "ca.crt" + s.rootSRL = certificatePath + "ca.srl" + s.serverKEY = certificatePath + "server.key" + s.serverPEM = certificatePath + "server.crt" + s.serverCSR = certificatePath + "server.csr" + + body := "Hello" + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, body) + }) + + s.server = http.Server{ + Addr: "127.0.0.1:6061", + Handler: handler, + } + + go func() { + if err := s.server.ListenAndServeTLS(s.serverPEM, s.serverKEY); err != nil { + s.T().Logf("failed to start server: %s", err.Error()) + } + }() +} + +func (s *TestSuite) TearDownSuite() { + s.server.Shutdown(context.Background()) + + os.Remove(s.serverKEY) + os.Remove(s.serverPEM) + os.Remove(s.serverCSR) + os.Remove(s.rootKEY) + os.Remove(s.rootPEM) + os.Remove(s.rootSRL) +} + +// This is test for no verify client +func (s *TestSuite) TestNewTLSConfig() { + // no verify client + _, err := http.Get("https://127.0.0.1:6061") + assert.NotNil(s.T(), err) + // client skip server certificate verify + tr := &http.Transport{ + TLSClientConfig: NewTLSConfig(WithSkipVerify()), + } + client := &http.Client{Transport: tr} + resp, err := client.Get("https://127.0.0.1:6061") + assert.Nil(s.T(), err) + defer resp.Body.Close() + res, err := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), string(res), "Hello") +} + +// This is test for verify client +func (s *TestSuite) TestNewTLSConfig2() { + tr := &http.Transport{ + TLSClientConfig: NewTLSConfig(WithRootCAs(s.rootPEM), + WithCertsByEncryptedKey(s.serverPEM, s.serverKEY, ""), WithSkipVerify()), + } + client := &http.Client{Transport: tr} + resp, _ := client.Get("https://127.0.0.1:6061") + defer resp.Body.Close() + res, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), string(res), "Hello") +} + +func TestOptionTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func TestWithFunc(t *testing.T) { + WithCertsContent(nil, nil) + WithCerts("", "") + WithClientCAs() + patch := gomonkey.ApplyFunc(LoadRootCAs, func(caFiles ...string) (*x509.CertPool, error) { + return nil, errors.New("LoadRootCAs failed") + }) + WithClientCAs("") + patch.Reset() +} + +func TestVerifyCert(t *testing.T) { + var raw [][]byte + tlsConfig = &tls.Config{} + tlsConfig.ClientCAs = x509.NewCertPool() + err := VerifyCert(raw, nil) + assert.NotNil(t, err) + + raw = [][]byte{ + []byte("0"), + []byte("1"), + } + err = VerifyCert(raw, nil) + assert.NotNil(t, err) + + patch1 := gomonkey.ApplyFunc(x509.ParseCertificate, func([]byte) (*x509.Certificate, error) { + return &x509.Certificate{}, nil + }) + VerifyCert(raw, nil) + patch1.Reset() +} + +func TestApply(t *testing.T) { + cli := clientCAOption{} + cli.apply(&tls.Config{}) +} diff --git a/functionsystem/apps/meta_service/common/tls/tls.go b/functionsystem/apps/meta_service/common/tls/tls.go new file mode 100644 index 0000000000000000000000000000000000000000..e0b62ea45ffbeea7f5653d4952f67b1c39f96104 --- /dev/null +++ b/functionsystem/apps/meta_service/common/tls/tls.go @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package tls provides tls utils +package tls + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "time" + + "meta_service/common/logger/log" + "meta_service/common/reader" +) + +// MutualTLSConfig indicates tls config +type MutualTLSConfig struct { + TLSEnable bool `json:"tlsEnable" yaml:"tlsEnable" valid:"optional"` + RootCAFile string `json:"rootCAFile" yaml:"rootCAFile" valid:"optional"` + ModuleCertFile string `json:"moduleCertFile" yaml:"moduleCertFile" valid:"optional"` + ModuleKeyFile string `json:"moduleKeyFile" yaml:"moduleKeyFile" valid:"optional"` + ServerName string `json:"serverName" yaml:"serverName" valid:"optional"` + SecretName string `json:"secretName" yaml:"secretName" valid:"optional"` + PwdFile string `json:"pwdFile" yaml:"pwdFile" valid:"optional"` + DecryptTool string `json:"sslDecryptTool" yaml:"sslDecryptTool" valid:"optional"` +} + +// MutualSSLConfig indicates ssl config +type MutualSSLConfig struct { + SSLEnable bool `json:"sslEnable" yaml:"sslEnable" valid:"optional"` + RootCAFile string `json:"rootCAFile" yaml:"rootCAFile" valid:"optional"` + ModuleCertFile string `json:"moduleCertFile" yaml:"moduleCertFile" valid:"optional"` + ModuleKeyFile string `json:"moduleKeyFile" yaml:"moduleKeyFile" valid:"optional"` + ServerName string `json:"serverName" yaml:"serverName" valid:"optional"` + PwdFile string `json:"pwdFile" yaml:"pwdFile" valid:"optional"` + DecryptTool string `json:"sslDecryptTool" yaml:"sslDecryptTool" valid:"optional"` +} + +// BuildClientTLSConfOpts is to build an option array for mostly used client tlsConf +func BuildClientTLSConfOpts(mutualConf MutualTLSConfig) []Option { + var opts []Option + passPhase, err := reader.ReadFileWithTimeout(mutualConf.PwdFile) + if err != nil { + log.GetLogger().Errorf("failed to read file PwdFile: %s", err.Error()) + return opts + } + opts = append(opts, WithRootCAs(mutualConf.RootCAFile), + WithCertsByEncryptedKey(mutualConf.ModuleCertFile, mutualConf.ModuleKeyFile, + string(passPhase)), + WithServerName(mutualConf.ServerName)) + return opts +} + +// BuildServerTLSConfOpts is to build an option array for mostly used server tlsConf +func BuildServerTLSConfOpts(mutualConf MutualTLSConfig) []Option { + var opts []Option + var passPhase []byte + var err error + if mutualConf.PwdFile != "" { + passPhase, err = reader.ReadFileWithTimeout(mutualConf.PwdFile) + if err != nil { + log.GetLogger().Errorf("failed to read file PwdFile: %s", err.Error()) + return opts + } + } + opts = append(opts, WithRootCAs(mutualConf.RootCAFile), + WithCertsByEncryptedKey(mutualConf.ModuleCertFile, mutualConf.ModuleKeyFile, + string(passPhase)), + WithClientCAs(mutualConf.RootCAFile), + WithClientAuthType(tls.RequireAndVerifyClientCert)) + return opts +} + +// LoadRootCAs returns system cert pool with caFiles added +func LoadRootCAs(caFiles ...string) (*x509.CertPool, error) { + rootCAs := x509.NewCertPool() + for _, file := range caFiles { + cert, err := reader.ReadFileWithTimeout(file) + if err != nil { + return nil, err + } + if !rootCAs.AppendCertsFromPEM(cert) { + return nil, err + } + } + return rootCAs, nil +} + +// VerifyCert Used to verity the server certificate +func VerifyCert(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + certs := make([]*x509.Certificate, len(rawCerts)) + if len(certs) == 0 { + log.GetLogger().Errorf("cert number is 0") + return errors.New("cert number is 0") + } + opts := x509.VerifyOptions{ + Roots: tlsConfig.ClientCAs, + CurrentTime: time.Now(), + DNSName: "", + Intermediates: x509.NewCertPool(), + } + for i, asn1Data := range rawCerts { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + log.GetLogger().Errorf("failed to parse certificate from server: %s", err.Error()) + return err + } + certs[i] = cert + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + _, err := certs[0].Verify(opts) + return err +} diff --git a/functionsystem/apps/meta_service/common/trace/trace.go b/functionsystem/apps/meta_service/common/trace/trace.go new file mode 100644 index 0000000000000000000000000000000000000000..69fd9c6a6f9b3e16ce73511f82acc00a4fad8276 --- /dev/null +++ b/functionsystem/apps/meta_service/common/trace/trace.go @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package trace for tracing system +package trace + +// Service - +type Service struct { + TraceAK string `json:"tracing_ak" valid:",optional"` + TraceSK string `json:"tracing_sk" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` +} diff --git a/functionsystem/apps/meta_service/common/types/functiontypes/functiontypes.go b/functionsystem/apps/meta_service/common/types/functiontypes/functiontypes.go new file mode 100644 index 0000000000000000000000000000000000000000..5ca889bb83d7933fd4d0abe6d5df34a56461679a --- /dev/null +++ b/functionsystem/apps/meta_service/common/types/functiontypes/functiontypes.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package functiontypes contains function info stored on etcd +package functiontypes + +// ResourceMetaData include resource data such as cpu and memory +type ResourceMetaData struct { + CPU int `json:"cpu"` + Memory int `json:"memory"` + CustomResources string `json:"customResources"` +} + +// LayerBucket - +type LayerBucket struct { + BucketURL string `json:"bucketUrl"` + ObjectID string `json:"objectId"` + BucketID string `json:"bucketId"` + AppID string `json:"appId"` + Sha256 string `json:"sha256"` +} + +// ExtendedMetaData - +type ExtendedMetaData struct { + UserType string `json:"user_type" valid:"optional"` + InstanceMetaData map[string]int `json:"instance_meta_data" valid:"optional"` + ExtendedHandler map[string]string `json:"extended_handler" valid:",optional"` + ExtendedTimeout map[string]int `json:"extended_timeout" valid:",optional"` +} diff --git a/functionsystem/apps/meta_service/common/types/serve.go b/functionsystem/apps/meta_service/common/types/serve.go new file mode 100644 index 0000000000000000000000000000000000000000..2d17724caf9bbb3ab08acb9577c6a7263700ef73 --- /dev/null +++ b/functionsystem/apps/meta_service/common/types/serve.go @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package types + +import ( + "fmt" + "regexp" + + "meta_service/common/constants" +) + +const ( + defaultServeAppRuntime = "python3.9" + defaultServeAppTimeout = 900 + defaultServeAppCpu = 1000 + defaultServeAppMemory = 1024 + defaultServeAppConcurrentNum = 1000 +) + +// ServeDeploySchema - +type ServeDeploySchema struct { + Applications []ServeApplicationSchema `json:"applications"` +} + +// ServeApplicationSchema - +type ServeApplicationSchema struct { + Name string `json:"name"` + RoutePrefix string `json:"route_prefix"` + ImportPath string `json:"import_path"` + RuntimeEnv ServeRuntimeEnvSchema `json:"runtime_env"` + Deployments []ServeDeploymentSchema `json:"deployments"` +} + +// ServeDeploymentSchema - +type ServeDeploymentSchema struct { + Name string `json:"name"` + NumReplicas int64 `json:"num_replicas"` + HealthCheckPeriodS int64 `json:"health_check_period_s"` + HealthCheckTimeoutS int64 `json:"health_check_timeout_s"` +} + +// ServeRuntimeEnvSchema - +type ServeRuntimeEnvSchema struct { + Pip []string `json:"pip"` + WorkingDir string `json:"working_dir"` + EnvVars map[string]any `json:"env_vars"` +} + +// ServeFuncWithKeysAndFunctionMetaInfo - +type ServeFuncWithKeysAndFunctionMetaInfo struct { + FuncMetaKey string + InstanceMetaKey string + FuncMetaInfo *FunctionMetaInfo +} + +// Validate serve deploy schema by set of rules +func (s *ServeDeploySchema) Validate() error { + // 1. app name unique + appNameSet := make(map[string]struct{}) + for _, app := range s.Applications { + if _, ok := appNameSet[app.Name]; ok { + return fmt.Errorf("duplicated application name: %s", app.Name) + } + appNameSet[app.Name] = struct{}{} + } + // 2. app routes unique + appRouteSet := make(map[string]struct{}) + for _, app := range s.Applications { + if _, ok := appRouteSet[app.RoutePrefix]; ok { + return fmt.Errorf("duplicated application route prefix: %s", app.RoutePrefix) + } + appRouteSet[app.RoutePrefix] = struct{}{} + } + // 3. app name non empty + for _, app := range s.Applications { + if app.Name == "" { + return fmt.Errorf("application names must be nonempty") + } + } + return nil +} + +// ToFaaSFuncMetas - +func (s *ServeDeploySchema) ToFaaSFuncMetas() []*ServeFuncWithKeysAndFunctionMetaInfo { + var allMetas []*ServeFuncWithKeysAndFunctionMetaInfo + for _, a := range s.Applications { + // we don't really check it there are some repeated part? and just assume translate won't fail + for _, deploymentFuncMeta := range a.ToFaaSFuncMetas() { + allMetas = append(allMetas, deploymentFuncMeta) + } + } + return allMetas +} + +// ToFaaSFuncMetas - +func (s *ServeApplicationSchema) ToFaaSFuncMetas() []*ServeFuncWithKeysAndFunctionMetaInfo { + var allMetas []*ServeFuncWithKeysAndFunctionMetaInfo + for _, d := range s.Deployments { + meta := d.ToFaaSFuncMeta(s) + allMetas = append(allMetas, meta) + } + return allMetas +} + +// ToFaaSFuncMeta - +func (s *ServeDeploymentSchema) ToFaaSFuncMeta( + belongedApp *ServeApplicationSchema, +) *ServeFuncWithKeysAndFunctionMetaInfo { + faasFuncUrn := NewServeFunctionKeyWithDefault() + faasFuncUrn.AppName = belongedApp.Name + faasFuncUrn.DeploymentName = s.Name + + // make a copied app to make it contains only this deployment info + copiedApp := *belongedApp + copiedApp.Deployments = []ServeDeploymentSchema{*s} + + return &ServeFuncWithKeysAndFunctionMetaInfo{ + FuncMetaKey: faasFuncUrn.ToFuncMetaKey(), + InstanceMetaKey: faasFuncUrn.ToInstancesMetaKey(), + FuncMetaInfo: &FunctionMetaInfo{ + FuncMetaData: FuncMetaData{ + Name: faasFuncUrn.DeploymentName, + Runtime: defaultServeAppRuntime, + Timeout: defaultServeAppTimeout, + Version: faasFuncUrn.Version, + FunctionURN: faasFuncUrn.ToFaasFunctionUrn(), + TenantID: faasFuncUrn.TenantID, + FunctionVersionURN: faasFuncUrn.ToFaasFunctionVersionUrn(), + FuncName: faasFuncUrn.DeploymentName, + BusinessType: constants.BusinessTypeServe, + }, + ResourceMetaData: ResourceMetaData{ + CPU: defaultServeAppCpu, + Memory: defaultServeAppMemory, + }, + InstanceMetaData: InstanceMetaData{ + MaxInstance: s.NumReplicas, + MinInstance: s.NumReplicas, + ConcurrentNum: defaultServeAppConcurrentNum, + IdleMode: false, + }, + ExtendedMetaData: ExtendedMetaData{ + ServeDeploySchema: ServeDeploySchema{ + Applications: []ServeApplicationSchema{ + copiedApp, + }, + }, + }, + }, + } +} + +const ( + defaultTenantID = "12345678901234561234567890123456" + defaultFuncVersion = "latest" + + faasMetaKey = constants.MetaFuncKey + instanceMetaKey = "/instances/business/yrk/cluster/cluster001/tenant/%s/function/%s/version/%s" + faasFuncURN6tuplePattern = "sn:cn:yrk:%s:function:%s" + faasFuncURN7tuplePattern = "sn:cn:yrk:%s:function:%s:%s" +) + +// ServeFunctionKey is a faas urn with necessary parts +type ServeFunctionKey struct { + TenantID string + AppName string + DeploymentName string + Version string +} + +// NewServeFunctionKeyWithDefault returns a struct with default values +func NewServeFunctionKeyWithDefault() *ServeFunctionKey { + return &ServeFunctionKey{ + TenantID: defaultTenantID, + Version: defaultFuncVersion, + } +} + +// ToFuncNameTriplet - 0@svc@func +func (f *ServeFunctionKey) ToFuncNameTriplet() string { + return fmt.Sprintf("0@%s@%s", f.AppName, f.DeploymentName) +} + +// ToFuncMetaKey - /sn/functions/business/yrk/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest +func (f *ServeFunctionKey) ToFuncMetaKey() string { + return fmt.Sprintf(faasMetaKey, f.TenantID, f.ToFuncNameTriplet(), f.Version) +} + +// ToInstancesMetaKey - /instances/business/yrk/cluster/cluster001/tenant/125...346/function/0@svc@func/version/latest +func (f *ServeFunctionKey) ToInstancesMetaKey() string { + return fmt.Sprintf(instanceMetaKey, f.TenantID, f.ToFuncNameTriplet(), f.Version) +} + +// ToFaasFunctionUrn - sn:cn:yrk:12345678901234561234567890123456:function:0@service@function +func (f *ServeFunctionKey) ToFaasFunctionUrn() string { + return fmt.Sprintf(faasFuncURN6tuplePattern, f.TenantID, f.ToFuncNameTriplet()) +} + +// ToFaasFunctionVersionUrn - sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func:latest +func (f *ServeFunctionKey) ToFaasFunctionVersionUrn() string { + return fmt.Sprintf(faasFuncURN7tuplePattern, f.TenantID, f.ToFuncNameTriplet(), f.Version) +} + +// FromFaasFunctionKey - 12345678901234561234567890123456/0@svc@func/latest +func (f *ServeFunctionKey) FromFaasFunctionKey(funcKey string) error { + const ( + serveFaasFuncKeyMatchesIdxTenantID = iota + 1 + serveFaasFuncKeyMatchesIdxAppName + serveFaasFuncKeyMatchesIdxDeploymentName + serveFaasFuncKeyMatchesIdxVersion + serveFaasFuncKeyMatchesIdxMax + ) + re := regexp.MustCompile(`^([a-zA-Z0-9]*)/.*@([^@]+)@([^/]+)/(.*)$`) + matches := re.FindStringSubmatch(funcKey) + if len(matches) < serveFaasFuncKeyMatchesIdxMax { + return fmt.Errorf("extract failed from %s", funcKey) + } + f.TenantID = matches[serveFaasFuncKeyMatchesIdxTenantID] + f.AppName = matches[serveFaasFuncKeyMatchesIdxAppName] + f.DeploymentName = matches[serveFaasFuncKeyMatchesIdxDeploymentName] + f.Version = matches[serveFaasFuncKeyMatchesIdxVersion] + return nil +} diff --git a/functionsystem/apps/meta_service/common/types/serve_test.go b/functionsystem/apps/meta_service/common/types/serve_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4c73210104d42147b7539f803143683bbb663306 --- /dev/null +++ b/functionsystem/apps/meta_service/common/types/serve_test.go @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2023 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types - +package types + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestServeFunctionKeyTrans(t *testing.T) { + k := NewServeFunctionKeyWithDefault() + k.AppName = "svc" + k.DeploymentName = "func" + convey.Convey("Given a serve function key", t, func() { + convey.Convey("When trans to a func name triplet", func() { + convey.So(k.ToFuncNameTriplet(), convey.ShouldEqual, "0@svc@func") + }) + convey.Convey("When trans to a func meta key", func() { + convey.So(k.ToFuncMetaKey(), convey.ShouldEqual, + "/sn/functions/business/yrk/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + }) + convey.Convey("When trans to a instance meta key", func() { + convey.So(k.ToInstancesMetaKey(), convey.ShouldEqual, + "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + }) + convey.Convey("When trans to a FaasFunctionUrn", func() { + convey.So(k.ToFaasFunctionUrn(), convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func") + }) + convey.Convey("When trans ToFaasFunctionVersionUrn", func() { + convey.So(k.ToFaasFunctionVersionUrn(), convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func:latest") + }) + }) +} + +func TestServeDeploySchema_ToFaaSFuncMetas(t *testing.T) { + convey.Convey("Test ServeDeploySchema ToFaaSFuncMetas", t, func() { + // Setup mock data + app1 := ServeApplicationSchema{ + Name: "app1", + RoutePrefix: "/app1", + ImportPath: "path1", + RuntimeEnv: ServeRuntimeEnvSchema{ + Pip: []string{"package1", "package2"}, + WorkingDir: "/app1", + EnvVars: map[string]any{"key1": "value1"}, + }, + Deployments: []ServeDeploymentSchema{ + { + Name: "deployment1", + NumReplicas: 2, + HealthCheckPeriodS: 30, + HealthCheckTimeoutS: 10, + }, + }, + } + + serveDeploy := ServeDeploySchema{ + Applications: []ServeApplicationSchema{app1}, + } + + convey.Convey("It should return correct faas function metas", func() { + result := serveDeploy.ToFaaSFuncMetas() + convey.So(len(result), convey.ShouldBeGreaterThan, 0) + convey.So(result[0].FuncMetaKey, convey.ShouldNotBeEmpty) + }) + }) +} + +func TestServeFunctionKey(t *testing.T) { + convey.Convey("Test FromFaasFunctionKey", t, func() { + convey.Convey("It should return correct faas function metas", func() { + key := "12345678901234561234567890123456/0@svc@func/latest" + sfk := ServeFunctionKey{} + err := sfk.FromFaasFunctionKey(key) + convey.So(err, convey.ShouldBeNil) + convey.So(sfk.Version, convey.ShouldEqual, "latest") + convey.So(sfk.AppName, convey.ShouldEqual, "svc") + convey.So(sfk.DeploymentName, convey.ShouldEqual, "func") + convey.So(sfk.TenantID, convey.ShouldEqual, "12345678901234561234567890123456") + }) + convey.Convey("It should return incorrect faas function metas", func() { + key := "12345678901234561234567890123456/0@svc@func" + sfk := ServeFunctionKey{} + err := sfk.FromFaasFunctionKey(key) + convey.So(err, convey.ShouldNotBeNil) + }) + }) + + convey.Convey("Test FaasKey Test", t, func() { + convey.Convey("test default faas key", func() { + sfk := NewServeFunctionKeyWithDefault() + convey.So(sfk.TenantID, convey.ShouldEqual, defaultTenantID) + convey.So(sfk.Version, convey.ShouldEqual, defaultFuncVersion) + }) + + convey.Convey("test convert", func() { + sfk := NewServeFunctionKeyWithDefault() + sfk.AppName = "svc" + sfk.DeploymentName = "func" + + convey.So(sfk.ToFuncNameTriplet(), + convey.ShouldEqual, + "0@svc@func") + convey.So(sfk.ToFuncMetaKey(), + convey.ShouldEqual, + "/sn/functions/business/yrk/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + convey.So(sfk.ToInstancesMetaKey(), + convey.ShouldEqual, + "/instances/business/yrk/cluster/cluster001/tenant/12345678901234561234567890123456/function/0@svc@func/version/latest") + convey.So(sfk.ToFaasFunctionUrn(), + convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func") + convey.So(sfk.ToFaasFunctionVersionUrn(), + convey.ShouldEqual, + "sn:cn:yrk:12345678901234561234567890123456:function:0@svc@func:latest") + }) + + convey.Convey("It should return incorrect faas function metas", func() { + sfk := NewServeFunctionKeyWithDefault() + convey.So(sfk.TenantID, convey.ShouldEqual, defaultTenantID) + convey.So(sfk.Version, convey.ShouldEqual, defaultFuncVersion) + }) + }) +} + +func TestServeDeploySchemaValidate(t *testing.T) { + convey.Convey("Test Validate", t, func() { + sds := ServeDeploySchema{ + Applications: []ServeApplicationSchema{ + { + Name: "app1", + RoutePrefix: "/app1", + ImportPath: "path1", + RuntimeEnv: ServeRuntimeEnvSchema{ + Pip: []string{"package1", "package2"}, + WorkingDir: "/app1", + EnvVars: map[string]any{"key1": "value1"}, + }, + Deployments: []ServeDeploymentSchema{ + { + Name: "deployment1", + NumReplicas: 2, + HealthCheckPeriodS: 30, + HealthCheckTimeoutS: 10, + }, + }, + }, + }, + } + convey.Convey("on repeated app name", func() { + sdsOther := sds + app0 := sdsOther.Applications[0] + sdsOther.Applications = append(sdsOther.Applications, app0) + + err := sdsOther.Validate() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("on repeated route prefix", func() { + sdsOther := sds + app0 := sdsOther.Applications[0] + app0.Name = "othername" + sdsOther.Applications = append(sdsOther.Applications, app0) + + err := sdsOther.Validate() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("on empty app name", func() { + sdsOther := sds + app0 := sdsOther.Applications[0] + app0.Name = "" + app0.RoutePrefix = "/other" + sdsOther.Applications = append(sdsOther.Applications, app0) + + err := sdsOther.Validate() + convey.So(err, convey.ShouldNotBeNil) + }) + convey.Convey("ok", func() { + err := sds.Validate() + convey.So(err, convey.ShouldBeNil) + }) + }) +} diff --git a/functionsystem/apps/meta_service/common/types/tcpCalltypes.go b/functionsystem/apps/meta_service/common/types/tcpCalltypes.go new file mode 100644 index 0000000000000000000000000000000000000000..6eadb94e1b9315f808d2748f8a4840b445da01eb --- /dev/null +++ b/functionsystem/apps/meta_service/common/types/tcpCalltypes.go @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types - +package types + +import ( + "context" + "sync/atomic" +) + +// TCPRequest contains users request +type TCPRequest struct { + LogType string + RawData []byte + RequestID uint64 + UserData map[string]string + Priority int + DirectRequest bool + CanBackpressure bool + CPU string + Memory string + InvokeType string + AffinityInfo AffinityInfo + GroupInfo GroupInfo + ResourceMetaData map[string]float32 + TraceCtx context.Context +} + +// TCPResponse contains users response +type TCPResponse struct { + ErrorCode uint32 + ErrorMessage string + RawData []byte + Logs string + RequestID uint64 + Summary string +} + +// Position position where to do requests +type Position struct { + InstanceID string + NodeID string +} + +// DispatcherRequest call request struct +type DispatcherRequest struct { + Request *TCPRequest + Response chan *TCPResponse + Position Position + FunctionKey string + TraceID string + NodeLabel string + FutureID string + canceled uint32 + WorkerID string +} + +// Cancel set status of request to canceled +func (dr *DispatcherRequest) Cancel() { + atomic.StoreUint32(&dr.canceled, 1) +} + +// IsCanceled query whether request is canceled +func (dr *DispatcherRequest) IsCanceled() bool { + return atomic.LoadUint32(&dr.canceled) == 1 +} diff --git a/functionsystem/apps/meta_service/common/types/types.go b/functionsystem/apps/meta_service/common/types/types.go new file mode 100644 index 0000000000000000000000000000000000000000..81a2f3583b7968848f44909d2cfe7a2e700b8825 --- /dev/null +++ b/functionsystem/apps/meta_service/common/types/types.go @@ -0,0 +1,665 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package types - +package types + +import ( + "meta_service/common/logger/logservice" + "meta_service/common/trace" +) + +// DNSDomainInfo dns domain info +type DNSDomainInfo struct { + ID string `json:"id"` + DomainName string `json:"domain_name"` + Type string `json:"type" valid:",optional"` + ZoneType string `json:"zone_type" valid:",optional"` +} + +// Layer define layer info +type Layer struct { + BucketURL string `json:"bucketUrl" valid:"url,optional"` + ObjectID string `json:"objectId" valid:"stringlength(1|255),optional"` + BucketID string `json:"bucketId" valid:"stringlength(1|255),optional"` + AppID string `json:"appId" valid:"stringlength(1|128),optional"` + ETag string `json:"etag" valid:"optional"` + Link string `json:"link" valid:"optional"` + Name string `json:"name" valid:",optional"` + Sha256 string `json:"sha256" valid:"optional"` + DependencyType string `json:"dependencyType" valid:",optional"` +} + +// FunctionMetaInfo define function meta info for FunctionGraph +type FunctionMetaInfo struct { + FuncMetaData FuncMetaData `json:"funcMetaData" valid:",optional"` + S3MetaData S3MetaData `json:"s3MetaData" valid:",optional"` + CodeMetaData CodeMetaData `json:"codeMetaData" valid:",optional"` + EnvMetaData EnvMetaData `json:"envMetaData" valid:",optional"` + ResourceMetaData ResourceMetaData `json:"resourceMetaData" valid:",optional"` + InstanceMetaData InstanceMetaData `json:"instanceMetaData" valid:",optional"` + ExtendedMetaData ExtendedMetaData `json:"extendedMetaData" valid:",optional"` +} + +// FuncMetaData define meta data of functions +type FuncMetaData struct { + Layers []*Layer `json:"layers" valid:",optional"` + Name string `json:"name"` + FunctionDescription string `json:"description" valid:"stringlength(1|1024)"` + FunctionURN string `json:"functionUrn"` + TenantID string `json:"tenantId"` + AgentID string `json:"agentId" valid:",optional"` + EnvironmentID string `json:"environmentId" valid:",optional"` + Tags map[string]string `json:"tags" valid:",optional"` + FunctionUpdateTime string `json:"functionUpdateTime" valid:",optional"` + FunctionVersionURN string `json:"functionVersionUrn"` + RevisionID string `json:"revisionId" valid:"stringlength(1|20),optional"` + CodeSize int `json:"codeSize" valid:"int"` + CodeSha512 string `json:"codeSha512" valid:"stringlength(1|128),optional"` + Handler string `json:"handler" valid:"stringlength(1|255)"` + Runtime string `json:"runtime" valid:"stringlength(1|63)"` + Timeout int64 `json:"timeout" valid:"required"` + Version string `json:"version" valid:"stringlength(1|32)"` + DeadLetterConfig string `json:"deadLetterConfig" valid:"stringlength(1|255)"` + BusinessID string `json:"businessId" valid:"stringlength(1|32)"` + FunctionType string `json:"functionType" valid:",optional"` + FuncID string `json:"func_id" valid:",optional"` + FuncName string `json:"func_name" valid:",optional"` + DomainID string `json:"domain_id" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` + Service string `json:"service" valid:",optional"` + Dependencies string `json:"dependencies" valid:",optional"` + EnableCloudDebug string `json:"enable_cloud_debug" valid:",optional"` + IsStatefulFunction bool `json:"isStatefulFunction" valid:"optional"` + IsBridgeFunction bool `json:"isBridgeFunction" valid:"optional"` + IsStreamEnable bool `json:"isStreamEnable" valid:"optional"` + Type string `json:"type" valid:"optional"` + EnableAuthInHeader bool `json:"enable_auth_in_header" valid:"optional"` + DNSDomainCfg []DNSDomainInfo `json:"dns_domain_cfg" valid:",optional"` + VPCTriggerImage string `json:"vpcTriggerImage" valid:",optional"` + StateConfig StateConfig `json:"stateConfig" valid:",optional"` + BusinessType string `json:"businessType" valid:"optional"` +} + +// StateConfig ConsistentWithInstance- The lifecycle is consistent with that of the instance. +// Independent - The lifecycle is independent of instances. +type StateConfig struct { + LifeCycle string `json:"lifeCycle"` +} + +// FuncCode include function code file and link info +type FuncCode struct { + File string `json:"file" valid:",optional"` + Link string `json:"link" valid:",optional"` +} + +// S3MetaData define meta function info for OBS +type S3MetaData struct { + AppID string `json:"appId" valid:"stringlength(1|128),optional"` + BucketID string `json:"bucketId" valid:"stringlength(1|255),optional"` + ObjectID string `json:"objectId" valid:"stringlength(1|255),optional"` + BucketURL string `json:"bucketUrl" valid:"url,optional"` + CodeType string `json:"code_type" valid:",optional"` + CodeURL string `json:"code_url" valid:",optional"` + CodeFileName string `json:"code_filename" valid:",optional"` + FuncCode FuncCode `json:"func_code" valid:",optional"` +} + +// LocalMetaData - +type LocalMetaData struct { + StorageType string `json:"storage_type" valid:",optional"` + CodePath string `json:"code_path" valid:",optional"` +} + +// CodeMetaData - +type CodeMetaData struct { + Sha512 string `json:"sha512" valid:",optional"` + LocalMetaData + S3MetaData +} + +// EnvMetaData - +type EnvMetaData struct { + Environment string `json:"environment"` + EncryptedUserData string `json:"encrypted_user_data"` + EnvKey string `json:"envKey" valid:",optional"` + CryptoAlgorithm string `json:"cryptoAlgorithm" valid:",optional"` +} + +// ResourceMetaData include resource data such as cpu and memory +type ResourceMetaData struct { + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + GpuMemory int64 `json:"gpu_memory"` + EnableDynamicMemory bool `json:"enable_dynamic_memory" valid:",optional"` + CustomResources string `json:"customResources" valid:",optional"` + EnableTmpExpansion bool `json:"enable_tmp_expansion" valid:",optional"` + EphemeralStorage int `json:"ephemeral_storage" valid:"int,optional"` + CustomResourcesSpec string `json:"CustomResourcesSpec" valid:",optional"` +} + +// InstanceMetaData define instance meta data of FG functions +type InstanceMetaData struct { + MaxInstance int64 `json:"maxInstance" valid:",optional"` + MinInstance int64 `json:"minInstance" valid:",optional"` + ConcurrentNum int `json:"concurrentNum" valid:",optional"` + DiskLimit int64 `json:"diskLimit" valid:",optional"` + InstanceType string `json:"instanceType" valid:",optional"` + SchedulePolicy string `json:"schedulePolicy" valid:",optional"` + ScalePolicy string `json:"scalePolicy" valid:",optional"` + IdleMode bool `json:"idleMode" valid:",optional"` + PoolLabel string `json:"poolLabel"` + PoolID string `json:"poolId" valid:",optional"` +} + +// VpcConfig include info of function vpc +type VpcConfig struct { + ID string `json:"id,omitempty"` + DomainID string `json:"domain_id,omitempty"` + Namespace string `json:"namespace,omitempty"` + VpcName string `json:"vpc_name,omitempty"` + VpcID string `json:"vpc_id,omitempty"` + SubnetName string `json:"subnet_name,omitempty"` + SubnetID string `json:"subnet_id,omitempty"` + TenantCidr string `json:"tenant_cidr,omitempty"` + HostVMCidr string `json:"host_vm_cidr,omitempty"` + Gateway string `json:"gateway,omitempty"` + Xrole string `json:"xrole,omitempty"` +} + +// StrategyConfig - +type StrategyConfig struct { + Concurrency int `json:"concurrency" valid:",optional"` +} + +// Heartbeat define user custom heartbeat function config +type Heartbeat struct { + // Handler define heartbeat function entry + Handler string `json:"heartbeat_handler" valid:",optional"` +} + +// LogTankService - +type LogTankService struct { + GroupID string `json:"logGroupId" valid:",optional"` + StreamID string `json:"logStreamId" valid:",optional"` +} + +// TraceService - +type TraceService struct { + TraceAK string `json:"tracing_ak" valid:",optional"` + TraceSK string `json:"tracing_sk" valid:",optional"` + ProjectName string `json:"project_name" valid:",optional"` +} + +// CustomContainerConfig contains the metadata for custom container +type CustomContainerConfig struct { + ControlPath string `json:"control_path" valid:",optional"` + Image string `json:"image" valid:",optional"` + Command []string `json:"command" valid:",optional"` + Args []string `json:"args" valid:",optional"` + WorkingDir string `json:"working_dir" valid:",optional"` + UID int `json:"uid" valid:",optional"` + GID int `json:"gid" valid:",optional"` +} + +// ExtendedMetaData define external meta data of functions +type ExtendedMetaData struct { + ImageName string `json:"image_name" valid:",optional"` + Role Role `json:"role" valid:",optional"` + VpcConfig *VpcConfig `json:"func_vpc" valid:",optional"` + EndpointTenantVpc *VpcConfig `json:"endpoint_tenant_vpc" valid:",optional"` + FuncMountConfig *FuncMountConfig `json:"mount_config" valid:",optional"` + StrategyConfig StrategyConfig `json:"strategy_config" valid:",optional"` + ExtendConfig string `json:"extend_config" valid:",optional"` + Initializer Initializer `json:"initializer" valid:",optional"` + Heartbeat Heartbeat `json:"heartbeat" valid:",optional"` + EnterpriseProjectID string `json:"enterprise_project_id" valid:",optional"` + LogTankService LogTankService `json:"log_tank_service" valid:",optional"` + TraceService TraceService `json:"tracing_config" valid:",optional"` + CustomContainerConfig CustomContainerConfig `json:"custom_container_config" valid:",optional"` + AsyncConfigLoaded bool `json:"async_config_loaded" valid:",optional"` + RestoreHook RestoreHook `json:"restore_hook,omitempty" valid:",optional"` + NetworkController NetworkController `json:"network_controller" valid:",optional"` + UserAgency UserAgency `json:"user_agency" valid:",optional"` + CustomFilebeatConfig CustomFilebeatConfig `json:"custom_filebeat_config"` + CustomHealthCheck CustomHealthCheck `json:"custom_health_check" valid:",optional"` + DynamicConfig DynamicConfigEvent `json:"dynamic_config" valid:",optional"` + CustomGracefulShutdown CustomGracefulShutdown `json:"runtime_graceful_shutdown"` + PreStop PreStop `json:"pre_stop"` + RaspConfig RaspConfig `json:"rasp_config"` + ServeDeploySchema ServeDeploySchema `json:"serveDeploySchema" valid:"optional"` + ImagePullConfig ImagePullConfig `json:"imagePullConfig,omitempty"` +} + +// CustomHealthCheck custom health check +type CustomHealthCheck struct { + TimeoutSeconds int `json:"timeoutSeconds" valid:",optional"` + PeriodSeconds int `json:"periodSeconds" valid:",optional"` + FailureThreshold int `json:"failureThreshold" valid:",optional"` +} + +// DynamicConfigEvent dynamic config etcd event +type DynamicConfigEvent struct { + Enabled bool `json:"enabled"` // use for signature + UpdateTime string `json:"update_time"` + ConfigContent []KV `json:"config_content"` +} + +// CustomGracefulShutdown define the option of custom container's runtime graceful shutdown +type CustomGracefulShutdown struct { + MaxShutdownTimeout int `json:"maxShutdownTimeout"` +} + +// ImagePullConfig image pull config +type ImagePullConfig struct { + Secrets []string `json:"secrets,omitempty"` +} + +// KV config key and value +type KV struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// CustomFilebeatConfig custom filebeat config +type CustomFilebeatConfig struct { + SidecarConfigInfo *SidecarConfigInfo `json:"sidecarConfigInfo"` + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + Version string `json:"version"` + ImageAddress string `json:"imageAddress"` +} + +// RaspConfig rasp config key and value +type RaspConfig struct { + InitImage string `json:"init-image"` + RaspImage string `json:"rasp-image"` + RaspServerIP string `json:"rasp-server-ip"` + RaspServerPort string `json:"rasp-server-port"` + Envs []KV `json:"envs"` +} + +// SidecarConfigInfo sidecat config info +type SidecarConfigInfo struct { + ConfigFiles []CustomLogConfigFile `json:"configFiles"` + LiveNessShell string `json:"livenessShell"` + ReadNessShell string `json:"readnessShell"` + PreStopCommands string `json:"preStopCommands"` +} + +// CustomLogConfigFile custom log config file +type CustomLogConfigFile struct { + Path string `json:"path"` + Data string `json:"data"` + Secret bool `json:"secret"` +} + +// UserAgency define AK/SK of user's agency +type UserAgency struct { + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + Token string `json:"token"` + SecurityAk string `json:"securityAk"` + SecuritySk string `json:"securitySk"` + SecurityToken string `json:"securityToken"` +} + +// VpcInfo contains the information of VPC access restriction +type VpcInfo struct { + VpcName string `json:"vpc_name,omitempty"` + VpcID string `json:"vpc_id,omitempty"` +} + +// NetworkController contains some special network settings +type NetworkController struct { + DisablePublicNetwork bool `json:"disable_public_network" valid:",optional"` + TriggerAccessVpcs []VpcInfo `json:"trigger_access_vpcs" valid:",optional"` +} + +// RestoreHook include restorehook handler and timeout +type RestoreHook struct { + Handler string `json:"restore_hook_handler,omitempty" valid:",optional"` + Timeout int64 `json:"restore_hook_timeout,omitempty" valid:",optional"` +} + +// HTTPResponse is general http response +type HTTPResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// InnerInstanceData is the function instance data stored in ETCD +type InnerInstanceData struct { + IP string `json:"ip"` + Port string `json:"port"` + Status string `json:"status"` + P2pPort string `json:"p2pPort"` + GrpcPort string `json:"grpcPort,omitempty"` + NodeIP string `json:"nodeIP,omitempty"` + NodePort string `json:"nodePort,omitempty"` + NodeName string `json:"nodeName,omitempty"` + NodeID string `json:"nodeID,omitempty"` + Applier string `json:"applier,omitempty"` // silimar to OwnerIP + OwnerIP string `json:"ownerIP,omitempty"` + FuncSig string `json:"functionSignature,omitempty"` + Reserved bool `json:"reserved,omitempty"` + CPU int64 `json:"cpu,omitempty"` + Memory int64 `json:"memory,omitempty"` + GroupID string `json:"groupID,omitempty"` + StackID string `json:"stackID,omitempty"` + CustomResources map[string]int64 `json:"customResources,omitempty" valid:"optional"` +} + +// FunctionSpec define function spec +type FunctionSpec struct { + LogicInstanceID string `json:"logicInstanceID"` + BusAddress string `json:"busAddress"` + URN string `json:"urn"` + Handler string `json:"handler"` + EntryFile string `json:"entry_file"` + Timeout int64 `json:"timeout"` + EnvKey string `json:"env_key"` + Environment string `json:"env"` + EncryptedUserData string `json:"encrypted_user_data"` + Language string `json:"language"` + CodeSha256 string `json:"codeSha256,omitempty"` + ResourceMetaData ResourceMetaData `json:"resourceMetaData"` + Initializer Initializer `json:"initializer"` + PreStop PreStop `json:"pre_stop"` + Role Role `json:"role" valid:",optional"` + DomainID string `json:"domain_id" valid:",optional"` + LogTankService logservice.LogTankService `json:"logTankService"` + TraceService trace.Service `json:"traceService"` + FuncMountConfig FuncMountConfig `json:"mount_config" valid:",optional"` + HookHandler map[string]string `json:"hookHandler" valid:",optional"` +} + +// Initializer include initializer handler and timeout +type Initializer struct { + Handler string `json:"initializer_handler" valid:",optional"` + Timeout int64 `json:"initializer_timeout" valid:",optional"` +} + +// PreStop include pre_stop handler and timeout +type PreStop struct { + Handler string `json:"pre_stop_handler" valid:",optional"` + Timeout int64 `json:"pre_stop_timeout" valid:",optional"` +} + +// FuncMountConfig function mount config +type FuncMountConfig struct { + FuncMountUser FuncMountUser `json:"mount_user" valid:",optional"` + FuncMounts []FuncMount `json:"func_mounts" valid:",optional"` +} + +// FuncMountUser function mount user +type FuncMountUser struct { + UserID int `json:"user_id" valid:",optional"` + GroupID int `json:"user_group_id" valid:",optional"` +} + +// FuncMount function mount +type FuncMount struct { + MountType string `json:"mount_type" valid:",optional"` + MountResource string `json:"mount_resource" valid:",optional"` + MountSharePath string `json:"mount_share_path" valid:",optional"` + LocalMountPath string `json:"local_mount_path" valid:",optional"` + Status string `json:"status" valid:",optional"` +} + +// Role include x_role and app_x_role +type Role struct { + XRole string `json:"xrole" valid:",optional"` + AppXRole string `json:"app_xrole" valid:",optional"` +} + +// FunctionDeploymentSpec define function deployment spec +type FunctionDeploymentSpec struct { + BucketID string `json:"bucket_id"` + ObjectID string `json:"object_id"` + Layers string `json:"layers"` + DeployDir string `json:"deploydir"` +} + +// FunctionMessage define function message +type FunctionMessage struct { + Function FunctionSpec `json:"function"` + Deployment FunctionDeploymentSpec `json:"deployment"` +} + +// InstanceResource describes the cpu and memory info of an instance +type InstanceResource struct { + CPU string `json:"cpu"` + Memory string `json:"memory"` + CustomResources map[string]int64 `json:"customresources"` +} + +// Worker define a worker +type Worker struct { + Instances []*Instance `json:"instances"` + FunctionName string `json:"functionname"` + FunctionVersion string `json:"functionversion"` + Tenant string `json:"tenant"` + Business string `json:"business"` +} + +// Instance define a instance +type Instance struct { + IP string `json:"ip"` + Port string `json:"port"` + GrpcPort string `json:"grpcPort"` + InstanceID string `json:"instanceID,omitempty"` + DeployedIP string `json:"deployed_ip"` + DeployedNode string `json:"deployed_node"` + DeployedNodeID string `json:"deployed_node_id"` + TenantID string `json:"tenant_id"` +} + +// InstanceCreationRequest is used to create instance +type InstanceCreationRequest struct { + LogicInstanceID string `json:"logicInstanceID"` + FuncName string `json:"functionName"` + Applier string `json:"applier"` + DeployNode string `json:"deployNode"` + Business string `json:"business"` + TenantID string `json:"tenantID"` + Version string `json:"version"` + OwnerIP string `json:"ownerIP"` + TraceID string `json:"traceID"` + TriggerFlag string `json:"triggerFlag"` + VersionUrn string `json:"versionUrn"` + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + GroupID string `json:"groupID"` + StackID string `json:"stackID"` + CustomResources map[string]int64 `json:"customResources,omitempty" valid:"optional"` +} + +// InstanceCreationSuccessResponse is the struct returned by workermanager upon successful instance creation +type InstanceCreationSuccessResponse struct { + HTTPResponse + Worker *Worker `json:"worker"` + Instance *Instance `json:"instance"` +} + +// InstanceDeletionRequest is used to delete instance +type InstanceDeletionRequest struct { + InstanceID string `json:"instanceID"` + FuncName string `json:"functionName"` + FuncVersion string `json:"functionVersion"` + TenantID string `json:"tenantID"` + BusinessID string `json:"businessID"` + Applier string `json:"applier"` + Force bool `json:"force"` +} + +// InstanceDeletionResponse is the struct returned by workermanager upon successful instance deletion +type InstanceDeletionResponse struct { + HTTPResponse + Reserved bool `json:"reserved"` +} + +// HookArgs keeps args of hook +type HookArgs struct { + FuncArgs []byte // Call() request in worker + SrcTenant string + DstTenant string + StateID string + LogType string + StateKey string // for trigger state call + FunctionVersion string // for trigger state call + ExternalRequest bool // for trigger state call + ServiceID string + TraceID string + InvokeType string +} + +// ResourceStack stores properties of resource stack +type ResourceStack struct { + StackID string `json:"id" valid:"required"` + CPU int64 `json:"cpu" valid:"required"` + Mem int64 `json:"mem" valid:"required"` + CustomResources map[string]int64 `json:"customResources,omitempty" valid:"optional"` +} + +// ResourceGroup stores properties of resource group +type ResourceGroup struct { + GroupID string `json:"id" valid:"required"` + DeployOption string `json:"deployOption" valid:"required"` + GroupState string `json:"groupState" valid:"required"` + ResourceStacks []ResourceStack `json:"resourceStacks" valid:"required"` + ScheduledStacks map[string][]ResourceStack `json:"scheduledStacks,omitempty" valid:"optional"` +} + +// AffinityInfo is data affinity information +type AffinityInfo struct { + AffinityRequest AffinityRequest + AffinityNode string // if AffinityNode is not empty, the affinity node has been calculated + NeedToForward bool +} + +// AffinityRequest is affinity request parameter +type AffinityRequest struct { + Strategy string `json:"strategy"` + ObjectIDs []string `json:"object_ids"` +} + +// GroupInfo stores groupID and stackID +type GroupInfo struct { + GroupID string `json:"groupID"` + StackID string `json:"stackID"` +} + +// InvokeOption contains invoke options +type InvokeOption struct { + AffinityRequest AffinityRequest + GroupInfo GroupInfo + ResourceMetaData map[string]float32 +} + +// ScheduleConfig defines schedule config +type ScheduleConfig struct { + Policy int `json:"policy" valid:"optional"` + GetMetricsTickInterval int `json:"getMetricsTickInterval" valid:"optional"` + MaxGetMetricsFailedCount int `json:"maxGetMetricsFailedCount" valid:"optional"` + ForwardScheduleFirst bool `json:"forwardScheduleResourceNotEnough" valid:"optional"` + SleepingMemThreshold float64 `json:"sleepingMemoryThreshold" valid:"optional"` + SelectInstanceToSleepingPolicy string `json:"selectInstanceToSleepingPolicy" valid:"optional"` + ResourceManagementType string `json:"resourceManagementType" valid:"optional"` +} + +// MetricsData shows the quantities of a specific resource +type MetricsData struct { + TotalResource float64 `json:"totalResource"` + InUseResource float64 `json:"inUseResource"` +} + +// ResourceMetrics contains several resources' MetricsData +type ResourceMetrics map[string]MetricsData + +// WorkerMetrics stores metrics used for scheduler +type WorkerMetrics struct { + SystemResources ResourceMetrics + // key levels: functionUrn instanceID + FunctionResources map[string]map[string]ResourceMetrics +} + +// InstanceInfo stores instance info for cli query +type InstanceInfo struct { + InstanceID string `json:"InstanceID"` + Status string `json:"Status"` +} + +// InnerWorkerData is the worker data stored in ETCD +type InnerWorkerData struct { + IP string `json:"ip"` + Port string `json:"port"` + NodeIP string `json:"nodeIP"` + P2pPort string `json:"p2pPort"` + NodeName string `json:"nodeName"` + NodeID string `json:"nodeID"` + WorkerAgentID string `json:"workerAgentID"` + AllocatableCPU int64 `json:"allocatableCPU"` + AllocatableMemory int64 `json:"allocatableMemory"` + AllocatableCustomResource map[string]int64 `json:"allocatableCustomResource"` +} + +// TerminateRequest sent from worker manager to worker to delete function instance +type TerminateRequest struct { + RuntimeID string `json:"runtime_id"` + FuncName string `json:"function_name"` + FuncVersion string `json:"function_version"` + TenantID string `json:"tenant_id"` + BusinessID string `json:"business_id" valid:"optional"` +} + +// FunctionInstance is the function instance data stored in ETCD +type FunctionInstance struct { + DeployedIP string `json:"deployedIP"` + DeployedNode string `json:"deployedNode"` + RuntimeID string `json:"runtimeID"` + RuntimeIP string `json:"runtimeIP"` + RuntimePort string `json:"runtimePort"` + FuncKey string `json:"funcKey"` + Resource InstanceResource `json:"resource"` + Status int32 `json:"status"` + Labels []string `json:"labels"` + IsDriver bool `json:"isDriver"` +} + +// HitsDataSource 匹配内容源 +type HitsDataSource struct { + Time string `json:"Time"` + Message string `json:"Message"` + PodName string `json:"PodName"` + FunctionName string `json:"DeploymentName"` + Version string `json:"Version"` + Level string `json:"Level"` + TraceID string `json:"TraceID"` + RuntimeID string `json:"RuntimeID"` + TenantID string `json:"TenantID"` + TimeNano int64 `json:"TimeNano"` +} + +// Device is the device store in etcd +type Device struct { + Model string `json:"model,omitempty"` + Hbm int `json:"hbm,omitempty"` + Type string `json:"type,omitempty"` + Count int `json:"count,omitempty"` + Latency int `json:"latency,omitempty"` + Stream int `json:"stream,omitempty"` +} diff --git a/functionsystem/apps/meta_service/common/types/types_test.go b/functionsystem/apps/meta_service/common/types/types_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1d125122d4ee3e84ccc565ee47594cbf31ec8dec --- /dev/null +++ b/functionsystem/apps/meta_service/common/types/types_test.go @@ -0,0 +1,9 @@ +package types + +import "testing" + +func TestTypes(t *testing.T) { + dr := &DispatcherRequest{} + dr.Cancel() + dr.IsCanceled() +} diff --git a/functionsystem/apps/meta_service/common/urnutils/gadgets.go b/functionsystem/apps/meta_service/common/urnutils/gadgets.go new file mode 100644 index 0000000000000000000000000000000000000000..02dd7a6ee278119fd35cf1957dbc1db5c87c24c5 --- /dev/null +++ b/functionsystem/apps/meta_service/common/urnutils/gadgets.go @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package urnutils + +import ( + "strings" + + "meta_service/common/constants" +) + +// Separator is the current system special +var ( + Separator = "-" +) + +const ( + // ServiceIDPrefix is the prefix of the function with serviceID. + ServiceIDPrefix = "0" + + // DefaultSeparator is a character that separates functions and services. + DefaultSeparator = "-" + + // ServicePrefix is the prefix of the function with serviceID. + ServicePrefix = "0-" + + // FaaSServiceIDPrefix is the prefix of the function with serviceID. + FaaSServiceIDPrefix = "0" + + // DefaultFaaSSeparator is a character that separates functions and services. + DefaultFaaSSeparator = "@" + + // FaaSServicePrefix is the prefix of the function with serviceID. + FaaSServicePrefix = "0@" + + // TenantProductSplitStr separator between a tenant and a product + TenantProductSplitStr = "@" + + minEleSize = 3 +) + +// ComplexFuncName contains service ID and raw function name +type ComplexFuncName struct { + prefix string + ServiceID string + FuncName string +} + +// NewComplexFuncName - +func NewComplexFuncName(svcID, funcName string) *ComplexFuncName { + return &ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: svcID, + FuncName: funcName, + } +} + +// ParseFrom parse ComplexFuncName from string +func (c *ComplexFuncName) ParseFrom(name string) *ComplexFuncName { + fields := strings.Split(name, Separator) + if len(fields) < minEleSize || fields[0] != ServiceIDPrefix { + c.prefix = "" + c.ServiceID = "" + c.FuncName = name + return c + } + idx := 0 + c.prefix = fields[idx] + idx++ + c.ServiceID = fields[idx] + // $prefix$separator$ServiceID$separator$FuncName equals name + c.FuncName = name[(len(c.prefix) + len(Separator) + len(c.ServiceID) + len(Separator)):] + return c +} + +// ParseFromFaaS parse ComplexFuncName from string +func (c *ComplexFuncName) ParseFromFaaS(name string) *ComplexFuncName { + fields := strings.Split(name, DefaultFaaSSeparator) + if len(fields) < minEleSize || fields[0] != ServiceIDPrefix { + c.prefix = "" + c.ServiceID = "" + c.FuncName = name + return c + } + idx := 0 + c.prefix = fields[idx] + idx++ + c.ServiceID = fields[idx] + // $prefix$separator$ServiceID$separator$FuncName equals name + c.FuncName = name[(len(c.prefix) + len(DefaultFaaSSeparator) + len(c.ServiceID) + len(DefaultFaaSSeparator)):] + return c +} + +// String - +func (c *ComplexFuncName) String() string { + return strings.Join([]string{c.prefix, c.ServiceID, c.FuncName}, Separator) +} + +// FaaSString - +func (c *ComplexFuncName) FaaSString() string { + return strings.Join([]string{c.prefix, c.ServiceID, c.FuncName}, DefaultFaaSSeparator) +} + +// GetSvcIDWithPrefix get serviceID with prefix from function name +func (c *ComplexFuncName) GetSvcIDWithPrefix() string { + return c.prefix + Separator + c.ServiceID +} + +// SetSeparator - +func SetSeparator(separator string) { + if separator != "" { + Separator = separator + } +} + +// GetPureFaaSFunctionName get pure functionName from complexFuncName, eq: 0@service@functionName +func GetPureFaaSFunctionName(complexFuncName string) string { + c := &ComplexFuncName{} + c.ParseFromFaaS(complexFuncName) + return c.FuncName +} + +// GetPureFaaSService get pure service from complexFuncName, eq: 0@service@functionName +func GetPureFaaSService(complexFuncName string) string { + c := &ComplexFuncName{} + c.ParseFromFaaS(complexFuncName) + return c.ServiceID +} + +// ComplexFaaSFuncName complex funcName for faaS function +func ComplexFaaSFuncName(serviceName, functionName string) string { + name := functionName + if serviceName != "" { + c := NewComplexFuncName(serviceName, functionName) + name = c.FaaSString() + } + return name +} + +// GetFunctionVersion - +func GetFunctionVersion(kind string) string { + if kind == constants.Faas { + return constants.DefaultLatestFaaSVersion + } + return constants.DefaultLatestVersion +} diff --git a/functionsystem/apps/meta_service/common/urnutils/gadgets_test.go b/functionsystem/apps/meta_service/common/urnutils/gadgets_test.go new file mode 100644 index 0000000000000000000000000000000000000000..764726f6cb3f39d5937e846dff7964f714d61782 --- /dev/null +++ b/functionsystem/apps/meta_service/common/urnutils/gadgets_test.go @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package urnutils + +import ( + "reflect" + "testing" + + "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" +) + +func TestComplexFuncName_GetSvcIDWithPrefix(t *testing.T) { + tests := []struct { + name string + fields ComplexFuncName + want string + }{ + { + name: "normal", + fields: ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFuncName", + }, + want: "0-absserviceid", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ComplexFuncName{ + prefix: tt.fields.prefix, + ServiceID: tt.fields.ServiceID, + FuncName: tt.fields.FuncName, + } + if got := c.GetSvcIDWithPrefix(); got != tt.want { + t.Errorf("GetSvcIDWithPrefix() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestComplexFuncName_ParseFrom(t *testing.T) { + type args struct { + name string + } + tests := []struct { + name string + args args + want *ComplexFuncName + }{ + { + name: "normal", + args: args{ + name: "0:absserviceid:absFunc:Name", + }, + want: &ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFunc:Name", + }, + }, + } + SetSeparator(":") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ComplexFuncName{} + if got := c.ParseFrom(tt.args.name); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseFrom() = %v, want %v", got, tt.want) + } + }) + } + SetSeparator("-") + + pName := "1:serviceId:absFunc:Name" + cName := &ComplexFuncName{} + cName.ParseFrom(pName) + assert.Equal(t, pName, cName.FuncName) +} + +func TestComplexFuncName_String(t *testing.T) { + tests := []struct { + name string + fields ComplexFuncName + want string + }{ + { + name: "normal", + fields: ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFunc-Name", + }, + want: "0-absserviceid-absFunc-Name", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ComplexFuncName{ + prefix: tt.fields.prefix, + ServiceID: tt.fields.ServiceID, + FuncName: tt.fields.FuncName, + } + if got := c.String(); got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewComplexFuncName(t *testing.T) { + type args struct { + svcID string + funcName string + } + tests := []struct { + name string + args args + want *ComplexFuncName + }{ + { + name: "normal", + args: args{ + svcID: "absserviceid", + funcName: "absFunc-Name", + }, + want: &ComplexFuncName{ + prefix: ServiceIDPrefix, + ServiceID: "absserviceid", + FuncName: "absFunc-Name", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewComplexFuncName(tt.args.svcID, tt.args.funcName); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewComplexFuncName() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetPureFaaSFunctionName(t *testing.T) { + convey.Convey("test GetPureFaaSFunctionName", t, func() { + funcName := GetPureFaaSFunctionName("0@yrservice@yrfunc") + convey.So(funcName, convey.ShouldEqual, "yrfunc") + }) +} + +func TestGetPureFaaSService(t *testing.T) { + convey.Convey("test GetPureFaaSService", t, func() { + funcSvc := GetPureFaaSService("0@yrservice@yrfunc") + convey.So(funcSvc, convey.ShouldEqual, "yrservice") + }) +} + +func TestComplexFaaSFuncName(t *testing.T) { + convey.Convey("test ComplexFaaSFuncName", t, func() { + funcName := ComplexFaaSFuncName("yrservice", "yrfunc") + convey.So(funcName, convey.ShouldEqual, "0@yrservice@yrfunc") + }) +} + +func TestGetFunctionVersion(t *testing.T) { + convey.Convey("test GetFunctionVersion", t, func() { + version := GetFunctionVersion("faas") + convey.So(version, convey.ShouldEqual, "latest") + version = GetFunctionVersion("yr-lib") + convey.So(version, convey.ShouldEqual, "$latest") + }) +} diff --git a/functionsystem/apps/meta_service/common/urnutils/urn_utils.go b/functionsystem/apps/meta_service/common/urnutils/urn_utils.go new file mode 100644 index 0000000000000000000000000000000000000000..8134b6a033465a398cef7c3ace79d62811269a6f --- /dev/null +++ b/functionsystem/apps/meta_service/common/urnutils/urn_utils.go @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package urnutils contains URN element definitions and tools +package urnutils + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "meta_service/common/functioncapability" + "meta_service/common/logger/log" +) + +// An example of a function URN: :::::: +// Indices of elements in BaseURN +const ( + // ProductIDIndex is the index of the product ID in a URN + ProductIDIndex = iota + // RegionIDIndex is the index of the region ID in a URN + RegionIDIndex + // BusinessIDIndex is the index of the business ID in a URN + BusinessIDIndex + // TenantIDIndex is the index of the tenant ID in a URN + TenantIDIndex + // FunctionSignIndex is the index of the product ID in a URN + FunctionSignIndex + // FunctionNameIndex is the index of the product name in a URN + FunctionNameIndex + // VersionIndex is the index of the version in a URN + VersionIndex + // URNLenWithVersion is the normal URN length with a version + URNLenWithVersion +) + +// An example of a function functionkey: // +const ( + // TenantIDIndex is the index of the tenant ID in a functionkey + TenantIDIndexKey = iota +) + +const ( + urnLenWithoutVersion = URNLenWithVersion - 1 + // URNSep is a URN separator of functions + URNSep = ":" + // FunctionKeySep is a functionkey separator of functions + FunctionKeySep = "/" + // DefaultURNProductID is the default product ID of a URN + DefaultURNProductID = "sn" + // DefaultURNRegion is the default region of a URN + DefaultURNRegion = "cn" + // DefaultURNFuncSign is the default function sign of a URN + DefaultURNFuncSign = "function" + defaultURNLayerSign = "layer" + // DefaultURNVersion is the default version of a URN + DefaultURNVersion = "$latest" + anonymization = "****" + anonymizeLen = 3 + + // BranchAliasPrefix is used to remove "!" from aliasing rules at the begining of "!" + BranchAliasPrefix = 1 + // BranchAliasRule is an aliased rule that begins with an "!" + BranchAliasRule = "!" + functionNameStartIndex = 2 + // ServiceNameIndex is index of service name in urn + ServiceNameIndex = 1 + funcNameMinLen = 3 + // DefaultFunctionMaxLen is max length of function name + DefaultFunctionMaxLen = 63 +) + +var functionGraphFuncNameRegexp = regexp.MustCompile("^[a-zA-Z]([a-zA-Z0-9_-]*[a-zA-Z0-9])?$") + +// BaseURN contains elements of a product URN. It can expand to FunctionURN, LayerURN and WorkerURN +type BaseURN struct { + ProductID string + RegionID string + BusinessID string + TenantID string + TypeSign string + Name string + Version string +} + +// String serializes elements of function URN struct to string +func (p *BaseURN) String() string { + urn := fmt.Sprintf("%s:%s:%s:%s:%s:%s", p.ProductID, p.RegionID, + p.BusinessID, p.TenantID, p.TypeSign, p.Name) + if p.Version != "" { + return fmt.Sprintf("%s:%s", urn, p.Version) + } + return urn +} + +// ParseFrom parses elements from a function URN +func (p *BaseURN) ParseFrom(urn string) error { + elements := strings.Split(urn, URNSep) + urnLen := len(elements) + if urnLen < urnLenWithoutVersion || urnLen > URNLenWithVersion { + return fmt.Errorf("failed to parse urn from: %s, invalid length: %d", urn, urnLen) + } + p.ProductID = elements[ProductIDIndex] + p.RegionID = elements[RegionIDIndex] + p.BusinessID = elements[BusinessIDIndex] + p.TenantID = elements[TenantIDIndex] + p.TypeSign = elements[FunctionSignIndex] + p.Name = elements[FunctionNameIndex] + if urnLen == URNLenWithVersion { + p.Version = elements[VersionIndex] + } + return nil +} + +// StringWithoutVersion return string without version +func (p *BaseURN) StringWithoutVersion() string { + return fmt.Sprintf("%s:%s:%s:%s:%s:%s", p.ProductID, p.RegionID, + p.BusinessID, p.TenantID, p.TypeSign, p.Name) +} + +// GetFunctionInfo collects function information from a URN +func GetFunctionInfo(urn string) (BaseURN, error) { + var parsedURN BaseURN + if err := parsedURN.ParseFrom(urn); err != nil { + log.GetLogger().Errorf("error while parsing an URN: %s", err.Error()) + return BaseURN{}, errors.New("parsing an URN error") + } + return parsedURN, nil +} + +// GetFuncInfoWithVersion collects function information and distinguishes if the URN contains a version +func GetFuncInfoWithVersion(urn string) (BaseURN, error) { + parsedURN, err := GetFunctionInfo(urn) + if err != nil { + return parsedURN, err + } + if parsedURN.Version == "" { + log.GetLogger().Errorf("incorrect URN length: %s", Anonymize(urn)) + return parsedURN, errors.New("incorrect URN length, no version") + } + return parsedURN, nil +} + +// ParseAliasURN is used to remove "!" from the beginning of the alias +func ParseAliasURN(aliasURN string) string { + elements := strings.Split(aliasURN, URNSep) + if len(elements) == URNLenWithVersion { + if strings.HasPrefix(elements[VersionIndex], BranchAliasRule) { + elements[VersionIndex] = elements[VersionIndex][BranchAliasPrefix:] + } + return strings.Join(elements, ":") + } + return aliasURN +} + +// GetAlias returns an alias +func (p *BaseURN) GetAlias() string { + if p.Version == DefaultURNVersion { + return "" + } + if _, err := strconv.Atoi(p.Version); err == nil { + return "" + } + return p.Version +} + +// GetAliasForFuncBranch returns an alias for function branch +func (p *BaseURN) GetAliasForFuncBranch() string { + if strings.HasPrefix(p.Version, BranchAliasRule) { + // remove "!" from the beginning of the alias + return p.Version[BranchAliasPrefix:] + } + return "" +} + +// Valid check whether the self-verification function name complies with the specifications. +func (p *BaseURN) Valid(functionCapability int) error { + serviceID, functionName, err := GetFunctionNameAndServiceName(p.Name) + if err != nil { + log.GetLogger().Errorf("failed to get serviceID and functionName") + return err + } + if functionCapability == functioncapability.Fusion { + if !(functionGraphFuncNameRegexp.MatchString(serviceID) || + functionGraphFuncNameRegexp.MatchString(functionName)) { + errmsg := "failed to match reg%s" + log.GetLogger().Errorf(errmsg, functionGraphFuncNameRegexp) + return fmt.Errorf(errmsg, functionGraphFuncNameRegexp) + } + if len(serviceID) > DefaultFunctionMaxLen || len(functionName) > DefaultFunctionMaxLen { + errmsg := "serviceID or functionName's len is out of range %d" + log.GetLogger().Errorf(errmsg, DefaultFunctionMaxLen) + return fmt.Errorf(errmsg, DefaultFunctionMaxLen) + } + } + return nil +} + +// GetFunctionNameAndServiceName returns serviceName and FunctionName +func GetFunctionNameAndServiceName(funcName string) (string, string, error) { + if strings.HasPrefix(funcName, ServiceIDPrefix) { + split := strings.Split(funcName, Separator) + if len(split) < funcNameMinLen { + log.GetLogger().Errorf("incorrect function name length: %s", len(split)) + return "", "", errors.New("parsing a function name error") + } + return split[ServiceNameIndex], strings.Join(split[functionNameStartIndex:], Separator), nil + } + log.GetLogger().Errorf("incorrect function name: %s", funcName) + return "", "", errors.New("parsing a function name error") +} + +// Anonymize anonymize input str to xxx****xxx +func Anonymize(str string) string { + if len(str) < anonymizeLen+1+anonymizeLen { + return anonymization + } + return str[:anonymizeLen] + anonymization + str[len(str)-anonymizeLen:] +} + +// AnonymizeTenantURN Anonymize tenant info in urn +func AnonymizeTenantURN(urn string) string { + elements := strings.Split(urn, URNSep) + urnLen := len(elements) + if urnLen < urnLenWithoutVersion || urnLen > URNLenWithVersion { + return urn + } + elements[TenantIDIndex] = Anonymize(elements[TenantIDIndex]) + return strings.Join(elements, URNSep) +} + +// AnonymizeTenantKey Anonymize tenant info in functionkey +func AnonymizeTenantKey(functionKey string) string { + elements := strings.Split(functionKey, FunctionKeySep) + keyLen := len(elements) + if TenantIDIndexKey >= keyLen { + return functionKey + } + elements[TenantIDIndexKey] = Anonymize(elements[TenantIDIndexKey]) + return strings.Join(elements, FunctionKeySep) +} + +// AnonymizeTenantURNSlice Anonymize tenant info in urn slice +func AnonymizeTenantURNSlice(urns []string) []string { + var anonymizeUrns []string + for i := 0; i < len(urns); i++ { + anonymizeUrn := AnonymizeTenantURN(urns[i]) + anonymizeUrns = append(anonymizeUrns, anonymizeUrn) + } + return anonymizeUrns +} diff --git a/functionsystem/apps/meta_service/common/urnutils/urn_utils_test.go b/functionsystem/apps/meta_service/common/urnutils/urn_utils_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3cf550ca82c87a889016503b1f1d58eb29e04ef8 --- /dev/null +++ b/functionsystem/apps/meta_service/common/urnutils/urn_utils_test.go @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package urnutils + +import ( + "reflect" + "testing" + + "meta_service/common/functioncapability" + + "github.com/stretchr/testify/assert" +) + +func TestProductUrn_ParseFrom(t *testing.T) { + absURN := BaseURN{ + "absPrefix", + "absZone", + "absBusinessID", + "absTenantID", + "absProductID", + "absName", + "latest", + } + absURNStr := "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest" + type args struct { + urn string + } + tests := []struct { + name string + fields BaseURN + args args + want BaseURN + }{ + { + name: "normal test", + args: args{ + absURNStr, + }, + want: absURN, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &BaseURN{} + if _ = p.ParseFrom(tt.args.urn); !reflect.DeepEqual(*p, tt.want) { + t.Errorf("ParseFrom() p = %v, want %v", *p, tt.want) + } + }) + } +} + +func TestProductUrn_String(t *testing.T) { + tests := []struct { + name string + fields BaseURN + want string + }{ + { + "stringify with version", + BaseURN{ + "absPrefix", + "absZone", + "absBusinessID", + "absTenantID", + "absProductID", + "absName", + "latest", + }, + "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest", + }, + { + "stringify without version", + BaseURN{ + ProductID: "absPrefix", + RegionID: "absZone", + BusinessID: "absBusinessID", + TenantID: "absTenantID", + TypeSign: "absProductID", + Name: "absName", + }, + "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &BaseURN{ + ProductID: tt.fields.ProductID, + RegionID: tt.fields.RegionID, + BusinessID: tt.fields.BusinessID, + TenantID: tt.fields.TenantID, + TypeSign: tt.fields.TypeSign, + Name: tt.fields.Name, + Version: tt.fields.Version, + } + if got := p.String(); got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestProductUrn_StringWithoutVersion(t *testing.T) { + tests := []struct { + name string + fields BaseURN + want string + }{ + { + "stringify without version", + BaseURN{ + "absPrefix", + "absZone", + "absBusinessID", + "absTenantID", + "absProductID", + "absName", + "latest", + }, + "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &BaseURN{ + ProductID: tt.fields.ProductID, + RegionID: tt.fields.RegionID, + BusinessID: tt.fields.BusinessID, + TenantID: tt.fields.TenantID, + TypeSign: tt.fields.TypeSign, + Name: tt.fields.Name, + Version: tt.fields.Version, + } + if got := p.StringWithoutVersion(); got != tt.want { + t.Errorf("StringWithoutVersion() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnonymize(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"0", anonymization}, + {"123", anonymization}, + {"123456", anonymization}, + {"1234567", "123****567"}, + {"12345678901234546", "123****546"}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, Anonymize(tt.input)) + } +} + +func TestAnonymizeTenantURN(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName", "absPrefix:absZone:absBusinessID:abs****tID:absProductID:absName"}, + {"absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest", "absPrefix:absZone:absBusinessID:abs****tID:absProductID:absName:latest"}, + {"a:b:c", "a:b:c"}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, AnonymizeTenantURN(tt.input)) + } +} + +func TestBaseURN_Valid(t *testing.T) { + Separator = "@" + functionCapability := functioncapability.Fusion + urn := BaseURN{ + ProductID: "", + RegionID: "", + BusinessID: "", + TenantID: "", + TypeSign: "", + Name: "0@a_-9AA@AA", + Version: "", + } + success := urn.Valid(functionCapability) + assert.Equal(t, nil, success) + + urn.Name = "0@a_-9AA@ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt" + success = urn.Valid(functionCapability) + assert.Equal(t, nil, success) + + urn.Name = "0@a_-9AA@tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt" + err := urn.Valid(functionCapability) + assert.NotEqual(t, nil, err) + + urn.Name = "@func" + err = urn.Valid(functionCapability) + assert.NotEqual(t, nil, err) + + urn.Name = "0@func" + err = urn.Valid(functionCapability) + assert.NotEqual(t, nil, err) + + urn.Name = "0@^@^" + err = urn.Valid(functionCapability) + assert.NotEqual(t, nil, err) + + Separator = "-" +} + +func TestBaseURN_GetAlias(t *testing.T) { + urn := BaseURN{ + ProductID: "", + RegionID: "", + BusinessID: "", + TenantID: "", + TypeSign: "", + Name: "0@a_-9AA@AA", + Version: DefaultURNVersion, + } + + alias := urn.GetAlias() + assert.Equal(t, "", alias) + + urn.Version = "old" + alias = urn.GetAlias() + assert.Equal(t, "old", alias) +} + +func TestGetFuncInfoWithVersion(t *testing.T) { + urn := "urn" + _, err := GetFuncInfoWithVersion(urn) + assert.NotEqual(t, nil, err) + + urn = "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName" + _, err = GetFuncInfoWithVersion(urn) + assert.NotEqual(t, nil, err) + + urn = "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest" + parsedURN, err := GetFuncInfoWithVersion(urn) + assert.Equal(t, "absName", parsedURN.Name) +} + +func TestAnonymizeTenantKey(t *testing.T) { + inputKey := "" + outputKey := AnonymizeTenantKey(inputKey) + assert.Equal(t, "****", outputKey) + + inputKey = "input/key" + outputKey = AnonymizeTenantKey(inputKey) + assert.Equal(t, "****/key", outputKey) +} + +func TestParseAliasURN(t *testing.T) { + urn := "" + alias := ParseAliasURN(urn) + assert.Equal(t, urn, alias) + + urn = "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:!latest" + alias = ParseAliasURN(urn) + assert.Equal(t, "absPrefix:absZone:absBusinessID:absTenantID:absProductID:absName:latest", alias) +} + +func TestAnonymizeTenantURNSlice(t *testing.T) { + inUrn := []string{"in", "in/urn"} + outUrn := AnonymizeTenantURNSlice(inUrn) + assert.Equal(t, "in", outUrn[0]) + assert.Equal(t, "in/urn", outUrn[1]) +} + +func TestBaseURN_GetAliasForFuncBranch(t *testing.T) { + urn := BaseURN{ + ProductID: "", + RegionID: "", + BusinessID: "", + TenantID: "", + TypeSign: "", + Name: "0@a_-9AA@AA", + Version: "!latest", + } + + alias := urn.GetAliasForFuncBranch() + assert.Equal(t, "latest", alias) + + urn.Version = "latest" + alias = urn.GetAliasForFuncBranch() + assert.Equal(t, "", alias) +} diff --git a/functionsystem/apps/meta_service/common/utils/helper.go b/functionsystem/apps/meta_service/common/utils/helper.go new file mode 100644 index 0000000000000000000000000000000000000000..6d2b674eb46a983113d5b8edcf7f661807977cb2 --- /dev/null +++ b/functionsystem/apps/meta_service/common/utils/helper.go @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "meta_service/common/reader" +) + +const ( + defaultPath = "/home/sn" + defaultBinPath = "/home/sn/bin" + defaultConfigPath = "/home/sn/config/config.json" + defaultLogConfigPath = "/home/sn/config/log.json" + DefaultFunctionPath = "/home/sn/config/function.yaml" +) + +// IsFile returns true if the path is a file +func IsFile(path string) bool { + file, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return false + } + return file.Mode().IsRegular() +} + +// IsDir returns true if the path is a dir +func IsDir(path string) bool { + dir, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return false + } + + return dir.IsDir() +} + +// FileExists returns true if the path exists +func FileExists(path string) bool { + _, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return false + } + return true +} + +// FileSize return path file size +func FileSize(path string) int64 { + fileInfo, err := reader.ReadFileInfoWithTimeout(path) + if err != nil { + return 0 + } + return fileInfo.Size() +} + +// IsHexString judge If Hex String +func IsHexString(str string) bool { + str = strings.ToLower(str) + + for _, c := range str { + if c < '0' || (c > '9' && c < 'a') || c > 'f' { + return false + } + } + + return true +} + +// ValidateFilePath verify the legitimacy of the file path +func ValidateFilePath(path string) error { + absPath, err := filepath.Abs(path) + if err != nil || !strings.HasPrefix(path, absPath) { + return errors.New("invalid file path, expect to be configured as an absolute path") + } + return nil +} + +// GetBinPath get path of exec bin file +func GetBinPath() (string, error) { + bin, err := os.Executable() + if err != nil { + return "", err + } + binPath := filepath.Dir(bin) + return binPath, nil +} + +// GetConfigPath get config.json file path +func GetConfigPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return defaultConfigPath, nil + } + return binPath + "/../config/config.json", nil +} + +// GetFunctionConfigPath get function.yaml file path +func GetFunctionConfigPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return DefaultFunctionPath, nil + } + return binPath + "/../config/function.yaml", nil +} + +// GetLogConfigPath get log.json file path +func GetLogConfigPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return defaultLogConfigPath, nil + } + return binPath + "/../config/log.json", nil +} + +// GetDefaultPath get default path +func GetDefaultPath() (string, error) { + binPath, err := GetBinPath() + if err != nil { + return "", err + } + if binPath == defaultBinPath { + return defaultPath, nil + } + return binPath + "/..", nil +} diff --git a/functionsystem/apps/meta_service/common/utils/helper_test.go b/functionsystem/apps/meta_service/common/utils/helper_test.go new file mode 100644 index 0000000000000000000000000000000000000000..4707d25245a4c6916f2b4fbb794a6dd995e3710a --- /dev/null +++ b/functionsystem/apps/meta_service/common/utils/helper_test.go @@ -0,0 +1,394 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + . "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/suite" +) + +type IsFileTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *IsFileTestSuite) SetupSuite() { + var err error + + // Create temp dir for IsFileTestSuite + suite.tempDir, err = ioutil.TempDir("", "isfile-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *IsDirTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *IsFileTestSuite) TestPositive() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function isFile() returns true when file is created + suite.Require().True(IsFile(tempFile.Name())) + +} + +// TestFileIsNotExist Test File Is Not Exist +func (suite *IsFileTestSuite) TestFileIsNotExist() { + + // Set path to unexisted file + tempFile := filepath.Join(suite.tempDir, "somePath.txt") + + // Verify that function isFile() returns false when file doesn't exist in the system + suite.Require().False(IsFile(tempFile)) +} + +// TestFileIsADirectory Test File Is A Directory +func (suite *IsFileTestSuite) TestFileIsADirectory() { + suite.Require().False(IsFile(suite.tempDir)) +} + +type IsDirTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *IsDirTestSuite) SetupSuite() { + var err error + + // Create temp dir for IsDirTestSuite + suite.tempDir, err = ioutil.TempDir("", "isdir-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *IsFileTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *IsDirTestSuite) TestPositive() { + + // Verify that function IsDir() returns true when directory exists in the system + suite.Require().True(IsDir(suite.tempDir)) +} + +// TestNegative Test Negative +func (suite *IsDirTestSuite) TestNegative() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function IsDir( returns false when file instead of directory is function argument + suite.Require().False(IsDir(tempFile.Name())) +} + +type FileExistTestSuite struct { + suite.Suite + tempDir string +} + +// SetupSuite Setup Suite +func (suite *FileExistTestSuite) SetupSuite() { + var err error + + // Create temp dir for FileExistTestSuite + suite.tempDir, err = ioutil.TempDir("", "file_exists-test") + suite.Require().NoError(err) +} + +// TearDownSuite TearDown Suite +func (suite *FileExistTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.tempDir) +} + +// TestPositive Test Positive +func (suite *FileExistTestSuite) TestPositive() { + + // Create temp file + tempFile, err := ioutil.TempFile(suite.tempDir, "temp_file") + suite.Require().NoError(err) + defer os.Remove(tempFile.Name()) + + // Verify that function FileExists() returns true when file is exist + suite.Require().True(FileExists(tempFile.Name())) +} + +// TestFileNotExist Test File Not Exist +func (suite *FileExistTestSuite) TestFileNotExist() { + + // Set path to unexisted file + tempFile := filepath.Join(suite.tempDir, "somePath.txt") + + // Verify that function FileExists() returns false when file doesn't exist + suite.Require().False(FileExists(tempFile)) +} + +// TestFileIsNotAFile Test File Is Not A File +func (suite *FileExistTestSuite) TestFileIsNotAFile() { + + // Verify that function returns true when folder is exist in the system + suite.Require().True(FileExists(suite.tempDir)) +} + +// TestHelperTestSuite Test Helper Test Suite +func TestHelperTestSuite(t *testing.T) { + suite.Run(t, new(FileExistTestSuite)) + suite.Run(t, new(IsDirTestSuite)) + suite.Run(t, new(IsFileTestSuite)) +} + +// TestHelperTestSuite Test Helper Test Suite +func TestGetDefaultPath(t *testing.T) { + Convey("test get defaultpath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetDefaultPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/home/sn") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test get defaultpath 1 ", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetDefaultPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/..") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test get defaultpath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetDefaultPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +// TestGetFunctionConfigPath - +func TestGetFunctionConfigPath(t *testing.T) { + Convey("test GetFunctionConfigPath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetFunctionConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/../config/function.yaml") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetFunctionConfigPath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetFunctionConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, DefaultFunctionPath) + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetFunctionConfigPath 3", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetFunctionConfigPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +// TestGetLogConfigPath - +func TestGetLogConfigPath(t *testing.T) { + Convey("test GetLogConfigPath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetLogConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/../config/log.json") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetLogConfigPath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetLogConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, defaultLogConfigPath) + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetLogConfigPath 3", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetLogConfigPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +// TestGetConfigPath - +func TestGetConfigPath(t *testing.T) { + Convey("test GetConfigPath", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return "/opt", nil + }), + } + path, res := GetConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, "/opt/../config/config.json") + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetConfigPath 2", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, nil + }), + } + path, res := GetConfigPath() + So(res, ShouldEqual, nil) + So(path, ShouldEqual, defaultConfigPath) + for i := range patches { + patches[i].Reset() + } + }) + Convey("test GetConfigPath 3", t, func() { + patches := [...]*Patches{ + ApplyFunc(GetBinPath, func() (string, error) { + return defaultBinPath, errors.New("failed") + }), + } + path, res := GetConfigPath() + So(res, ShouldNotEqual, nil) + So(res.Error(), ShouldEqual, "failed") + So(path, ShouldEqual, "") + for i := range patches { + patches[i].Reset() + } + }) +} + +func TestGetBinPath(t *testing.T) { + GetBinPath() +} + +func TestGetResourcePath(t *testing.T) { + GetResourcePath() + + os.Setenv("ResourcePath", "") + GetResourcePath() + + patch := ApplyFunc(exec.LookPath, func(string) (string, error) { + return "", errors.New("test") + }) + GetResourcePath() + patch.Reset() + + patch = ApplyFunc(filepath.Abs, func(string) (string, error) { + return "", errors.New("test") + }) + GetResourcePath() + fmt.Println() + patch.Reset() +} + +func TestIsDir(t *testing.T) { + IsDir("") +} + +func TestGetServicesPath(t *testing.T) { + GetServicesPath() + os.Setenv("ServicesPath", "") + GetServicesPath() + + patch := ApplyFunc(exec.LookPath, func(string) (string, error) { + return "", errors.New("test") + }) + GetServicesPath() + patch.Reset() + + patch = ApplyFunc(filepath.Abs, func(string) (string, error) { + return "", errors.New("test") + }) + GetServicesPath() + fmt.Println() + patch.Reset() +} diff --git a/functionsystem/apps/meta_service/common/utils/net_helper.go b/functionsystem/apps/meta_service/common/utils/net_helper.go new file mode 100644 index 0000000000000000000000000000000000000000..302f44a53bda4b210de546f0bb82d230bd95f712 --- /dev/null +++ b/functionsystem/apps/meta_service/common/utils/net_helper.go @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "errors" + "fmt" + "net" + "os" + "runtime" + "strconv" + "strings" + "syscall" + + "meta_service/common/constants" +) + +const ( + normalExitCode = 1 + AddressAlreadyUsedExitCode = 98 + WSAEADDRINUSE = 10048 + addressLen = 2 + ipIndex = 0 + portIndex = 1 +) + +// ProcessBindErrorAndExit will deal with err type +func ProcessBindErrorAndExit(err error) { + fmt.Printf("failed to listen address, err: %s", err.Error()) + if isErrorAddressAlreadyInUse(err) { + os.Exit(AddressAlreadyUsedExitCode) + } + os.Exit(normalExitCode) +} + +func isErrorAddressAlreadyInUse(err error) bool { + var eOsSyscall *os.SyscallError + if !errors.As(err, &eOsSyscall) { + return false + } + var errErrno syscall.Errno + if !errors.As(eOsSyscall, &errErrno) { + return false + } + if errErrno == syscall.EADDRINUSE { + return true + } + if runtime.GOOS == "windows" && errErrno == WSAEADDRINUSE { + return true + } + return false +} + +// CheckAddress check whether the address is valid +func CheckAddress(addr string) bool { + addrArg := strings.Split(addr, ":") + if len(addrArg) != addressLen { + return false + } + ip := net.ParseIP(addrArg[ipIndex]) + if ip == nil { + return false + } + port, err := strconv.Atoi(addrArg[portIndex]) + if err != nil { + return false + } + if port < 0 || port > constants.MaxPort { + return false + } + return true +} diff --git a/functionsystem/apps/meta_service/common/utils/net_helper_test.go b/functionsystem/apps/meta_service/common/utils/net_helper_test.go new file mode 100644 index 0000000000000000000000000000000000000000..28db96420bf535ed074ccf2ebde60a387c05f254 --- /dev/null +++ b/functionsystem/apps/meta_service/common/utils/net_helper_test.go @@ -0,0 +1,73 @@ +package utils + +import ( + "errors" + "net" + "os" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" +) + +func TestIsErrorAddressAlreadyInUse(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:55555") + defer func(listener net.Listener) { + _ = listener.Close() + }(listener) + assert.Nil(t, err) + listener2, err := net.Listen("tcp", "127.0.0.1:55555") + assert.True(t, isErrorAddressAlreadyInUse(err)) + if err == nil { + _ = listener2.Close() + } +} + +func TestProcessBindErrorAndExit(t *testing.T) { + flag := false + patches := gomonkey.ApplyFunc(isErrorAddressAlreadyInUse, func(err error) bool { + return flag + }).ApplyFunc(os.Exit, func(code int) { + return + }) + defer patches.Reset() + ProcessBindErrorAndExit(errors.New("mock err")) + flag = true + ProcessBindErrorAndExit(errors.New("mock err2")) +} + +func TestCheckAddress(t *testing.T) { + test := []struct { + addr string + wanted bool + }{ + { + addr: "111", + wanted: false, + }, + { + addr: "asdasd:asdasd", + wanted: false, + }, + { + addr: "127.0.0.1:asd", + wanted: false, + }, + { + addr: "127.0.0.1:994651", + wanted: false, + }, + { + addr: "127.0.0.1:6379", + wanted: true, + }, + } + for _, tt := range test { + res := CheckAddress(tt.addr) + if tt.wanted { + assert.True(t, res) + } else { + assert.False(t, res) + } + } +} diff --git a/functionsystem/apps/meta_service/common/utils/resourcepath.go b/functionsystem/apps/meta_service/common/utils/resourcepath.go new file mode 100644 index 0000000000000000000000000000000000000000..7f0fe35c913c8df8dd759fb9e29ad684f01d6016 --- /dev/null +++ b/functionsystem/apps/meta_service/common/utils/resourcepath.go @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" +) + +// GetResourcePath Get Resource Path +func GetResourcePath() string { + return getPath("ResourcePath", "resource") +} + +// GetServicesPath Get Services Path +func GetServicesPath() string { + return getPath("ServicesPath", "service-config") +} + +func getPath(env, defaultPath string) string { + envPath := os.Getenv(env) + if envPath == "" { + var err error + cliPath, err := exec.LookPath(os.Args[0]) + if err != nil { + return envPath + } + envPath, err = filepath.Abs(filepath.Dir(cliPath)) + // do not return this error + if err != nil { + fmt.Printf("GetResourcePath abs filepath dir error") + } + envPath = strings.Replace(envPath, "\\", "/", -1) + envPath = path.Join(path.Dir(envPath), defaultPath) + } else { + envPath = strings.Replace(envPath, "\\", "/", -1) + } + + return envPath +} diff --git a/functionsystem/apps/meta_service/common/utils/tools.go b/functionsystem/apps/meta_service/common/utils/tools.go new file mode 100644 index 0000000000000000000000000000000000000000..e795da435d683586abd680142719b763dd8792a5 --- /dev/null +++ b/functionsystem/apps/meta_service/common/utils/tools.go @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils for common functions +package utils + +import ( + "bufio" + "encoding/binary" + "io" + "math" + "meta_service/common/constants" + "meta_service/common/reader" + "net" + "os" + "reflect" + "strconv" + "strings" + "unsafe" + + "github.com/pborman/uuid" +) + +const ( + // OriginDefaultTimeout is 900 + OriginDefaultTimeout = 900 + // maxTimeout is 100 days + maxTimeout = 100 * 24 * 3600 + bytesToMb = 1024 * 1024 + uint64ArrayLength = 8 +) + +// AzEnv set defaultaz env +func AzEnv() string { + az := os.Getenv(constants.ZoneKey) + if az == "" { + az = constants.DefaultAZ + } + if len(az) > constants.ZoneNameLen { + az = az[0 : constants.ZoneNameLen-1] + } + return az +} + +// Domain2IP convert domain to ip +func Domain2IP(endpoint string) (string, error) { + var host, port string + var err error + host = endpoint + if strings.Contains(endpoint, ":") { + host, port, err = net.SplitHostPort(endpoint) + if err != nil { + return "", err + } + } + if net.ParseIP(host) != nil { + return endpoint, nil + } + ips, err := net.LookupHost(host) + if err != nil { + return "", err + } + if port == "" { + return ips[0], nil + } + return net.JoinHostPort(ips[0], port), nil +} + +// DeepCopy will generate a new copy of original collection type +// currently this function is not recursive so elements will not be deep copied +func DeepCopy(origin interface{}) interface{} { + oriTyp := reflect.TypeOf(origin) + oriVal := reflect.ValueOf(origin) + switch oriTyp.Kind() { + case reflect.Slice: + elemType := oriTyp.Elem() + length := oriVal.Len() + capacity := oriVal.Cap() + newObj := reflect.MakeSlice(reflect.SliceOf(elemType), length, capacity) + reflect.Copy(newObj, oriVal) + return newObj.Interface() + case reflect.Map: + newObj := reflect.MakeMapWithSize(oriTyp, len(oriVal.MapKeys())) + for _, key := range oriVal.MapKeys() { + value := oriVal.MapIndex(key) + newObj.SetMapIndex(key, value) + } + return newObj.Interface() + default: + return nil + } +} + +// ValidateTimeout check timeout +func ValidateTimeout(timeout *int64, defaultTimeout int64) { + if *timeout <= 0 { + *timeout = defaultTimeout + return + } + *timeout = *timeout + defaultTimeout - OriginDefaultTimeout + if *timeout > maxTimeout { + *timeout = maxTimeout + } +} + +// IsDataSystemEnable return the datasystem enable flag +func IsDataSystemEnable() bool { + branch, err := strconv.ParseBool(os.Getenv(constants.DataSystemBranchEnvKey)) + if err != nil { + branch = false + } + return branch +} + +// ClearStringMemory - +func ClearStringMemory(s string) { + bs := *(*[]byte)(unsafe.Pointer(&s)) + ClearByteMemory(bs) +} + +// ClearByteMemory - +func ClearByteMemory(b []byte) { + for i := 0; i < len(b); i++ { + b[i] = 0 + } +} + +// Float64ToByte - +func Float64ToByte(float float64) []byte { + bits := math.Float64bits(float) + bytes := make([]byte, 8) + binary.LittleEndian.PutUint64(bytes, bits) + return bytes +} + +// ByteToFloat64 - +func ByteToFloat64(bytes []byte) float64 { + // bounds check to guarantee safety of function Uint64 + if len(bytes) != uint64ArrayLength { + return 0 + } + bits := binary.LittleEndian.Uint64(bytes) + return math.Float64frombits(bits) +} + +// GetSystemMemoryUsed - +func GetSystemMemoryUsed() float64 { + srcFile, err := os.Open("/sys/fs/cgroup/memory/memory.stat") + if err != nil { + return 0 + } + defer srcFile.Close() + + reader := bufio.NewReader(srcFile) + for { + lineBytes, _, err := reader.ReadLine() + if err == io.EOF { + break + } + + lineStr := string(lineBytes) + if strings.Contains(lineStr, "rss ") { + rssStr := strings.TrimPrefix(lineStr, "rss ") + rssStr = strings.Trim(rssStr, "\n") + + value, err := strconv.ParseInt(rssStr, 10, 64) + if err != nil { + break + } + return float64(value) / bytesToMb + } + } + return 0 +} + +// ExistPath whether path exists +func ExistPath(path string) bool { + _, err := reader.ReadFileInfoWithTimeout(path) + if err != nil && os.IsNotExist(err) { + return false + } + return true +} + +// UniqueID get unique ID +func UniqueID() string { + return uuid.NewRandom().String() +} diff --git a/functionsystem/apps/meta_service/common/utils/tools_test.go b/functionsystem/apps/meta_service/common/utils/tools_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b357289acf6c11877d9fbfb8293797edfbcd8208 --- /dev/null +++ b/functionsystem/apps/meta_service/common/utils/tools_test.go @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "net" + "os" + "testing" + + "meta_service/common/constants" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" +) + +const ( + DefaultTimeout = 900 +) + +// TestDomain2IP convert domain to ip +// If this test case fails, the problem is caused by inline optimization of the Go compiler. +// go test add "-gcflags="all=-N -l",the case will pass +func TestDomain2IP(t *testing.T) { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(net.LookupHost, func(_ string) ([]string, error) { + return []string{"1.1.1.1"}, nil + }), + } + defer func() { + for idx := range patches { + patches[idx].Reset() + } + }() + + type args struct { + endpoint string + } + tests := []struct { + args args + want string + wantErr bool + }{ + { + args{endpoint: "1.1.1.1:9000"}, + "1.1.1.1:9000", + false, + }, + { + args{endpoint: "1.1.1.1"}, + "1.1.1.1", + false, + }, + { + args{endpoint: "test:9000"}, + "1.1.1.1:9000", + false, + }, + { + args{endpoint: "test"}, + "1.1.1.1", + false, + }, + } + for _, tt := range tests { + got, err := Domain2IP(tt.args.endpoint) + if (err != nil) != tt.wantErr { + t.Errorf("Domain2IP() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Domain2IP() got = %v, want %v", got, tt.want) + } + } +} + +func TestFloat64ToByte(t *testing.T) { + value := 123.45 + bytesValue := Float64ToByte(value) + if ByteToFloat64(bytesValue) != value { + t.Errorf("Float64ToByte and ByteToFloat64 failed") + } + + bytes := []byte{'a'} + ByteToFloat64(bytes) +} + +func TestGetSystemMemoryUsed(t *testing.T) { + if GetSystemMemoryUsed() == 0 { + t.Log("GetSystemMemoryUsed is zero") + } +} + +func TestExistPath(t *testing.T) { + path := os.Args[0] + if !ExistPath(path) { + t.Errorf("test path exist true failed, path: %s", path) + } + if ExistPath(path + "abc") { + t.Errorf("test path exist false failed, path: %s", path+"abc") + } +} + +func TestFileSize(t *testing.T) { + ret := FileSize("test/file") + assert.Equal(t, ret, int64(0)) +} + +func TestIsDataSystemEnable(t *testing.T) { + ret := IsDataSystemEnable() + assert.Equal(t, ret, false) + + os.Setenv(constants.DataSystemBranchEnvKey, "t") + ret = IsDataSystemEnable() + assert.Equal(t, ret, true) +} + +func TestUniqueID(t *testing.T) { + uuid1 := UniqueID() + uuid2 := UniqueID() + assert.NotEqual(t, uuid1, uuid2) +} + +func TestDeepCopy(t *testing.T) { + srcString := "" + copyString := DeepCopy(srcString) + assert.Equal(t, nil, copyString) + + srcSlice := make([]int, 3) + copySlice := DeepCopy(srcSlice) + assert.Equal(t, 3, len(copySlice.([]int))) + + srcMap := make(map[int]int) + srcMap[0] = 1 + copyMap := DeepCopy(srcMap) + assert.Equal(t, 1, len(copyMap.(map[int]int))) +} + +func TestValidateTimeout(t *testing.T) { + var timeout int64 + timeout = 0 + ValidateTimeout(&timeout, DefaultTimeout) + assert.Equal(t, int64(DefaultTimeout), timeout) + + timeout = maxTimeout + 1 + ValidateTimeout(&timeout, DefaultTimeout) + assert.Equal(t, int64(maxTimeout), timeout) +} + +func TestAzEnv(t *testing.T) { + azString := AzEnv() + assert.Equal(t, "defaultaz", azString) +} + +func TestClearByteMemory(t *testing.T) { + ClearByteMemory([]byte{'a'}) +} diff --git a/functionsystem/apps/meta_service/common/uuid/uuid.go b/functionsystem/apps/meta_service/common/uuid/uuid.go new file mode 100644 index 0000000000000000000000000000000000000000..75f764dc890fd3eab35e9684ad3a84f96e658084 --- /dev/null +++ b/functionsystem/apps/meta_service/common/uuid/uuid.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package uuid for common functions +package uuid + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/hex" + "errors" + "hash" + "io" + "strings" +) + +const ( + defaultByteNum = 16 + indexFour = 4 + indexSix = 6 + indexEight = 8 + indexNine = 9 + indexTen = 10 + indexThirteen = 13 + indexFourteen = 14 + indexEighteen = 18 + indexNineteen = 19 + indexTwentyThree = 23 + indexTwentyFour = 24 + indexThirtySix = 36 + defaultSHA1BaseVersion = 5 +) + +// RandomUUID - +type RandomUUID [defaultByteNum]byte + +var ( + rander = rand.Reader // random function + // NameSpaceURL Well known namespace IDs and UUIDs + NameSpaceURL, _ = parseUUID("6ba7b811-9dad-11d1-80b4-00c04fd430c8") +) + +// New - +func New() RandomUUID { + return mustUUID(newRandom()) +} + +func mustUUID(uuid RandomUUID, err error) RandomUUID { + if err != nil { + return RandomUUID{} + } + return uuid +} + +func newRandom() (RandomUUID, error) { + return newRandomFromReader(rander) +} + +func newRandomFromReader(r io.Reader) (RandomUUID, error) { + var randomUUID RandomUUID + _, err := io.ReadFull(r, randomUUID[:]) + if err != nil { + return RandomUUID{}, err + } + randomUUID[indexSix] = (randomUUID[indexSix] & 0x0f) | 0x40 // Version 4 + randomUUID[indexEight] = (randomUUID[indexEight] & 0x3f) | 0x80 // Variant is 10 + return randomUUID, nil +} + +// String- +func (uuid RandomUUID) String() string { + var buf [indexThirtySix]byte + encodeHex(buf[:], uuid) + return string(buf[:]) +} + +func encodeHex(dstBuf []byte, uuid RandomUUID) { + hex.Encode(dstBuf, uuid[:indexFour]) + dstBuf[indexEight] = '-' + hex.Encode(dstBuf[indexNine:indexThirteen], uuid[indexFour:indexSix]) + dstBuf[indexThirteen] = '-' + hex.Encode(dstBuf[indexFourteen:indexEighteen], uuid[indexSix:indexEight]) + dstBuf[indexEighteen] = '-' + hex.Encode(dstBuf[indexNineteen:indexTwentyThree], uuid[indexEight:indexTen]) + dstBuf[indexTwentyThree] = '-' + hex.Encode(dstBuf[indexTwentyFour:], uuid[indexTen:]) +} + +// NewSHA1 - +func NewSHA1(space RandomUUID, data []byte) RandomUUID { + return NewHash(sha1.New(), space, data, defaultSHA1BaseVersion) +} + +// NewHash returns a new RandomUUID derived from the hash of space concatenated with +// data generated by h. The hash should be at least 16 byte in length. The +// first 16 bytes of the hash are used to form the RandomUUID. +func NewHash(sha1Hash hash.Hash, space RandomUUID, data []byte, version int) RandomUUID { + sha1Hash.Reset() + if _, err := sha1Hash.Write(space[:]); err != nil { + return RandomUUID{} + } + if _, err := sha1Hash.Write(data); err != nil { + return RandomUUID{} + } + s := sha1Hash.Sum(nil) + var uuid RandomUUID + copy(uuid[:], s) + // Set the version bits in the RandomUUID. + uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4) // The version bits are located at positions 13-15. + // Set the variant bits in the RandomUUID. + uuid[8] = (uuid[8] & 0x3f) | 0x80 // The variant bits are located at positions 8-11 (counting from 0). + return uuid +} + +func parseUUID(uuidStr string) (RandomUUID, error) { + const separator = "-" + + uuidStr = strings.ReplaceAll(uuidStr, separator, "") + + if len(uuidStr) != 32 { // Check if the length of the RandomUUID string is exactly 32 characters (16 bytes). + return RandomUUID{}, errors.New("invalid RandomUUID length") + } + + part1, part2 := uuidStr[:16], uuidStr[16:] // Split the RandomUUID string into two parts, each representing 8 bytes. + + b1, err := hex.DecodeString(part1) + if err != nil { + return RandomUUID{}, err + } + b2, err := hex.DecodeString(part2) + if err != nil { + return RandomUUID{}, err + } + + var uuid RandomUUID + copy(uuid[:8], b1) // Copy the first 8 bytes into the RandomUUID variable. + copy(uuid[8:], b2) // Copy the remaining 8 bytes into the RandomUUID variable. + + return uuid, nil +} diff --git a/functionsystem/apps/meta_service/common/uuid/uuid_test.go b/functionsystem/apps/meta_service/common/uuid/uuid_test.go new file mode 100644 index 0000000000000000000000000000000000000000..bdb450868deeddaab25be59212842a4afb503af2 --- /dev/null +++ b/functionsystem/apps/meta_service/common/uuid/uuid_test.go @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package uuid for common functions +package uuid + +import ( + "testing" +) + +func TestNew(t *testing.T) { + m := make(map[RandomUUID]bool) + for x := 1; x < 32; x++ { + s := New() + if m[s] { + t.Errorf("New returned duplicated RandomUUID %s", s) + } + m[s] = true + } +} + +func TestSHA1(t *testing.T) { + uuid := NewSHA1(NameSpaceURL, []byte("python.org")).String() + want := "7af94e2b-4dd9-50f0-9c9a-8a48519bdef0" + if uuid != want { + t.Errorf("SHA1: got %q expected %q", uuid, want) + } +} + +func Test_parseRandomUUID(t *testing.T) { + type args struct { + uuidStr string + } + validRandomUUID := "6ba7b811-9dad-11d1-80b4-00c04fd430c8" + invalidFormatRandomUUID := "6ba7b8119dad11d180b400c04fd430c8" + illegalCharRandomUUID := "6ba7b811-9dad-11d1-80b4-00c04fd430cG" + shortRandomUUID := "6ba7b811-9dad-11d1-80b4" + longRandomUUID := "6ba7b811-9dad-11d1-80b4-00c04fd430c8-extra" + emptyRandomUUID := "" + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "Test_parseRandomUUID_with_validRandomUUID", + args: args{ + uuidStr: validRandomUUID, + }, + want: true, + wantErr: false, + }, + { + name: "Test_parseRandomUUID_with_invalidFormatRandomUUID", + args: args{ + uuidStr: invalidFormatRandomUUID, + }, + want: false, + wantErr: false, + }, + { + name: "Test_parseRandomUUID_with_illegalCharRandomUUID", + args: args{ + uuidStr: illegalCharRandomUUID, + }, + want: false, + wantErr: true, + }, + { + name: "Test_parseRandomUUID_with_shortRandomUUID", + args: args{ + uuidStr: shortRandomUUID, + }, + want: false, + wantErr: true, + }, + { + name: "Test_parseRandomUUID_with_longRandomUUID", + args: args{ + uuidStr: longRandomUUID, + }, + want: false, + wantErr: true, + }, + { + name: "Test_parseRandomUUID_with_emptyRandomUUID", + args: args{ + uuidStr: emptyRandomUUID, + }, + want: false, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseUUID(tt.args.uuidStr) + if (err != nil) != tt.wantErr { + t.Errorf("parseRandomUUID() error = %v, wantErr %v", err, tt.wantErr) + return + } + if (got.String() != tt.args.uuidStr) == tt.want { + t.Errorf("parseRandomUUID() got = %v, want %v", got.String(), tt.args.uuidStr) + } + }) + } +} + +func BenchmarkParseRandomUUID(b *testing.B) { + uuidStr := "6ba7b811-9dad-11d1-80b4-00c04fd430c8" + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := parseUUID(uuidStr) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/functionsystem/apps/meta_service/common/versions/versions.go b/functionsystem/apps/meta_service/common/versions/versions.go new file mode 100644 index 0000000000000000000000000000000000000000..980d7d8b7c192e0457b7a0023771929485d8cf04 --- /dev/null +++ b/functionsystem/apps/meta_service/common/versions/versions.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package versions for version info +package versions + +var ( + // BuildVersion build version + buildVersion = "UnKnown" + // GitBranch git branch + gitBranch = "Unknown" + // GitHash git hash + gitHash = "UnKnown" +) + +// GetBuildVersion get build version +func GetBuildVersion() string { + return buildVersion +} + +// GetGitBranch get git branch +func GetGitBranch() string { + return gitBranch +} + +// GetGitHash get git hash +func GetGitHash() string { + return gitHash +} diff --git a/functionsystem/apps/meta_service/etcd/client.go b/functionsystem/apps/meta_service/etcd/client.go new file mode 100644 index 0000000000000000000000000000000000000000..0a5917dc56b031cfb4a1e751ff99706986ca1abb --- /dev/null +++ b/functionsystem/apps/meta_service/etcd/client.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package etcd creates an etcdv3 client for meta service. +package etcd + +import ( + "meta_service/common/logger/log" + "meta_service/function_repo/config" + + "meta_service/common/etcd3" +) + +// NewClient returns a etcd client +func NewClient() (*etcd3.EtcdClient, error) { + cli, err := etcd3.NewEtcdWatcher(config.RepoCfg.EtcdCfg) + if err != nil { + log.GetLogger().Errorf("failed to new etcd client :%s", err) + return nil, err + } + return cli, nil +} + +// NewMetaClient returns a meta etcd client +func NewMetaClient() (*etcd3.EtcdClient, error) { + cli, err := etcd3.NewEtcdWatcher(config.RepoCfg.MetaEtcdCfg) + if err != nil { + log.GetLogger().Errorf("failed to new meta etcd client :%s", err) + return nil, err + } + return cli, nil +} diff --git a/functionsystem/apps/meta_service/etcd/client_test.go b/functionsystem/apps/meta_service/etcd/client_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3dedc7f4ee3b393a92c633fce94c95ad8d22550d --- /dev/null +++ b/functionsystem/apps/meta_service/etcd/client_test.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package etcd + +import ( + "errors" + "testing" + + "meta_service/function_repo/config" + + "meta_service/common/etcd3" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/goconvey/convey" +) + +func TestInitializeFailed(t *testing.T) { + convey.Convey("TestNewClientFailed", t, func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(etcd3.NewEtcdWatcher, func(config etcd3.EtcdConfig) (*etcd3.EtcdClient, error) { + return &etcd3.EtcdClient{}, errors.New("test error") + }) + config.RepoCfg = new(config.Configs) + _, err := NewClient() + convey.So(err, convey.ShouldNotEqual, nil) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/config/type.go b/functionsystem/apps/meta_service/function_repo/config/type.go new file mode 100644 index 0000000000000000000000000000000000000000..b705b07b61c8c71933b4c868eb0da8b42a2b764c --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/config/type.go @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package config implements config map of repository +package config + +import ( + "encoding/json" + "os" + "strings" + + "github.com/asaskevich/govalidator/v11" + + "meta_service/common/etcd3" + "meta_service/common/logger/log" + "meta_service/common/reader" + "meta_service/common/tls" + "meta_service/common/utils" + "meta_service/function_repo/utils/constants" +) + +const ( + // DefaultTimeout define a defaultTimeout + DefaultTimeout = 900 + + envS3AccessKey = "S3_ACCESS_KEY" + envS3SecretKey = "S3_SECRET_KEY" +) + +// RepoCfg is config map of repository +var RepoCfg *Configs + +// FunctionType is a type of function, only use service now +type FunctionType string + +const ( + // ComFunctionType common function type + ComFunctionType FunctionType = "service" + + caFilePath = "/home/sn/resource/etcd/ca.crt" + + certFilePath = "/home/sn/resource/etcd/client.crt" + + keyFilePath = "/home/sn/resource/etcd/client.key" + + passphraseFilePath = "/home/sn/resource/etcd/passphrase" +) + +type envConfig struct { + TimeZone string `json:"timeZone" valid:"stringlength(1|255)"` + NodejsLdPath string `json:"nodejsLdPath" valid:"stringlength(1|255)"` + NodejsPath string `json:"nodejsPath" valid:"stringlength(1|255)"` + JavaLdPath string `json:"javaLdPath" valid:"stringlength(1|255)"` + JavaPath string `json:"javaPath" valid:"stringlength(1|255)"` + CppLdPath string `json:"cppLdPath" valid:"stringlength(1|255)"` + PythonLdPath string `json:"pythonLdPath" valid:"stringlength(1|255)"` + PythonPath string `json:"pythonPath" valid:"stringlength(1|255)"` +} + +type funcDefault struct { + Version string `json:"version" valid:"stringlength(1|32)"` + EnvPrefix string `json:"envPrefix" valid:"stringlength(1|32)"` + PageIndex uint `json:"pageIndex" valid:"numeric"` + PageSize uint `json:"pageSize" valid:"numeric"` + CPUList []int64 `json:"cpuList"` + MemoryList []int64 `json:"memoryList"` + Timeout int64 `json:"timeout" valid:"numeric"` + DefaultMinInstance string `json:"defaultMinInstance"` + DefaultMaxInstance string `json:"defaultMaxInstance"` + DefaultConcurrentNum string `json:"defaultConcurrentNum"` + MaxInstanceUpperLimit string `json:"maxInstanceUpperLimit"` + ConcurrentNumUpperLimit string `json:"concurrentNumUpperLimit"` + MinCPU int64 `json:"minCpu" valid:"numeric"` + MinMemory int64 `json:"minMemory" valid:"numeric"` + MaxCPU int64 `json:"maxCpu" valid:"numeric"` + MaxMemory int64 `json:"maxMemory" valid:"numeric"` +} + +// PackageConfig defines the configuration for function packages' and layer packages' validations. +type PackageConfig struct { + UploadTmpPath string `json:"uploadTempPath,omitempty" valid:"optional"` + ZipFileSizeMaxMB uint `json:"zipFileSizeMaxMB" valid:"numeric"` + UnzipFileSizeMaxMB uint `json:"unzipFileSizeMaxMB" valid:"numeric"` + FileCountsMax uint `json:"fileCountsMax" valid:"numeric"` + DirDepthMax uint `json:"dirDepthMax" valid:"numeric"` + IOReadTimeout uint `json:"ioReadTimeout" valid:"numeric,required"` +} + +// BucketConfig defines bucket configuration for tenant, each tenant may have one or more buckets. +type BucketConfig struct { + BucketID string `json:"bucketId" valid:"stringlength(1|255)"` + BusinessID string `json:"businessId" valid:"stringlength(1|255)"` + AppID string `json:"appId" valid:"stringlength(1|255)"` + AppSecret string `json:"appSecret" valid:"stringlength(1|255)"` + URL string `json:"url" valid:"stringlength(1|255)"` + Writable int `json:"writable" valid:"in(0|1)"` + Desc string `json:"description" valid:"stringlength(1|255)"` + CreateTime string `json:"createTime,omitempty" valid:"optional"` + UpdateTime string `json:"updateTime,omitempty" valid:"optional"` +} + +type funcConfig struct { + DefaultCfg funcDefault `json:"default,omitempty" valid:"optional"` + PackageCfg PackageConfig `json:"package,omitempty" valid:"optional"` + VersionMax uint `json:"versionMax" valid:"optional"` + AliasMax uint `json:"aliasMax" valid:"optional"` + LayerMax int `json:"layerMax" valid:"optional"` + InstanceLabelMax int `json:"instanceLabelMax" valid:"optional"` +} + +type triggerType struct { + SourceProvider string `json:"sourceProvider" valid:"stringlength(1|255)"` + Effect string `json:"effect" valid:"stringlength(1|255)"` + Action string `json:"action" valid:"stringlength(1|255)"` +} + +type triggerConfig struct { + URLPrefix string `json:"urlPrefix" valid:"url"` + TriggerType []triggerType `json:"type" valid:"required"` +} + +type s3Config struct { + Endpoint string `json:"endpoint" valid:"url,required"` + AccessKey string `json:"accessKey" valid:"stringlength(1|255)"` + SecretKey string `json:"secretKey" valid:"stringlength(1|255)"` + Secure bool `json:"secure"` + URLExpires uint `json:"presignedUrlExpires" valid:"numeric"` + Timeout uint `json:"timeout" valid:"numeric"` + CaFile string `json:"caFile,omitempty" valid:"optional"` + TrustedCA bool `json:"trustedCA,omitempty" valid:"optional"` +} + +type fileServerConfig struct { + StorageType string `json:"storageType" valid:"required"` + S3 s3Config `json:"s3" valid:"optional"` +} + +type urnConfig struct { + Prefix string `json:"prefix" valid:"stringlength(1|16)"` + Zone string `json:"zone" valid:"stringlength(1|16)"` + ResourceType string `json:"resourceType" valid:"stringlength(1|16)"` +} + +type serverCfg struct { + IP string `json:"ip"` + Port uint `json:"port"` +} + +// Configs dump config from file +type Configs struct { + EtcdCfg etcd3.EtcdConfig `json:"etcd" valid:"required"` + MetaEtcdEnable bool `json:"metaEtcdEnable"` + MetaEtcdCfg etcd3.EtcdConfig `json:"metaEtcd"` + FunctionCfg funcConfig `json:"function" valid:"required"` + TriggerCfg triggerConfig `json:"trigger" valid:"required"` + BucketCfg []BucketConfig `json:"bucket" valid:"required"` + RuntimeType []string `json:"runtimeType" valid:"required"` + CompatibleRuntimeType []string `json:"compatibleRuntimeType" valid:"required"` + FileServer fileServerConfig `json:"fileServer" valid:"required"` + URNCfg urnConfig `json:"urn"` + EnvCfg envConfig `json:"env"` + ServerCfg serverCfg `json:"server"` + MutualTLSConfig tls.MutualTLSConfig `json:"mutualTLSConfig" valid:"optional"` + MutualSSLConfig tls.MutualSSLConfig `json:"mutualSSLConfig" valid:"optional"` + DecryptAlgorithm string `json:"decryptAlgorithm" valid:"optional"` + Clusters string `json:"clusters"` + ClusterID map[string]bool `json:"-"` +} + +// InitConfig get config info from configPath +func InitConfig(filename string) error { + data, err := reader.ReadFileWithTimeout(filename) + if err != nil { + log.GetLogger().Errorf("failed to read config, filename: %s, error: %s", filename, err.Error()) + return err + } + + RepoCfg = new(Configs) + + err = json.Unmarshal(data, RepoCfg) + if err != nil { + log.GetLogger().Errorf("failed to unmarshal config, configPath: %s, error: %s", filename, err.Error()) + return err + } + if RepoCfg.EtcdCfg.AuthType == "TLS" { + setEtcdResourceCerts() + } else { + RepoCfg.EtcdCfg = etcd3.GetETCDCertificatePath(RepoCfg.EtcdCfg, RepoCfg.MutualTLSConfig) + RepoCfg.MetaEtcdCfg = etcd3.GetETCDCertificatePath(RepoCfg.MetaEtcdCfg, RepoCfg.MutualTLSConfig) + } + + utils.ValidateTimeout(&RepoCfg.FunctionCfg.DefaultCfg.Timeout, DefaultTimeout) + _, err = govalidator.ValidateStruct(RepoCfg) + if err != nil { + log.GetLogger().Errorf("failed to validate config, err: %s", err.Error()) + return err + } + SetTLSConfig(RepoCfg) + setS3AccessInfoByEnv(RepoCfg) + RepoCfg.ClusterID = make(map[string]bool) + RepoCfg.ClusterID[constants.DefaultClusterID] = true + if RepoCfg.Clusters != "" { + splits := strings.Split(RepoCfg.Clusters, ",") + for _, item := range splits { + if item != "" { + RepoCfg.ClusterID[item] = true + } + } + } + return nil +} + +func setS3AccessInfoByEnv(cfg *Configs) { + // the ak sk will be covered by ENV + // so whether keep the env same with values in config file + // or make it empty, so it will use the values in config file + if ak := os.Getenv(envS3AccessKey); ak != "" { + cfg.FileServer.S3.AccessKey = ak + } + if sk := os.Getenv(envS3SecretKey); sk != "" { + cfg.FileServer.S3.SecretKey = sk + } +} + +func setEtcdResourceCerts() { + if RepoCfg.EtcdCfg.CaFile == "" { + RepoCfg.EtcdCfg.CaFile = caFilePath + } + if RepoCfg.EtcdCfg.CertFile == "" { + RepoCfg.EtcdCfg.CertFile = certFilePath + } + if RepoCfg.EtcdCfg.KeyFile == "" { + RepoCfg.EtcdCfg.KeyFile = keyFilePath + } + if RepoCfg.EtcdCfg.PassphraseFile == "" { + RepoCfg.EtcdCfg.PassphraseFile = passphraseFilePath + } + if RepoCfg.MetaEtcdCfg.CaFile == "" { + RepoCfg.MetaEtcdCfg.CaFile = caFilePath + } + if RepoCfg.MetaEtcdCfg.CertFile == "" { + RepoCfg.MetaEtcdCfg.CertFile = certFilePath + } + if RepoCfg.MetaEtcdCfg.KeyFile == "" { + RepoCfg.MetaEtcdCfg.KeyFile = keyFilePath + } + if RepoCfg.MetaEtcdCfg.PassphraseFile == "" { + RepoCfg.MetaEtcdCfg.PassphraseFile = passphraseFilePath + } +} + +func SetTLSConfig(metaServCfg *Configs) { + metaServCfg.MutualTLSConfig.TLSEnable = metaServCfg.MutualSSLConfig.SSLEnable + metaServCfg.MutualTLSConfig.RootCAFile = metaServCfg.MutualSSLConfig.RootCAFile + metaServCfg.MutualTLSConfig.ModuleCertFile = metaServCfg.MutualSSLConfig.ModuleCertFile + metaServCfg.MutualTLSConfig.ModuleKeyFile = metaServCfg.MutualSSLConfig.ModuleKeyFile + metaServCfg.MutualTLSConfig.PwdFile = metaServCfg.MutualSSLConfig.PwdFile + metaServCfg.MutualTLSConfig.ServerName = metaServCfg.MutualSSLConfig.ServerName + metaServCfg.MutualTLSConfig.DecryptTool = metaServCfg.MutualSSLConfig.DecryptTool +} diff --git a/functionsystem/apps/meta_service/function_repo/config/type_test.go b/functionsystem/apps/meta_service/function_repo/config/type_test.go new file mode 100644 index 0000000000000000000000000000000000000000..93977cd4efe15601687e08c59d9a456a42f8b4d0 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/config/type_test.go @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "encoding/json" + "io/ioutil" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/goconvey/convey" + + "meta_service/common/etcd3" +) + +func TestInitConfigFailed(t *testing.T) { + convey.Convey("Test GetFunctionConfig", t, func() { + convey.Convey("ReadFileFailed", func() { + err := InitConfig("test") + convey.ShouldNotBeNil(err) + }) + convey.Convey("UnmarshalFile", func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { return []byte("00"), nil }) + err := InitConfig("test") + convey.ShouldNotBeNil(err) + }) + convey.Convey("validateStruct", func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + RepoCfg = &Configs{ + EtcdCfg: etcd3.EtcdConfig{SslEnable: false}, + FunctionCfg: funcConfig{ + DefaultCfg: funcDefault{Timeout: 1}, + }, + } + data, _ := json.Marshal(RepoCfg) + patches.ApplyFunc(ioutil.ReadFile, func(filename string) ([]byte, error) { return data, nil }) + err := InitConfig("test") + convey.ShouldNotBeNil(err) + }) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/errmsg/errors.go b/functionsystem/apps/meta_service/function_repo/errmsg/errors.go new file mode 100644 index 0000000000000000000000000000000000000000..9dd177adec6c9ab6b5c46449dfbd48302647c6e3 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/errmsg/errors.go @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package errmsg defines error messages +package errmsg + +import ( + "meta_service/common/snerror" +) + +const ( + descriptionSeparator = ". " + + // FunctionNameExist means function name exist + FunctionNameExist = 4101 + // AliasNameAlreadyExists means alias name exists + AliasNameAlreadyExists = 4102 + // TotalRoutingWeightNotOneHundred means total routing weight is not 100 + TotalRoutingWeightNotOneHundred = 4103 + // InvalidUserParam means user's input parameter is not valid + InvalidUserParam = 4104 + // FunctionVersionDeletionForbidden means function version has relate alias, so cannot be deleted + FunctionVersionDeletionForbidden = 4105 + // LayerVersionSizeOutOfLimit means layer version is out of limit + LayerVersionSizeOutOfLimit = 4106 + // TenantLayerSizeOutOfLimit means layer size within the tenant is out of limit + TenantLayerSizeOutOfLimit = 4107 + // LayerVersionNumOutOfLimit means layer version number is out of limit + LayerVersionNumOutOfLimit = 4108 + // TriggerNumOutOfLimit means number of triggers of the function is out of limit + TriggerNumOutOfLimit = 4109 + // FunctionVersionOutOfLimit means function version exceeds limit + FunctionVersionOutOfLimit = 4110 + // AliasOutOfLimit means alias number of function exceeds limit + AliasOutOfLimit = 4111 + // LayerIsUsed means layer if used by function, so cannot be deleted + LayerIsUsed = 4112 + // LayerVersionNotFound means layer version is not created + LayerVersionNotFound = 4113 + // RepeatedPublishmentError means there is no difference between two publishment + RepeatedPublishmentError = 4114 + // FunctionNotFound means function is not created or is deleted + FunctionNotFound = 4115 + // FunctionVersionNotFound means function version is not created or is deleted + FunctionVersionNotFound = 4116 + // AliasNameNotFound means alias is not created or is deleted + AliasNameNotFound = 4117 + // TriggerNotFound means trigger is not created or is deleted + TriggerNotFound = 4118 + // LayerNotFound means layer is not created or is deleted + LayerNotFound = 4119 + // PoolNotFound means pool is not created or is deleted + PoolNotFound = 4120 + + // PageInfoError means page info out of list boundary + PageInfoError = 4123 + // TriggerPathRepeated means trigger's path is used by other trigger + TriggerPathRepeated = 4124 + // DuplicateCompatibleRuntimes means duplicate items found in compatible runtimes list + DuplicateCompatibleRuntimes = 4125 + // CompatibleRuntimeError means some items of compatible runtimes are not in configuration list + CompatibleRuntimeError = 4126 + // ZipFileCountError means count of files inside zip file is out of limit of configuration + ZipFileCountError = 4127 + // ZipFilePathError means some file path inside zip file is not valid + ZipFilePathError = 4128 + // ZipFileUnzipSizeError means unzipped files size of zip file is out of limit of configuration + ZipFileUnzipSizeError = 4129 + // ZipFileSizeError means size of zip file is out of limit of configuration + ZipFileSizeError = 4130 + + // RevisionIDError means revision ID of request does not match that of entry to operate in storage + RevisionIDError = 4134 + + // SaveFileError means saving temporary file has some error when handling zip file of package + SaveFileError = 121016 + // UploadFileError means uploading file has some error when handling zip file of package + UploadFileError = 121017 + // EmptyAliasAndVersion means alias name and version are empty + EmptyAliasAndVersion = 121018 + // ReadingPackageTimeout means timeout while reading a package + ReadingPackageTimeout = 121019 + + // BucketNotFound means bucket info is not found for specific business ID + BucketNotFound = 121026 + // ZipFileError means error occurs when handling zip file of package + ZipFileError = 121029 + // DownloadFileError means error occurs when getting presigned downloading URL from package storage + DownloadFileError = 121030 + // DeleteFileError means error occurs when deleting object of package storage + DeleteFileError = 121032 + // InvalidFunctionLayer means layer's tenant info is not match that of its request's context + InvalidFunctionLayer = 121036 + + // InvalidQueryURN means URN in query request is invalid + InvalidQueryURN = 121046 + // InvalidQualifier means qualifier in request is invalid, it represents either version or alias + InvalidQualifier = 121047 + // ReadBodyError means error occurs when handling HTTP request's body + ReadBodyError = 121048 + // AuthCheckError means error occurs when authentication HTTP request + AuthCheckError = 121049 + // InvalidJSONBody means error occurs when parsing HTTP request's body to JSON format + InvalidJSONBody = 121052 + // InvalidParamErrorCode means error occurs when parsing HTTP request's body to JSON format + InvalidParamErrorCode = 130600 + // TriggerIDNotFound means trigger ID in request is not found in storage + TriggerIDNotFound = 121057 + // FunctionNameFormatErr means function name format is wrong in storage + FunctionNameFormatErr = 121058 + + // KVNotFound means etcd has no such kv + KVNotFound = 122001 + // EtcdError means an error in etcd + EtcdError = 122002 + // TransactionFailed means an etcd transaction has an error + TransactionFailed = 122003 + // UnmarshalFailed means unmarshal to json error + UnmarshalFailed = 122004 + // MarshalFailed means marshal to string error + MarshalFailed = 122005 + // VersionOrAliasEmpty means version or alias is empty string + VersionOrAliasEmpty = 122006 + // ResourceIDEmpty means resource ID is empty string + ResourceIDEmpty = 122007 + // NoTenantInfo means tenant info does not exist + NoTenantInfo = 122008 + // NoResourceInfo means resource info does not exist + NoResourceInfo = 122009 +) + +var checkInputParamMsg = "check input parameters" + +var errorMsg = map[int]string{ + InvalidUserParam: userMessage("%s", checkInputParamMsg), + FunctionNameExist: userMessage("the function name already exists", "rename your function"), + RevisionIDError: "revisionID is not the same", + FunctionVersionNotFound: userMessage("function [%s] version [%s] is not found", checkInputParamMsg), + FunctionVersionDeletionForbidden: userMessage("this version is occupied by an alias", "remove the "+ + "mapping first"), + FunctionNotFound: userMessage("function [%s] is not found", checkInputParamMsg), + LayerVersionSizeOutOfLimit: userMessage("the version size [%d] is larger than max config size [%d]", + "delete or resize"), + TenantLayerSizeOutOfLimit: userMessage("the tenant layer size [%d] is larger than the maximum config size [%d"+ + "]", "consult the administrator"), + LayerVersionNumOutOfLimit: userMessage("the maximum layer version number is larger than the config num [%d]", + checkInputParamMsg), + LayerIsUsed: userMessage("layer version %d has been bound to the function", + "please unbind it before performing the operation"), + PageInfoError: userMessage("the page information is out of the query range", checkInputParamMsg), + LayerNotFound: userMessage("layer [%s] not found", checkInputParamMsg), + DuplicateCompatibleRuntimes: userMessage("duplicated compatibleRuntimes exists", checkInputParamMsg), + CompatibleRuntimeError: userMessage("compatible runtime is invalid", checkInputParamMsg), + TriggerPathRepeated: userMessage("trigger path is repeated", checkInputParamMsg), + TriggerNumOutOfLimit: userMessage("the number of triggers exceeds the upper limit [%d]", + "delete or resize the limit"), + TriggerNotFound: userMessage("trigger [%s] is not found", checkInputParamMsg), + TriggerIDNotFound: userMessage("trigger [%s] is not found", checkInputParamMsg), + AliasNameNotFound: userMessage("functionName [%s] and aliasName [%s] do not exist", checkInputParamMsg), + AliasOutOfLimit: userMessage("the number of existing function alias is greater than the set value [%d]", + "delete or resize the limit"), + InvalidJSONBody: "request body is not a valid JSON object", + TotalRoutingWeightNotOneHundred: userMessage("total routing weight is not 100", + "check the routing weight. the sum of the values is 100"), +} + +var ( + // KeyNotFoundError means key does not exist in etcd + KeyNotFoundError = snerror.New(KVNotFound, "KV not exist in etcd") + // EtcdInternalError means etcd has some error + EtcdInternalError = snerror.New(EtcdError, "etcd internal error") + // EtcdTransactionFailedError means etcd transaction has some error + EtcdTransactionFailedError = snerror.New(TransactionFailed, "etcd transaction failed") + // UnmarshalError means unmarshal to json error + UnmarshalError = snerror.New(UnmarshalFailed, "failed to unmarshal") + // MarshalError means marshal to string error + MarshalError = snerror.New(MarshalFailed, "failed to marshal") + // VersionOrAliasError means version or alias is empty string + VersionOrAliasError = snerror.New(VersionOrAliasEmpty, "version or alias name is empty") + // ResourceIDError means resource ID is empty string + ResourceIDError = snerror.New(ResourceIDEmpty, "resource ID is empty") + // PageError means page info out of list boundary + PageError = snerror.New(PageInfoError, "page size cannot be 0") + // PoolNotFoundError means pool id is not found + PoolNotFoundError = snerror.New(PoolNotFound, userMessage("pool does not exist or has been deleted", + checkInputParamMsg)) +) + +// ErrorMessage returns a text for the error code. It returns the empty +// string if the code is unknown. +func ErrorMessage(code int) string { + return errorMsg[code] +} + +// userMessage constructs user message which composes of two parts: message and suggestion +func userMessage(msg string, suggestions string) string { + return msg + descriptionSeparator + suggestions +} + +// New generates an error with code and format +func New(code int, v ...interface{}) snerror.SNError { + return snerror.NewWithFmtMsg(code, ErrorMessage(code), v...) +} + +// NewParamError generates an error of type InvalidUserParam with specified error message. +func NewParamError(msg string, v ...interface{}) snerror.SNError { + return snerror.NewWithFmtMsg(InvalidUserParam, userMessage(msg, checkInputParamMsg), v...) +} diff --git a/functionsystem/apps/meta_service/function_repo/errmsg/errors_test.go b/functionsystem/apps/meta_service/function_repo/errmsg/errors_test.go new file mode 100644 index 0000000000000000000000000000000000000000..762000d592008e872c92e5e4d5ba152815bbbb67 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/errmsg/errors_test.go @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package errmsg + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrorMessage(t *testing.T) { + assert.Equal(t, ErrorMessage(LayerNotFound), userMessage("layer [%s] not found", checkInputParamMsg)) +} + +func TestNew(t *testing.T) { + assert.NotNil(t, New(LayerNotFound)) +} + +func TestNewParamError(t *testing.T) { + assert.NotNil(t, NewParamError("fake error! ")) +} diff --git a/functionsystem/apps/meta_service/function_repo/model/alias.go b/functionsystem/apps/meta_service/function_repo/model/alias.go new file mode 100644 index 0000000000000000000000000000000000000000..45d32e2ae3716dec6cb8a49dd8b45321ff09bca6 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/alias.go @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +import "errors" + +const ( + // limit routingConfig to 16 in code, cli limit routingConfig to 2 + maxRoutingConfigNum = 16 +) + +// AliasRequest is alias create request +type AliasRequest struct { + Name string `json:"name" valid:"matches(^[a-z][0-9a-z-]{0\,14}[0-9a-z]$),maxstringlength(16)"` + FunctionVersion string `json:"functionVersion" valid:"maxstringlength(16)"` + Description string `json:"description,omitempty" valid:"maxstringlength(1024)"` + RoutingConfig map[string]int `json:"routingConfig" valid:"-"` +} + +// Validate validate alias create request +func (r *AliasRequest) Validate() error { + if len(r.RoutingConfig) > maxRoutingConfigNum { + return errors.New("alias router number is out of range") + } + return nil +} + +// AliasUpdateRequest is alias update request +type AliasUpdateRequest struct { + FunctionVersion string `json:"functionVersion" valid:"maxstringlength(16)"` + Description string `json:"description,omitempty" valid:"maxstringlength(1024)"` + RevisionID string `json:"revisionId" valid:"maxstringlength(20)"` + RoutingConfig map[string]int `json:"routingConfig,omitempty" valid:"-"` +} + +// Validate validate alias update request +func (r *AliasUpdateRequest) Validate() error { + if len(r.RoutingConfig) > maxRoutingConfigNum { + return errors.New("alias router number is out of range") + } + return nil +} + +// AliasQueryRequest is alias query request +type AliasQueryRequest struct { + FunctionName string `json:"functionName"` + AliasName string +} + +// AliasListQueryRequest is alias list query request +type AliasListQueryRequest struct { + FunctionName string + FunctionVersion string + PageIndex int + PageSize int +} + +// AliasDeleteRequest is alias delete request +type AliasDeleteRequest struct { + FunctionName string + AliasName string +} + +// AliasResponse is base response of alias operation +type AliasResponse struct { + AliasURN string `json:"aliasUrn"` + Name string `json:"name"` + FunctionVersion string `json:"functionVersion"` + Description string `json:"description,omitempty"` + RevisionID string `json:"revisionId"` + RoutingConfig map[string]int `json:"routingConfig"` +} + +// AliasCreateResponse is response of alias create +type AliasCreateResponse struct { + AliasResponse +} + +// AliasUpdateResponse is response of alias update +type AliasUpdateResponse struct { + AliasResponse +} + +// AliasQueryResponse is response of alias query +type AliasQueryResponse struct { + AliasResponse +} + +// AliasListQueryResponse is response of alias list query +type AliasListQueryResponse struct { + Total int `json:"total"` + Aliases []AliasResponse `json:"aliases"` +} diff --git a/functionsystem/apps/meta_service/function_repo/model/alias_test.go b/functionsystem/apps/meta_service/function_repo/model/alias_test.go new file mode 100644 index 0000000000000000000000000000000000000000..605c0f01312ad624d11cd4ae54ac41065e2fc596 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/alias_test.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" +) + +func TestValidateAlias(t *testing.T) { + r := &AliasRequest{} + ur := &AliasUpdateRequest{} + convey.Convey("TestValidateAlias", t, func() { + convey.Convey("out of range", func() { + tmp := map[string]int{ + "0": 1, "1": 1, "2": 1, "3": 1, "4": 1, "5": 1, "6": 1, "7": 1, "8": 1, "9": 1, "10": 1, "11": 1, + "12": 1, "13": 1, "14": 1, "15": 1, "16": 1, + } + r.RoutingConfig = tmp + err := r.Validate() + convey.ShouldNotBeNil(err) + + ur.RoutingConfig = tmp + err = ur.Validate() + convey.ShouldNotBeNil(err) + }) + convey.Convey("succeed", func() { + tmp := map[string]int{ + "0": 1, + } + r.RoutingConfig = tmp + err := r.Validate() + convey.ShouldNotBeNil(err) + + ur.RoutingConfig = tmp + err = ur.Validate() + convey.ShouldNotBeNil(err) + }) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/model/function.go b/functionsystem/apps/meta_service/function_repo/model/function.go new file mode 100644 index 0000000000000000000000000000000000000000..c338206335a7b365a4577814bf2c20df3b706963 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/function.go @@ -0,0 +1,551 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package model is reset api model +package model + +import ( + "encoding/json" + "fmt" + "reflect" + "strconv" + + "meta_service/common/base" + "meta_service/common/types" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" +) + +// CustomPoolType - +const CustomPoolType = "custom" + +// FunctionQueryInfo - +type FunctionQueryInfo struct { + FunctionName, FunctionVersion, AliasName string +} + +// S3CodePathInfo - +type S3CodePathInfo struct { + BucketID string `form:"bucketId" json:"bucketId"` + ObjectID string `form:"objectId" json:"objectId"` + BucketUrl string `form:"bucketUrl" json:"bucketUrl"` + Token string `form:"token" json:"token"` + Sha512 string `form:"sha512" json:"sha512"` +} + +// ResourceAffinitySelector - +type ResourceAffinitySelector struct { + Group string `form:"group" json:"group"` + Priority int `form:"priority" json:"priority"` +} + +type FunctionBasicInfo struct { + Layers []string `json:"layers"` + Memory int64 `json:"memory"` + CPU int64 `json:"cpu"` + PoolType string `json:"poolType"` + MinInstance string `json:"minInstance"` + MaxInstance string `json:"maxInstance"` + ConcurrentNum string `json:"concurrentNum"` + Timeout int64 `json:"timeout" valid:"required"` + Handler string `json:"handler" valid:"maxstringlength(64)"` + Kind string `form:"kind" json:"kind"` + Environment map[string]string `json:"environment"` + Description string `json:"description" valid:"maxstringlength(1024)"` + StorageType string `json:"storageType"` + CodePath string `json:"codePath"` + FunctionType string `json:"functionType" valid:",optional"` + FunctionUpdateTime string `json:"functionUpdateTime" valid:",optional"` + FuncName string `json:"func_name" valid:",optional"` + Service string `json:"service" valid:",optional"` + IsBridgeFunction bool `json:"isBridgeFunction" valid:"optional"` + EnableAuthInHeader bool `json:"enable_auth_in_header" valid:"optional"` + CacheInstance int `json:"cacheInstance"` + HookHandler map[string]string `json:"hookHandler"` + ExtendedHandler map[string]string `form:"extendedHandler" json:"extendedHandler"` + ExtendedTimeout map[string]int `form:"extendedTimeout" json:"extendedTimeout"` + CustomResources map[string]float64 `json:"customResources"` + Device types.Device `form:"device" json:"device,omitempty"` + S3CodePath S3CodePathInfo `form:"s3CodePath" json:"s3CodePath"` + SchedulePolicies []ResourceAffinitySelector `form:"schedulePolicies" json:"schedulePolicies"` + ResourceAffinitySelectors []ResourceAffinitySelector `form:"resourceAffinitySelectors" json:"resourceAffinitySelectors"` + CodeUploadType string `form:"codeUploadType" json:"codeUploadType"` + PoolID string `form:"poolId" json:"poolId" valid:"optional,poolId~pool id can contain only lowercase letters;digits and - it cannot start or end with - and cannot exceed 40 characters or less than 1 characters"` +} + +// FunctionCreateRequest - +type FunctionCreateRequest struct { + FunctionBasicInfo + Name string `json:"name" valid:"required,functionName~format is 0@{serviceName}@{funcName} or 0-{serviceName}-{funcName}; serviceName can only contains letters、digits and cannot exceed 16 characters; funcName can only contains lowercase letters、digits、- and cannot exceed 127 characters or end with -"` + Runtime string `json:"runtime" valid:"required"` + ReversedConcurrency int `json:"reversedConcurrency,omitempty"` + Tags map[string]string `json:"tags"` + Kind string `json:"kind"` +} + +func (c *FunctionBasicInfo) validateCommon(configs *config.Configs) error { + // initialize the maximum and minimum number of instances + c.setDefault(*configs) + minInstance, err := strconv.Atoi(c.MinInstance) + if err != nil { + return errmsg.NewParamError(fmt.Sprintf("%s must be an integer", "minInstance")) + } + maxInstance, err := strconv.Atoi(c.MaxInstance) + if err != nil { + return errmsg.NewParamError(fmt.Sprintf("%s must be an integer", "maxInstance")) + } + concurrentNum, err := strconv.Atoi(c.ConcurrentNum) + if err != nil { + return errmsg.NewParamError(fmt.Sprintf("%s must be an integer", "concurrentNum")) + } + err = checkWorkerParams(minInstance, maxInstance, *configs, concurrentNum) + if err != nil { + return err + } + return nil +} + +// Validate create function request +func (c *FunctionCreateRequest) Validate(configs config.Configs) error { + if err := checkLayersNum(len(c.Layers), configs); err != nil { + return err + } + if c.PoolType != CustomPoolType && reflect.DeepEqual(c.Device, types.Device{}) { + if err := checkCPUAndMemory(c.CPU, c.Memory); err != nil { + return err + } + } + if err := checkRuntime(configs, c.Runtime); err != nil { + return err + } + return c.validateCommon(&configs) +} + +func (c *FunctionBasicInfo) setDefault(configs config.Configs) { + if c.ConcurrentNum == "" { + c.ConcurrentNum = configs.FunctionCfg.DefaultCfg.DefaultConcurrentNum + } + if c.MinInstance == "" { + c.MinInstance = configs.FunctionCfg.DefaultCfg.DefaultMinInstance + } + if c.MaxInstance == "" { + c.MaxInstance = configs.FunctionCfg.DefaultCfg.DefaultMaxInstance + } + if c.Timeout == 0 { + c.Timeout = configs.FunctionCfg.DefaultCfg.Timeout + } +} + +func checkRuntime(configs config.Configs, runtime string) error { + runtimeList := configs.RuntimeType + err := errmsg.NewParamError("runtime " + runtime + " is not valid") + for _, v := range runtimeList { + if v == runtime { + err = nil + break + } + } + if err != nil { + return err + } + return nil +} + +// CodeDownloadRequest - +type CodeDownloadRequest struct { + FunctionName string + FunctionVersion string + AliasName string +} + +// FunctionGetRequest - +type FunctionGetRequest struct { + FunctionName string + FunctionVersion string + AliasName string + Kind string +} + +// FunctionGetResponse - +type FunctionGetResponse struct { + FunctionVersion + Created string `json:"created"` +} + +// FunctionVersionListGetResponse - +type FunctionVersionListGetResponse struct { + Versions []FunctionVersion `json:"versions"` + Total int `json:"total"` +} + +// FunctionInfo describes function info +type FunctionInfo struct { + base.FunctionCommonInfo + FuncLayer []Layer `json:"funcLayer"` + Device types.Device `json:"device,omitempty"` +} + +// GetFunctionResponse is response of getting function +type GetFunctionResponse struct { + FunctionInfo + Created string `json:"created"` +} + +// PageInfo is page info of query response +type PageInfo struct { + PageIndex string `json:"pageIndex"` + PageSize string `json:"pageSize"` + Total uint64 `json:"total"` +} + +// GetFunctionVersionsResponse is response of all versions of a function +type GetFunctionVersionsResponse struct { + Functions []GetFunctionResponse `json:"functions"` + PageInfo PageInfo `json:"pageInfo"` +} + +// FunctionListGetResponse - +type FunctionListGetResponse struct { + FunctionVersion []FunctionVersion `json:"functions"` + Total int `json:"total"` +} + +// FunctionDeleteResponse is alia of FunctionVersionListGetResponse +type FunctionDeleteResponse FunctionVersionListGetResponse + +// FunctionDraftGetRequest - +type FunctionDraftGetRequest struct { + FunctionName string + FunctionVersion string + AliasName string +} + +// FunctionPolicyRequest - +type FunctionPolicyRequest struct { + TrustedTenantID string + SourceProvider string + SourceAccount string + SourceID string + Effect string + Action string + RevisionID string +} + +// FunctionPolicyResponse - +type FunctionPolicyResponse struct { +} + +// DateTime time +type DateTime struct { + CreateTime string `json:"createTime"` + UpdateTime string `json:"updateTime"` +} + +// FunctionLayer is functionVersion's layer +type FunctionLayer struct { + Name string `json:"name"` + Version int `json:"version"` + Order int `json:"order"` +} + +// FunctionCodeUploadRequest is upload function package request +type FunctionCodeUploadRequest struct { + RevisionID string + Kind string + FileSize int64 +} + +// FunctionCodeUploadResponse is upload function package response +type FunctionCodeUploadResponse struct { + FunctionVersionURN string `json:"functionVersionUrn"` +} + +// PublishResponse - +type PublishResponse = FunctionGetResponse + +// PublishRequest - +type PublishRequest struct { + RevisionID string `json:"revisionId"` + VersionDesc string `json:"versionDesc"` + Kind string `json:"kind"` + VersionNumber string `json:"versionNumber"` +} + +// FunctionUpdateRequest update request +type FunctionUpdateRequest struct { + FunctionBasicInfo + RevisionID string `json:"revisionId"` +} + +// Validate verify and update function request +func (c *FunctionUpdateRequest) Validate(configs config.Configs) error { + // If the request runtime is nil, the request will not be updated. + if err := checkLayersNum(len(c.Layers), configs); err != nil { + return err + } + + if c.PoolType != CustomPoolType && reflect.DeepEqual(c.Device, types.Device{}) { + if err := checkCPUAndMemory(c.CPU, c.Memory); err != nil { + return err + } + } + + if err := checkRuntime(configs, config.RepoCfg.RuntimeType[0]); err != nil { + return err + } + return c.validateCommon(&configs) +} + +func checkLayersNum(layerLen int, configs config.Configs) error { + if layerLen > configs.FunctionCfg.LayerMax { + return errmsg.NewParamError("the number of function reference layers "+ + "cannot exceed %d", configs.FunctionCfg.LayerMax) + } + return nil +} + +func checkCPUAndMemory(cpu, memory int64) error { + if cpu < config.RepoCfg.FunctionCfg.DefaultCfg.MinCPU || cpu > config.RepoCfg.FunctionCfg.DefaultCfg.MaxCPU { + return errmsg.NewParamError(fmt.Sprintf("CPU not in range [%d, %d]", config.RepoCfg.FunctionCfg.DefaultCfg.MinCPU, + config.RepoCfg.FunctionCfg.DefaultCfg.MaxCPU)) + } else if memory < config.RepoCfg.FunctionCfg.DefaultCfg.MinMemory || memory > config.RepoCfg.FunctionCfg. + DefaultCfg.MaxMemory { + return errmsg.NewParamError(fmt.Sprintf("memory not in range [%d, %d]", + config.RepoCfg.FunctionCfg.DefaultCfg.MinMemory, config.RepoCfg.FunctionCfg.DefaultCfg.MaxMemory)) + } + return nil +} + +// check Worker Params +func checkWorkerParams(minInstance int, maxInstance int, configs config.Configs, concurrentNum int) error { + if minInstance > maxInstance { + return errmsg.NewParamError(fmt.Sprintf("the value of minInstance %d must be smaller "+ + "than that of maxInstance %d", minInstance, maxInstance)) + } + if minInstance < 0 { + return errmsg.NewParamError(fmt.Sprintf("the value of minInstance %d should be bigger than 0", + minInstance)) + } + maxInstanceUpperLimit, err := strconv.Atoi(configs.FunctionCfg.DefaultCfg.MaxInstanceUpperLimit) + if err != nil { + return errmsg.NewParamError(fmt.Sprintf("%s must be an integer", "configs.FunctionCfg."+ + "DefaultCfg.MaxInstanceUpperLimit")) + } + if maxInstance > maxInstanceUpperLimit { + return errmsg.NewParamError(fmt.Sprintf("the value of maxInstance %d must be smaller"+ + " than that of workerInstanceMax %d", maxInstance, maxInstanceUpperLimit)) + } + concurrentNumUpperLimit, err := strconv.Atoi(configs.FunctionCfg.DefaultCfg.ConcurrentNumUpperLimit) + if err != nil { + return errmsg.NewParamError(fmt.Sprintf("%s must be an integer", "configs.FunctionCfg."+ + "DefaultCfg.ConcurrentNumUpperLimit")) + } + if concurrentNum > concurrentNumUpperLimit { + return errmsg.NewParamError(fmt.Sprintf("the value of concurrentNum %d must be smaller than that "+ + "of workerConcurrentNumMax %d", concurrentNum, concurrentNumUpperLimit)) + } + if concurrentNum < 1 { + return errmsg.NewParamError(fmt.Sprintf("the value of concurrentNum %d should be bigger than 1 ", + concurrentNum)) + } + return nil +} + +// FunctionVersion indicates the function instance information of the interface. +type FunctionVersion struct { + Function + FunctionVersionURN string `json:"functionVersionUrn"` + RevisionID string `json:"revisionId"` + CodeSize int64 `json:"codeSize"` + CodeSha256 string `json:"codeSha256"` + BucketID string `json:"bucketId"` + ObjectID string `json:"objectId"` + Handler string `json:"handler"` + Layers []string `json:"layers"` + CPU int64 `json:"cpu"` + Memory int64 `json:"memory"` + Runtime string `json:"runtime"` + Timeout int64 `json:"timeout"` + VersionNumber string `json:"versionNumber"` + VersionDesc string `json:"versionDesc"` + Environment map[string]string `json:"environment"` + CustomResources map[string]float64 `json:"customResources"` + StatefulFlag int `json:"statefulFlag"` + LastModified string `json:"lastModified"` + Published string `json:"Published"` + MinInstance int64 `json:"minInstance"` + MaxInstance int64 `json:"maxInstance"` + ConcurrentNum int `json:"concurrentNum"` + FuncLayer []FunctionLayer `json:"funcLayer"` + Status string `json:"status"` + InstanceNum int64 `json:"instanceNum"` + Device types.Device `json:"device,omitempty"` +} + +// Function is function entity +type Function struct { + DateTime + FunctionURN string `json:"functionUrn"` + FunctionName string `json:"name"` + TenantID string `json:"tenantId"` + BusinessID string `json:"businessId"` + ProductID string `json:"productId"` + ReversedConcurrency int `json:"reversedConcurrency"` + Description string `json:"description"` + Tag map[string]string `json:"tag"` +} + +// FunctionForUser describes function info +type FunctionForUser struct { + ID string `json:"id"` + CreateTime string `json:"createTime"` + UpdateTime string `json:"updateTime"` + FunctionURN string `json:"functionUrn"` + FunctionName string `json:"name"` + TenantID string `json:"tenantId"` + BusinessID string `json:"businessId"` + ProductID string `json:"productId"` + ReversedConcurrency int `json:"reversedConcurrency"` + Description string `json:"description"` + Tag map[string]string `json:"tag"` + FunctionVersionURN string `json:"functionVersionUrn"` + RevisionID string `json:"revisionId"` + CodeSize int64 `json:"codeSize"` + CodeSha256 string `json:"codeSha256"` + BucketID string `json:"bucketId"` + ObjectID string `json:"objectId"` + Handler string `json:"handler"` + Layers []string `json:"layers"` + CPU int `json:"cpu"` + Memory int `json:"memory"` + Runtime string `json:"runtime"` + Timeout int `json:"timeout"` + VersionNumber string `json:"versionNumber"` + VersionDesc string `json:"versionDesc"` + Environment map[string]string `json:"environment"` + CustomResources map[string]float64 `json:"customResources"` + StatefulFlag int `json:"statefulFlag"` + LastModified string `json:"lastModified"` + Published string `json:"Published"` + MinInstance int `json:"minInstance"` + MaxInstance int `json:"maxInstance"` + ConcurrentNum int `json:"concurrentNum"` + FuncLayer []Layer `json:"funcLayer"` + Status string `json:"status"` + InstanceNum int `json:"instanceNum"` + Device types.Device `json:"device,omitempty"` +} + +// GetFunctionResponseForUser is response of function info +type GetFunctionResponseForUser struct { + FunctionForUser + Created string `json:"created"` +} + +// MarshalJSON marshals function info to json +func (r *GetFunctionResponseForUser) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Code int `json:"code"` + Message string `json:"message"` + Function GetFunctionResponseForUser `json:"function"` + }{Message: "SUCCESS", Function: *r}) +} + +// BaseRequest base request +type BaseRequest struct { + TraceID string `json:"-"` +} + +// BaseResponse base response +type BaseResponse struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// ReserveInsBaseInfo basic info for reserved instance +type ReserveInsBaseInfo struct { + FuncName string `json:"funcName" valid:",required"` + Version string `json:"version" valid:",required"` + TenantID string `json:"-"` + InstanceLabel string `json:"instanceLabel" valid:"maxstringlength(63)"` +} + +// InstanceConfig instance config +type InstanceConfig struct { + ClusterID string `json:"clusterId" valid:"maxstringlength(64)"` + MaxInstance int64 `json:"maxInstance" valid:"range(0|1000)"` + MinInstance int64 `json:"minInstance" valid:"range(0|1000)"` +} + +// ReserveInstanceConfig reserve instance config +type ReserveInstanceConfig struct { + InstanceConfig InstanceConfig + InstanceLabel string +} + +// CreateReserveInsRequest create reserved instance meta request +type CreateReserveInsRequest struct { + BaseRequest + ReserveInsBaseInfo + InstanceConfigInfos []InstanceConfig `json:"instanceConfigInfos"` +} + +// CreateReserveInsResponse create reserved instance meta response +type CreateReserveInsResponse struct { + BaseResponse + InstanceConfigInfos []InstanceConfig `json:"instanceConfigInfos"` + ReserveInsBaseInfo ReserveInsBaseInfo `json:"reserveInsBaseInfo"` +} + +// UpdateReserveInsRequest update reserved instance meta request +type UpdateReserveInsRequest struct { + CreateReserveInsRequest +} + +// UpdateReserveInsResponse update reserved instance meta response +type UpdateReserveInsResponse struct { + CreateReserveInsResponse +} + +// DeleteReserveInsRequest delete reserved instance meta request +type DeleteReserveInsRequest struct { + BaseRequest + ReserveInsBaseInfo +} + +// DeleteReserveInsResponse delete reserved instance meta request +type DeleteReserveInsResponse struct { + BaseResponse +} + +// GetReserveInsRequest get reserved instance meta request +type GetReserveInsRequest struct { + FuncName string `json:"funcName"` + Version string `json:"version"` + TenantID string `json:"-"` +} + +// GetReserveInsResult get reserved instance result +type GetReserveInsResult struct { + InstanceLabel string `json:"instanceLabel"` + InstanceConfigInfos []InstanceConfig `json:"instanceConfigInfos"` +} + +// GetReserveInsResponse get reserved instance meta response +type GetReserveInsResponse struct { + Total int `json:"total"` + GetReserveInsResults []GetReserveInsResult `json:"results"` +} diff --git a/functionsystem/apps/meta_service/function_repo/model/function_test.go b/functionsystem/apps/meta_service/function_repo/model/function_test.go new file mode 100644 index 0000000000000000000000000000000000000000..e0c790da7f4ce0ce44608d0c82a6af146e342bb8 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/function_test.go @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +import ( + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/config" +) + +func TestCheckParamsFailed(t *testing.T) { + if err := config.InitConfig("/home/sn/repo/config.json"); err != nil { + t.Fatalf(err.Error()) + } + convey.Convey("TestCheckRuntimeFailed", t, func() { + c := config.Configs{} + r := &FunctionCreateRequest{ + FunctionBasicInfo: FunctionBasicInfo{ + Layers: []string{"HOME", "USERPROFILE"}, + CPU: 1, + Memory: 2, + }, + } + convey.Convey("exceed layer", func() { + c.FunctionCfg.LayerMax = 0 + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("memory not in config", func() { + c.FunctionCfg.LayerMax = 10 + c.FunctionCfg.DefaultCfg.CPUList = []int64{1} + c.FunctionCfg.DefaultCfg.MemoryList = []int64{1} + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("CPU and memory do not match", func() { + c.FunctionCfg.LayerMax = 10 + c.FunctionCfg.DefaultCfg.CPUList = []int64{1} + c.FunctionCfg.DefaultCfg.MemoryList = []int64{1, 2} + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("UpdateRequestFailed", func() { + ur := &FunctionUpdateRequest{} + err := ur.Validate(c) + convey.ShouldNotBeNil(err) + }) + }) +} + +func TestCheckRuntimeFailed(t *testing.T) { + convey.Convey("TestCheckRuntimeFailed", t, func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(checkLayersNum, func(layerLen int, configs config.Configs) error { + return nil + }) + patches.ApplyFunc(checkCPUAndMemory, func(cpu, memory int64) error { + return nil + }) + c := config.Configs{} + convey.Convey(" runtime is not valid", func() { + r := &FunctionCreateRequest{} + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("UpdateRequestFailed", func() { + config.RepoCfg = new(config.Configs) + config.RepoCfg.RuntimeType = []string{"test"} + ur := &FunctionUpdateRequest{} + err := ur.Validate(c) + convey.ShouldNotBeNil(err) + }) + }) +} + +func TestValidateFailed(t *testing.T) { + convey.Convey("TestValidateFailed", t, func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(checkLayersNum, func(layerLen int, configs config.Configs) error { + return nil + }) + patches.ApplyFunc(checkCPUAndMemory, func(cpu, memory int64) error { + return nil + }) + patches.ApplyFunc(checkRuntime, func(configs config.Configs, runtime string) error { return nil }) + c := config.Configs{} + r := &FunctionCreateRequest{} + ur := &FunctionUpdateRequest{} + config.RepoCfg = new(config.Configs) + config.RepoCfg.RuntimeType = []string{"test"} + c.FunctionCfg.DefaultCfg.Timeout = 0 + convey.Convey("minInstance must be an integer", func() { + err := r.Validate(c) + convey.ShouldNotBeNil(err) + err = ur.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("maxInstance must be an integer", func() { + c.FunctionCfg.DefaultCfg.DefaultMinInstance = "1" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + err = ur.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("concurrentNum must be an integer", func() { + c.FunctionCfg.DefaultCfg.DefaultMinInstance = "1" + c.FunctionCfg.DefaultCfg.DefaultMaxInstance = "1" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + err = ur.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("TestWorker Instance Failed", func() { + c.FunctionCfg.DefaultCfg.DefaultMinInstance = "1" + c.FunctionCfg.DefaultCfg.DefaultMaxInstance = "1" + c.FunctionCfg.DefaultCfg.DefaultConcurrentNum = "1" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + err = ur.Validate(c) + convey.ShouldNotBeNil(err) + }) + }) +} + +func TestCheckWorkerParams(t *testing.T) { + convey.Convey("TestCheckWorkerParamsFailed", t, func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(checkLayersNum, func(layerLen int, configs config.Configs) error { + return nil + }) + patches.ApplyFunc(checkCPUAndMemory, func(cpu, memory int64) error { + return nil + }) + patches.ApplyFunc(checkRuntime, func(configs config.Configs, runtime string) error { return nil }) + c := config.Configs{} + r := &FunctionCreateRequest{ + FunctionBasicInfo: FunctionBasicInfo{ + MaxInstance: "1", + MinInstance: "1", + ConcurrentNum: "1", + }, + } + convey.Convey("minInstance must be smaller than maxInstance", func() { + r.MaxInstance = "0" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("minInstance should be bigger 0", func() { + r.MinInstance = "-1" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("maxInstanceUpperLimit must be an integer", func() { + c.FunctionCfg.DefaultCfg.MaxInstanceUpperLimit = "a" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("maxInstance must be smaller than workerInstanceMax", func() { + c.FunctionCfg.DefaultCfg.MaxInstanceUpperLimit = "0" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("concurrentNum must be an integer ", func() { + c.FunctionCfg.DefaultCfg.MaxInstanceUpperLimit = "2" + c.FunctionCfg.DefaultCfg.ConcurrentNumUpperLimit = "a" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("concurrentNum must be smaller than workerConcurrentNumMax", func() { + c.FunctionCfg.DefaultCfg.MaxInstanceUpperLimit = "2" + c.FunctionCfg.DefaultCfg.ConcurrentNumUpperLimit = "0" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + convey.Convey("concurrentNum should be bigger than 1", func() { + c.FunctionCfg.DefaultCfg.MaxInstanceUpperLimit = "2" + c.FunctionCfg.DefaultCfg.ConcurrentNumUpperLimit = "2" + r.ConcurrentNum = "0" + err := r.Validate(c) + convey.ShouldNotBeNil(err) + }) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/model/layer.go b/functionsystem/apps/meta_service/function_repo/model/layer.go new file mode 100644 index 0000000000000000000000000000000000000000..2ff7d5e56a58a91be1a7b77e9618391c49b967bb --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/layer.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +import ( + "time" +) + +// LayerQueryInfo contains layer name and version. usually parsed from a layerURN. +type LayerQueryInfo struct { + LayerName string + LayerVersion int +} + +// CreateLayerRequest is the request body when creating a layer. +type CreateLayerRequest struct { + ZipFileSize int64 + CompatibleRuntimes string + Description string + LicenseInfo string +} + +// GetLayerListRequest is the request body when getting a layer. +type GetLayerListRequest struct { + LayerName string `form:"layerName" valid:"maxstringlength(32)"` + LayerVersion int `form:"layerVersion" valid:"range(1|1000000)"` + CompatibleRuntime string `form:"compatibleRuntime" valid:"maxstringlength(64)"` + OwnerFlag int `form:"ownerFlag" valid:"range(0|2)"` + PageIndex int `form:"pageIndex" valid:"range(1|1000)"` + PageSize int `form:"pageSize" valid:"range(1|100)"` +} + +// GetLayerVersionListRequest is used when getting layer version list. +type GetLayerVersionListRequest struct { + LayerName string + CompatibleRuntime string + PageIndex int + PageSize int +} + +// UpdateLayerRequest is the request body when updating a layer. +type UpdateLayerRequest struct { + Description string `json:"description" valid:"maxstringlength(1024)"` + LicenseInfo string `json:"licenseInfo" valid:"maxstringlength(64)"` + LastUpdateTime time.Time `json:"lastUpdateTime" valid:",required"` + Version int `json:"version" valid:"range(1|10000)"` +} + +// Layer contains layer information. +type Layer struct { + Name string `json:"name"` + Version int `json:"versionNumber"` + LayerURN string `json:"layerUrn"` + LayerVersionURN string `json:"layerVersionUrn"` + LayerSize int64 `json:"layerSize"` + LayerSHA256 string `json:"layerSha256"` + CreateTime time.Time `json:"createTime"` + UpdateTime time.Time `json:"updateTime"` + CompatibleRuntimes []string `json:"compatibleRuntimes"` + Description string `json:"description"` + LicenseInfo string `json:"licenseInfo"` +} + +// LayerList is a list of layers. +type LayerList struct { + Total int `json:"total"` + LayerVersions []Layer `json:"layerVersions"` +} diff --git a/functionsystem/apps/meta_service/function_repo/model/podpool.go b/functionsystem/apps/meta_service/function_repo/model/podpool.go new file mode 100644 index 0000000000000000000000000000000000000000..fc208ef517949f6bbf44a08fddfe71691e75c6a3 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/podpool.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +// Resource - +type Resource struct { + Requests map[string]string `json:"requests"` + Limits map[string]string `json:"limits"` +} + +type IdleRecyclePolicy struct { + Reserved int `json:"reserved"` + Scaled int `json:"scaled"` +} + +// Pool - +type Pool struct { + Id string `json:"id"` + Group string `json:"group"` + Size int `json:"size"` + MaxSize int `json:"max_size"` + ReadyCount int `json:"ready_count"` + Status int `json:"status"` + Msg string `json:"msg"` + Image string `json:"image"` + InitImage string `json:"init_image"` + Reuse bool `json:"reuse"` + Labels map[string]string `json:"labels"` + Environment map[string]string `json:"environment"` + Resources Resource `json:"resources"` + Volumes string `json:"volumes"` + VolumeMounts string `json:"volume_mounts"` + Affinities string `json:"affinities"` + RuntimeClassName string `json:"runtime_class_name"` + NodeSelector map[string]string `json:"node_selector"` + Tolerations string `json:"tolerations"` + HorizontalPodAutoscalerSpec string `json:"horizontal_pod_autoscaler_spec"` + TopologySpreadConstraints string `json:"topology_spread_constraints"` + PodPendingDurationThreshold int `json:"pod_pending_duration_threshold"` + IdleRecycleTime IdleRecyclePolicy `json:"idle_recycle_time"` + Scalable bool `json:"scalable"` +} + +// PodPoolCreateRequest - +type PodPoolCreateRequest struct { + Pools []Pool `json:"pools"` +} + +// PodPoolCreateResponse - +type PodPoolCreateResponse struct { + FailedPools []string `json:"failed_pools"` +} + +// PodPoolUpdateRequest - +type PodPoolUpdateRequest struct { + ID string `json:"id"` + Size int `json:"size"` + MaxSize int `json:"max_size"` + HorizontalPodAutoscalerSpec string `json:"horizontal_pod_autoscaler_spec"` +} + +// PodPoolGetRequest - +type PodPoolGetRequest struct { + ID string + Group string + Limit int + Offset int +} + +// PodPoolGetResponse - +type PodPoolGetResponse struct { + Count int `json:"count"` + PodPools []Pool `json:"pools"` +} diff --git a/functionsystem/apps/meta_service/function_repo/model/service.go b/functionsystem/apps/meta_service/function_repo/model/service.go new file mode 100644 index 0000000000000000000000000000000000000000..609382fdfdff5b49b0f2cc2063cb6ea1a2d80965 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/service.go @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +// ServiceGetResponse is a reponse of getting function names request by sn service id +type ServiceGetResponse struct { + Total int + FunctionNames []string +} diff --git a/functionsystem/apps/meta_service/function_repo/model/trigger.go b/functionsystem/apps/meta_service/function_repo/model/trigger.go new file mode 100644 index 0000000000000000000000000000000000000000..b4b417b3045a10990a9be5fde6d374a1674ca030 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/trigger.go @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +import ( + "encoding/json" + "time" + + "meta_service/function_repo/errmsg" +) + +const ( + // HTTPType is http trigger + HTTPType = "HTTP" +) + +// TriggerInfo is a reply of create request and update request +type TriggerInfo struct { + TriggerID string `json:"triggerId"` + FuncID string `json:"funcId"` + TriggerType string `json:"triggerType"` + RevisionID string `json:"revisionId"` + CreateTime time.Time `json:"createTime"` + UpdateTime time.Time `json:"updateTime"` + RawSpec json.RawMessage `json:"spec"` + Spec interface{} `json:"-"` +} + +// localTriggerCreateRequest prevents recursive call to MarshalJSON and UnmarshalJSON +type localTriggerInfo TriggerInfo + +// MarshalJSON implements json.Marshaler +func (t TriggerInfo) MarshalJSON() ([]byte, error) { + l := (*localTriggerInfo)(&t) + return triggerMarshalSpec(l, l.Spec, &l.RawSpec) +} + +// UnmarshalJSON implements json.Unmarshaler +func (t *TriggerInfo) UnmarshalJSON(b []byte) error { + l := (*localTriggerInfo)(t) + return triggerUnmarshalSpec(b, l, &l.Spec, &l.TriggerType, &l.RawSpec) +} + +// TriggerSpec contains common fields for every specific trigger specs +type TriggerSpec struct { + FuncID string `json:"funcId"` + TriggerID string `json:"triggerId"` + TriggerType string `json:"triggerType"` +} + +// HTTPTriggerSpec is a request of http trigger. +type HTTPTriggerSpec struct { + TriggerSpec + HTTPMethod string `json:"httpMethod"` + ResourceID string `json:"resourceId"` + AuthFlag bool `json:"authFlag"` + AuthAlgorithm string `json:"authAlgorithm"` + TriggerURL string `json:"triggerUrl"` + AppID string `json:"appId"` + AppSecret string `json:"appSecret"` +} + +// TriggerCreateRequest is the request body when creating a trigger. +type TriggerCreateRequest struct { + FuncID string `json:"funcId" valid:"maxstringlength(255)"` + TriggerType string `json:"triggerType" valid:"required"` + RawSpec json.RawMessage `json:"spec"` + Spec interface{} `json:"-"` +} + +// localTriggerCreateRequest prevents recursive call to MarshalJSON and UnmarshalJSON +type localTriggerCreateRequest TriggerCreateRequest + +// MarshalJSON implements json.Marshaler +func (t TriggerCreateRequest) MarshalJSON() ([]byte, error) { + l := (*localTriggerCreateRequest)(&t) + return triggerMarshalSpec(l, l.Spec, &l.RawSpec) +} + +// UnmarshalJSON implements json.Unmarshaler +func (t *TriggerCreateRequest) UnmarshalJSON(b []byte) error { + l := (*localTriggerCreateRequest)(t) + return triggerUnmarshalSpec(b, l, &l.Spec, &l.TriggerType, &l.RawSpec) +} + +// TriggerResponse is the response when creating ,getting or updating a trigger. +type TriggerResponse struct { + TriggerInfo +} + +// TriggerListGetResponse is the response when getting triggers. +type TriggerListGetResponse struct { + Count int `json:"total"` + TriggerList []TriggerInfo `json:"triggers"` +} + +// TriggerUpdateRequest is the request body when updating a trigger. +type TriggerUpdateRequest struct { + TriggerID string `json:"triggerId" valid:"maxstringlength(64)"` + RevisionID string `json:"revisionId"` + TriggerMode string `json:"triggerMode"` + TriggerType string `json:"triggerType"` + RawSpec json.RawMessage `json:"spec"` + Spec interface{} `json:"-"` +} + +// localTriggerUpdateRequest prevents recursive call to MarshalJSON and UnmarshalJSON +type localTriggerUpdateRequest TriggerUpdateRequest + +// MarshalJSON implements json.Marshaler +func (t TriggerUpdateRequest) MarshalJSON() ([]byte, error) { + l := (*localTriggerUpdateRequest)(&t) + return triggerMarshalSpec(l, l.Spec, &l.RawSpec) +} + +// UnmarshalJSON implements json.Unmarshaler +func (t *TriggerUpdateRequest) UnmarshalJSON(b []byte) error { + l := (*localTriggerUpdateRequest)(t) + return triggerUnmarshalSpec(b, l, &l.Spec, &l.TriggerType, &l.RawSpec) +} + +func triggerMarshalSpec(v, spec interface{}, raw *json.RawMessage) ([]byte, error) { + b, err := json.Marshal(spec) + if err != nil { + return nil, err + } + *raw = json.RawMessage(b) + return json.Marshal(v) +} + +func triggerUnmarshalSpec(b []byte, v interface{}, spec *interface{}, typ *string, raw *json.RawMessage) error { + if err := json.Unmarshal(b, v); err != nil { + return err + } + + switch *typ { + case HTTPType: + *spec = &HTTPTriggerSpec{} + default: + return errmsg.NewParamError("invalid triggertype: %s", *typ) + } + if err := json.Unmarshal(*raw, spec); err != nil { + return errmsg.NewParamError(err.Error()) + } + return nil +} diff --git a/functionsystem/apps/meta_service/function_repo/model/trigger_test.go b/functionsystem/apps/meta_service/function_repo/model/trigger_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d73a582e471d7a939e3023ba9380e5ec9f80815c --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/model/trigger_test.go @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package model + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/goconvey/convey" +) + +func TestTriggerMarshal(t *testing.T) { + convey.Convey("TestValidateAlias", t, func() { + convey.Convey("test trigger create request", func() { + r := &TriggerCreateRequest{ + TriggerType: HTTPType, + } + ret, err := r.MarshalJSON() + convey.ShouldNotBeNil(err) + err = r.UnmarshalJSON(ret) + convey.ShouldNotBeNil(err) + }) + + convey.Convey("test trigger info", func() { + r := &TriggerInfo{} + ret, err := r.MarshalJSON() + convey.ShouldNotBeNil(err) + err = r.UnmarshalJSON(ret) + convey.ShouldNotBeNil(err) + }) + + convey.Convey("test trigger update request", func() { + r := &TriggerUpdateRequest{} + ret, err := r.MarshalJSON() + convey.ShouldNotBeNil(err) + err = r.UnmarshalJSON(ret) + convey.ShouldNotBeNil(err) + }) + }) +} + +func TestTriggerMarshalFailed(t *testing.T) { + convey.Convey("TestValidateAliasFailed", t, func() { + convey.Convey("test trigger create request failed", func() { + r := &TriggerCreateRequest{} + + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(json.Marshal, func(v interface{}) ([]byte, error) { + return []byte("aa"), errors.New("test error") + }) + patches.ApplyFunc(json.Unmarshal, func(data []byte, v interface{}) error { + return errors.New("test error") + }) + + ret, err := r.MarshalJSON() + convey.ShouldNotBeNil(err) + err = r.UnmarshalJSON(ret) + convey.ShouldNotBeNil(err) + }) + + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/interface.go b/functionsystem/apps/meta_service/function_repo/pkgstore/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..363c338de545b9e30a73058f8540d1853536b663 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/interface.go @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package pkgstore handles the upload and deletion of function code packages and layer packages +package pkgstore + +import ( + "errors" + "io" + "time" + + "meta_service/common/logger/log" + commonObs "meta_service/common/obs" + "meta_service/function_repo/config" + "meta_service/function_repo/server" +) + +const ( + // defaultMaxRetryCount default obs max retry count + defaultMaxRetryCount = 0 +) + +// PackageBuilder can create a new package. +type PackageBuilder interface { + // NewPackage creates a new package. name is the package name. reader is the input byte stream, n is reader's + // expected length. + NewPackage(c server.Context, name string, reader io.Reader, n int64) (Package, error) +} + +// Package represents a function code package or a layer package. +type Package interface { + // Name is the package name. + Name() string + + // FileName is the file of a package. + FileName() string + + // Move gives up its ownership of the underlying file and returns the file name. + Move() string + + // Signature returns a hash value that identifies the package. A typical signature algorithm is sha256. + Signature() string + + // Size returns the package file size. + Size() int64 + + // Close cleans up temporary resources holds by the package. + Close() error +} + +// UploaderBuilder can create a new uploader +type UploaderBuilder interface { + NewUploader(c server.Context, pkg Package) (Uploader, error) +} + +// Uploader is responsible for uploading a package. +type Uploader interface { + BucketID() string + ObjectID() string + + // Upload uploads the package. In case of error, user can call Rollback to undo. + Upload() error + + // Rollback undoes the upload. + Rollback() error +} + +// BucketChooser implements strategies choosing buckets for different businesses. +type BucketChooser interface { + Choose(businessID string) (string, error) + Find(businessID string, bucketID string) (config.BucketConfig, error) +} + +// Manager manages packages +type Manager interface { + Delete(bucketName, objectName string) error +} + +var ( + defaultPackageBuilder PackageBuilder + defaultUploaderBuilder UploaderBuilder = noopUploaderBuilder{} + defaultBucketChooser BucketChooser = noopChooser{} + defaultManager Manager = noopManager{} +) + +// Init inits the package. User should only call it ONCE. +func Init() error { + var err error + defaultPackageBuilder, err = newZipPackageBuilder(config.RepoCfg.FunctionCfg.PackageCfg) + if err != nil { + log.GetLogger().Errorf("failed to create zip package builder: %s", err.Error()) + return err + } + + switch config.RepoCfg.FileServer.StorageType { + case "local": + return nil + case "s3": + obsCfg := config.RepoCfg.FileServer.S3 + conf := commonObs.Option{ + Secure: obsCfg.Secure, + AccessKey: obsCfg.AccessKey, + SecretKey: obsCfg.SecretKey, + Endpoint: obsCfg.Endpoint, + CaFile: obsCfg.CaFile, + TrustedCA: obsCfg.TrustedCA, + MaxRetryCount: defaultMaxRetryCount, + Timeout: int(time.Duration(obsCfg.Timeout)), + } + cli, err := commonObs.NewObsClient(conf) + commonObs.ClearSecretKeyMemory() + if err != nil { + return err + } + + defaultBucketChooser = newObsBucketChooser(config.RepoCfg.BucketCfg) + defaultUploaderBuilder = newS3UploaderBuilder(cli, true, defaultBucketChooser) + defaultManager = newS3Manager(cli, time.Duration(obsCfg.URLExpires)) + return nil + default: + return errors.New("unknown storage type") + } +} + +// NewPackage calls defaultPackageBuilder.NewPackage. +func NewPackage(c server.Context, name string, reader io.Reader, n int64) (Package, error) { + return defaultPackageBuilder.NewPackage(c, name, reader, n) +} + +// NewUploader calls defaultUploaderBuilder.NewUploader. +func NewUploader(c server.Context, pkg Package) (Uploader, error) { + return defaultUploaderBuilder.NewUploader(c, pkg) +} + +// FindBucket calls defaultBucketChooser.Find. +func FindBucket(businessID string, bucketID string) (config.BucketConfig, error) { + return defaultBucketChooser.Find(businessID, bucketID) +} + +// Delete calls defaultManager.Delete. +func Delete(bucketID, objectID string) error { + return defaultManager.Delete(bucketID, objectID) +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/interface_test.go b/functionsystem/apps/meta_service/function_repo/pkgstore/interface_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6454f6e5bb07a8f927403e281be3c43abfcad620 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/interface_test.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package pkgstore handles the upload and deletion of function code packages and layer packages +package pkgstore + +import ( + "testing" + + "github.com/magiconair/properties/assert" +) + +func TestNewUploader(t *testing.T) { + uploader, _ := NewUploader(nil, nil) + bucketID := uploader.BucketID() + assert.Equal(t, bucketID, "") + objectID := uploader.ObjectID() + assert.Equal(t, objectID, "") + upLoad := uploader.Upload() + assert.Equal(t, upLoad, nil) + rollback := uploader.Rollback() + assert.Equal(t, rollback, nil) +} + +func TestNoopChooser(t *testing.T) { + chooser := noopChooser{} + r, _ := chooser.Choose("") + assert.Equal(t, r, "") + _, err := chooser.Find("", "") + assert.Equal(t, err, nil) +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/noop_manager.go b/functionsystem/apps/meta_service/function_repo/pkgstore/noop_manager.go new file mode 100644 index 0000000000000000000000000000000000000000..30b7605486cb8d038bcb057436e691a2a4881fb6 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/noop_manager.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package pkgstore handles the upload and deletion of function code packages and layer packages +package pkgstore + +import ( + "meta_service/function_repo/config" +) + +type noopChooser struct{} + +// Choose implements BucketChooser +func (noopChooser) Choose(businessID string) (string, error) { + return "", nil +} + +// Find implements BucketChooser +func (noopChooser) Find(businessID string, bucketID string) (config.BucketConfig, error) { + return config.BucketConfig{}, nil +} + +type noopManager struct{} + +// Delete implements Manager +func (noopManager) Delete(bucketName, objectName string) error { + return nil +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/noop_uploader.go b/functionsystem/apps/meta_service/function_repo/pkgstore/noop_uploader.go new file mode 100644 index 0000000000000000000000000000000000000000..3bf003f869197fafa288c9a53cc22765cbf838ba --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/noop_uploader.go @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pkgstore + +import ( + "meta_service/function_repo/server" +) + +type noopUploaderBuilder struct{} + +// NewUploader implements UploaderBuilder +func (noopUploaderBuilder) NewUploader(c server.Context, pkg Package) (Uploader, error) { + return noopUploader{}, nil +} + +type noopUploader struct{} + +// BucketID implements Uploader +func (noopUploader) BucketID() string { + return "" +} + +// ObjectID implements Uploader +func (noopUploader) ObjectID() string { + return "" +} + +// Upload implements Uploader +func (noopUploader) Upload() error { + return nil +} + +// Rollback implements Uploader +func (noopUploader) Rollback() error { + return nil +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/obs_manager.go b/functionsystem/apps/meta_service/function_repo/pkgstore/obs_manager.go new file mode 100644 index 0000000000000000000000000000000000000000..71c4611048d2db5685f805dc057fa709c2c7901e --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/obs_manager.go @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package pkgstore handles the upload and deletion of function code packages and layer packages +package pkgstore + +import ( + "crypto/rand" + "math/big" + "time" + + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + + "meta_service/common/logger/log" + "meta_service/common/snerror" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" +) + +type obsBucketChooser struct { + m map[string][]config.BucketConfig +} + +func newObsBucketChooser(buckets []config.BucketConfig) BucketChooser { + res := &obsBucketChooser{ + m: make(map[string][]config.BucketConfig, len(buckets)), + } +outer: + for _, b := range buckets { + v, exist := res.m[b.BusinessID] + if !exist { + res.m[b.BusinessID] = []config.BucketConfig{b} + continue + } + // skip duplicates + for _, bb := range v { + if bb.BucketID == b.BucketID { + continue outer + } + } + + v = append(v, b) + res.m[b.BusinessID] = v + } + return res +} + +// Choose implements BucketChooser +func (c *obsBucketChooser) Choose(businessID string) (string, error) { + v, ok := c.m[businessID] + if !ok { + log.GetLogger().Errorf("failed to find bucket for business id %s", businessID) + return "", snerror.NewWithFmtMsg( + errmsg.BucketNotFound, "failed to find bucket for business id %s", businessID) + } + res := v[getRandomNum(len(v))].BucketID + return res, nil +} + +func getRandomNum(n int) int { + randomIndex, err := rand.Int(rand.Reader, big.NewInt(int64(n))) + if err != nil { + log.GetLogger().Errorf("failed to generate random num %s", err.Error()) + return 0 + } + return int(randomIndex.Int64()) +} + +// Find implements BucketChooser +func (c *obsBucketChooser) Find(businessID string, bucketID string) (config.BucketConfig, error) { + v, ok := c.m[businessID] + if !ok { + log.GetLogger().Errorf("failed to find bucket for business id %s", businessID) + return config.BucketConfig{}, snerror.NewWithFmtMsg( + errmsg.BucketNotFound, "failed to find bucket for business id %s", businessID) + } + for _, conf := range v { + if conf.BucketID == bucketID { + return conf, nil + } + } + log.GetLogger().Errorf("failed to find bucket for business id %s and bucket id %s", businessID, bucketID) + return config.BucketConfig{}, snerror.NewWithFmtMsg( + errmsg.BucketNotFound, "failed to find bucket for business id %s and bucket id %s", businessID, bucketID) +} + +type s3Manager struct { + cli *obs.ObsClient + urlExpires time.Duration +} + +func newS3Manager(cli *obs.ObsClient, urlExpires time.Duration) *s3Manager { + return &s3Manager{ + cli: cli, + urlExpires: urlExpires, + } +} + +// Delete implements Manager +func (m *s3Manager) Delete(bucketName, objectName string) error { + if bucketName == "" || objectName == "" { + log.GetLogger().Warnf("deleting with empty bucketName or empty objectName") + return nil + } + + deleteParam := &obs.DeleteObjectInput{ + Bucket: bucketName, + Key: objectName, + } + _, err := m.cli.DeleteObject(deleteParam) + if err != nil { + log.GetLogger().Errorf("failed to delete object url from obs with bucketName %s, objectName %s: %s", + bucketName, objectName, err.Error()) + return snerror.NewWithError(errmsg.DeleteFileError, err) + } + return nil +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/obs_test.go b/functionsystem/apps/meta_service/function_repo/pkgstore/obs_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b7a7a3c7211123f73316edcd49e3100d47b5b226 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/obs_test.go @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pkgstore + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "meta_service/common/crypto" + commonObs "meta_service/common/obs" + "meta_service/function_repo/config" + "meta_service/function_repo/server" + "meta_service/function_repo/test/fakecontext" +) + +func TestObsBucketChooser(t *testing.T) { + chooser := newObsBucketChooser([]config.BucketConfig{ + { + BusinessID: "a", + BucketID: "xx", + }, + { + BusinessID: "b", + BucketID: "yy", + }, + { + BusinessID: "b", + BucketID: "zz", + }, + }) + + bucketID, err := chooser.Choose("a") + require.NoError(t, err) + assert.Equal(t, "xx", bucketID) + + _, err = chooser.Choose("not exist") + assert.Error(t, err) + + cfg, err := chooser.Find("b", "yy") + require.NoError(t, err) + assert.Equal(t, "b", cfg.BusinessID) + assert.Equal(t, "yy", cfg.BucketID) + + _, err = chooser.Find("b", "not exist") + assert.Error(t, err) + + _, err = chooser.Find("not exist", "not exist") + assert.Error(t, err) +} + +func TestObs(t *testing.T) { + fakeClient := &obs.ObsClient{} + var err error + + opt := commonObs.Option{ + Endpoint: "http://127.0.0.1:9000", + Secure: false, + AccessKey: "minioadmin", + SecretKey: "minioadmin", + MaxRetryCount: 3, + Timeout: 60, + } + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(crypto.Decrypt, func(cipherText []byte, secret []byte) (string, error) { + return "", nil + }), + gomonkey.ApplyFunc(commonObs.NewObsClient, func(o commonObs.Option) (*obs.ObsClient, error) { + return fakeClient, nil + }), + } + defer func() { + for index := range patches { + patches[index].Reset() + } + }() + fakeClient, _ = commonObs.NewObsClient(opt) + config.RepoCfg = &config.Configs{} + config.RepoCfg.FileServer.StorageType = "s3" + config.RepoCfg.FunctionCfg.PackageCfg = config.PackageConfig{ + UploadTmpPath: os.TempDir(), + ZipFileSizeMaxMB: 100, + UnzipFileSizeMaxMB: 100, + FileCountsMax: 100, + DirDepthMax: 100, + IOReadTimeout: 1000000, + } + config.RepoCfg.BucketCfg = []config.BucketConfig{ + { + BusinessID: "aa", + BucketID: "xxxx", + }, + } + + c := fakecontext.NewContext() + c.InitTenantInfo(server.TenantInfo{ + BusinessID: "aa", + TenantID: "", + ProductID: "", + }) + + // start + err = Init() + require.NoError(t, err) + + b, err := newZipFile([]zipFile{ + {filepath.Join("example", "readme.txt"), "This archive contains some text files."}, + {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, + {"todo.txt", "Get animal handling licence.\nWrite more examples."}, + }) + require.NoError(t, err, "create zip file") + + zipbuf := bytes.NewBuffer(b) + size := int64(zipbuf.Len()) + _, err = NewPackage(c, "obs-test", zipbuf, size) + require.NoError(t, err) +} + +func TestBucketID(t *testing.T) { + s3Uploader := s3Uploader{ + bucketName: "test", + objectName: "testObject", + } + assert.Equal(t, s3Uploader.BucketID(), s3Uploader.bucketName) + assert.Equal(t, s3Uploader.ObjectID(), s3Uploader.objectName) +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/obs_uploader.go b/functionsystem/apps/meta_service/function_repo/pkgstore/obs_uploader.go new file mode 100644 index 0000000000000000000000000000000000000000..af71928bd3790fa1945fd3349d9b095fa1e09148 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/obs_uploader.go @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pkgstore + +import ( + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + + "meta_service/common/logger/log" + "meta_service/common/reader" + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" +) + +const obsBucketNotExistCode = 404 + +type s3UploaderBuilder struct { + cli *obs.ObsClient + createBucketIfNotExist bool + chooser BucketChooser +} + +func newS3UploaderBuilder( + cli *obs.ObsClient, createBucketIfNotExist bool, chooser BucketChooser) UploaderBuilder { + + res := &s3UploaderBuilder{ + cli: cli, + createBucketIfNotExist: createBucketIfNotExist, + chooser: chooser, + } + return res +} + +// NewUploader implements PackageBuilder +func (b *s3UploaderBuilder) NewUploader(c server.Context, pkg Package) (Uploader, error) { + tenant, err := c.TenantInfo() + if err != nil { + return nil, err + } + + bucketName, err := b.chooser.Choose(tenant.BusinessID) + if err != nil { + return nil, err + } + + return &s3Uploader{ + cli: b.cli, + bucketName: bucketName, + objectName: pkg.Name(), + filePath: pkg.FileName(), + createBucketIfNotExist: b.createBucketIfNotExist, + }, nil +} + +type s3Uploader struct { + cli *obs.ObsClient + bucketName string + objectName string + filePath string + createBucketIfNotExist bool +} + +// BucketID implements Uploader +func (u *s3Uploader) BucketID() string { + return u.bucketName +} + +// ObjectID implements Uploader +func (u *s3Uploader) ObjectID() string { + return u.objectName +} + +func (u *s3Uploader) createBucket(err error) error { + if obsError, ok := err.(obs.ObsError); ok { + if obsError.StatusCode == obsBucketNotExistCode { + log.GetLogger().Infof("bucket %s does not exist", u.bucketName) + _, err := u.cli.CreateBucket(&obs.CreateBucketInput{ + Bucket: u.bucketName, + }) + if err != nil { + log.GetLogger().Errorf("failed to create bucket in S3, "+ + "bucketName: %s, objectName: %s, uploadFilePath: %s: %s", + u.bucketName, u.objectName, u.filePath, err.Error()) + return snerror.NewWithError(errmsg.UploadFileError, err) + } + } else { + log.GetLogger().Errorf("failed to check bucket exists in S3, "+ + "bucketName: %s, objectName: %s, uploadFilePath: %s: %s", + u.bucketName, u.objectName, u.filePath, err.Error()) + return snerror.NewWithError(errmsg.UploadFileError, err) + } + } else { + return snerror.NewWithError(errmsg.UploadFileError, err) + } + return nil +} + +// Upload implements Uploader +func (u *s3Uploader) Upload() error { + if u.createBucketIfNotExist { + _, err := u.cli.HeadBucket(u.bucketName) + if err != nil { + err := u.createBucket(err) + if err != nil { + return err + } + } + } + fileInfo, err := reader.ReadFileInfoWithTimeout(u.filePath) + if err != nil { + log.GetLogger().Errorf("failed to get file size. path: %s, error: %s", + u.filePath, err.Error()) + return snerror.NewWithError(errmsg.UploadFileError, err) + } + + fileInput := &obs.PutFileInput{} + fileInput.Bucket = u.bucketName + fileInput.Key = u.objectName + fileInput.SourceFile = u.filePath + fileInput.ContentLength = fileInfo.Size() + _, err = u.cli.PutFile(fileInput) + if err != nil { + log.GetLogger().Errorf("failed to writing to S3, "+ + "bucketName: %s, objectName: %s, uploadFilePath: %s: %s", + u.bucketName, u.objectName, u.filePath, err.Error()) + return snerror.NewWithError(errmsg.UploadFileError, err) + } + return nil +} + +// Rollback implements Uploader +func (u *s3Uploader) Rollback() error { + deleteParam := &obs.DeleteObjectInput{} + deleteParam.Bucket = u.bucketName + deleteParam.Key = u.objectName + _, err := u.cli.DeleteObject(&obs.DeleteObjectInput{ + Bucket: u.bucketName, + Key: u.objectName, + }) + if err != nil { + log.GetLogger().Errorf("failed to delete object from S3, "+ + "bucketName: %s, objectName: %s, uploadFilePath: %s: %s", + u.bucketName, u.objectName, u.filePath, err.Error()) + return snerror.NewWithError(errmsg.DeleteFileError, err) + } + return nil +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/zip_package.go b/functionsystem/apps/meta_service/function_repo/pkgstore/zip_package.go new file mode 100644 index 0000000000000000000000000000000000000000..a64500af404d691da924abb761148663dad07abb --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/zip_package.go @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pkgstore + +import ( + "archive/zip" + "crypto/sha512" + "encoding/hex" + "errors" + "hash" + "io" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "golang.org/x/text/unicode/norm" + + "meta_service/common/logger/log" + "meta_service/common/reader" + "meta_service/common/snerror" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" +) + +var ( + defaultHasher = sha512.New + + zipFileDirRegex *regexp.Regexp = regexp.MustCompile("") + + mbShift = 20 +) + +const uploadPathMode = 0700 + +type zipPackageBuilder struct { + TempFileDir string // must be absolute + TempFilePattern string + MaxZipFileSize int64 + MaxZipFileUnzipSize int64 + MaxZipFileCount int + MaxZipFileDepth int + MaxZipFileTimeout int + Hasher func() hash.Hash +} + +func newZipPackageBuilder(cfg config.PackageConfig) (PackageBuilder, error) { + if cfg.UploadTmpPath == "" { + cfg.UploadTmpPath = os.TempDir() + } else { + abs, err := filepath.Abs(cfg.UploadTmpPath) + if err != nil { + log.GetLogger().Errorf("invalid upload tmp path: %s", err.Error()) + return nil, err + } + if err := mkdirUploadPathIfNotExists(abs); err != nil { + log.GetLogger().Errorf("invalid upload tmp path: %s", err.Error()) + return nil, err + } + cfg.UploadTmpPath = abs + } + + return &zipPackageBuilder{ + TempFileDir: cfg.UploadTmpPath, + TempFilePattern: "function-repository*", + MaxZipFileSize: int64(cfg.ZipFileSizeMaxMB << mbShift), + MaxZipFileUnzipSize: int64(cfg.UnzipFileSizeMaxMB << mbShift), + MaxZipFileCount: int(cfg.FileCountsMax), + MaxZipFileDepth: int(cfg.DirDepthMax), + MaxZipFileTimeout: int(cfg.IOReadTimeout), + Hasher: defaultHasher, + }, nil +} + +func mkdirUploadPathIfNotExists(path string) error { + if _, err := reader.ReadFileInfoWithTimeout(path); err != nil { + if os.IsExist(err) { + return err + } + if err := os.Mkdir(path, uploadPathMode); err != nil { + return err + } + } + return nil +} + +// PackageWriter - +type PackageWriter struct { + StopCh <-chan time.Time + writerHash hash.Hash +} + +// Write processes the slices of the reading file by distinguish whether the timer stops or runs normally +func (pw *PackageWriter) Write(p []byte) (int, error) { + select { + case <-pw.StopCh: + return 0, snerror.New(errmsg.ReadingPackageTimeout, "timeout while writing a package") + default: + // Executes the normal reading logic + return pw.writerHash.Write(p) + } +} + +// NewPackage implements PackageBuilder +func (b *zipPackageBuilder) NewPackage(c server.Context, name string, reader io.Reader, n int64) (Package, error) { + if n > b.MaxZipFileSize { + log.GetLogger().Errorf("zip file size [%d] is larger than %d", n, b.MaxZipFileSize) + return nil, snerror.NewWithFmtMsg( + errmsg.ZipFileSizeError, "zip file size [%d] is larger than %d", n, b.MaxZipFileSize) + } + file, err := ioutil.TempFile(b.TempFileDir, b.TempFilePattern) + if err != nil { + return nil, snerror.New(errmsg.SaveFileError, "failed to create temp file") + } + defer func() { + err = file.Close() + if err != nil { + err2 := os.Remove(file.Name()) + if err2 != nil { + log.GetLogger().Errorf("clean temp file error: %s", err2.Error()) + } + } + }() + + h := b.Hasher() + timer := time.NewTimer(time.Duration(b.MaxZipFileTimeout) * time.Millisecond) + writer := &PackageWriter{ + StopCh: timer.C, + writerHash: h, + } + r := io.TeeReader(reader, writer) + if written, err := io.CopyN(file, r, n); err != nil { + var snErr snerror.SNError + if errors.As(err, &snErr) { + log.GetLogger().Errorf("failed to read a package file as meeting a timeout,"+ + " where the function name is %s", name) + return nil, err + } + log.GetLogger().Errorf("failed to write temp file: %s, written=%d, n=%d", err.Error(), written, n) + return nil, snerror.New(errmsg.SaveFileError, "failed to write temp file") + } + + if err = b.validateFile(file.Name()); err != nil { + log.GetLogger().Errorf("zip file %s is not valid: %s", name, err.Error()) + return nil, err + } + + return &zipPackage{ + name: name, + fileName: file.Name(), + signature: hex.EncodeToString(h.Sum(nil)), + size: n, + }, nil +} + +func (b *zipPackageBuilder) validateFile(filename string) error { + r, err := zip.OpenReader(filename) + if err != nil { + return snerror.NewWithError(errmsg.ZipFileError, err) + } + defer r.Close() + + if len(r.File) > b.MaxZipFileCount { + return snerror.NewWithFmtMsg( + errmsg.ZipFileCountError, "zip file count [%v] is larger than %v", len(r.File), b.MaxZipFileCount) + } + + var total int64 + for _, file := range r.File { + total += int64(file.UncompressedSize64) // assume int64 is big enough that this never wraps + if total > b.MaxZipFileUnzipSize { + return snerror.NewWithFmtMsg( + errmsg.ZipFileUnzipSizeError, "zip file unzip size is larger than %v", b.MaxZipFileUnzipSize) + } + + if !b.validateFilePath(filename, file.Name) { + return snerror.NewWithFmtMsg(errmsg.ZipFilePathError, "zip file path [%s] is invalid", file.Name) + } + } + + return nil +} + +func (b *zipPackageBuilder) validateFilePath(zipfile, path string) bool { + if path == "" || + strings.Contains(path, "~/") || + zipFileDirRegex == nil || + !zipFileDirRegex.MatchString(norm.NFC.String(path)) || + len(strings.Split(path, string(filepath.Separator))) > b.MaxZipFileDepth { + + return false + } + + abs, err := filepath.Abs(filepath.Join(zipfile, path)) + if err != nil { + return false + } + + return strings.HasPrefix(abs, b.TempFileDir) +} + +type zipPackage struct { + name string + fileName string + signature string + size int64 +} + +// Close implements Package +func (p *zipPackage) Close() error { + if p.fileName == "" { + return nil + } + + err := os.Remove(p.fileName) + if err != nil { + log.GetLogger().Errorf("failed to delete temporary file %s", p.fileName) + return snerror.NewWithFmtMsg(errmsg.ZipFileError, "failed to delete temporary file %s", p.fileName) + } + return nil +} + +// Name implements Package +func (p *zipPackage) Name() string { + return p.name +} + +// FileName implements Package +func (p *zipPackage) FileName() string { + return p.fileName +} + +// Move implements Package +func (p *zipPackage) Move() string { + fileName := p.fileName + p.fileName = "" + return fileName +} + +// Signature implements Package +func (p *zipPackage) Signature() string { + return p.signature +} + +// Size implements Package +func (p *zipPackage) Size() int64 { + return p.size +} diff --git a/functionsystem/apps/meta_service/function_repo/pkgstore/zip_package_test.go b/functionsystem/apps/meta_service/function_repo/pkgstore/zip_package_test.go new file mode 100644 index 0000000000000000000000000000000000000000..635d0eb503b848339ad33f3c6f300146694a75e4 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/pkgstore/zip_package_test.go @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pkgstore + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "errors" + "math" + "os" + "path/filepath" + "testing" + + "github.com/agiledragon/gomonkey" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "meta_service/function_repo/config" + "meta_service/function_repo/test/fakecontext" +) + +type zipFile struct { + Name, Body string +} + +func newZipFile(files []zipFile) ([]byte, error) { + buf := new(bytes.Buffer) + + w := zip.NewWriter(buf) + for _, file := range files { + f, err := w.Create(file.Name) + if err != nil { + return nil, err + } + + _, err = f.Write([]byte(file.Body)) + if err != nil { + return nil, err + } + } + + err := w.Close() + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func TestZipPackage(t *testing.T) { + b, err := newZipFile([]zipFile{ + {filepath.Join("example", "readme.txt"), "This archive contains some text files."}, + {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, + {"todo.txt", "Get animal handling licence.\nWrite more examples."}, + }) + require.NoError(t, err, "create zip file") + + zipbuf := bytes.NewBuffer(b) + c := fakecontext.NewContext() + builder, _ := newZipPackageBuilder(config.PackageConfig{ + UploadTmpPath: os.TempDir(), + ZipFileSizeMaxMB: 100, + UnzipFileSizeMaxMB: 100, + FileCountsMax: 100, + DirDepthMax: 100, + IOReadTimeout: 1000000, + }) + + size := int64(zipbuf.Len()) + pkg, err := builder.NewPackage(c, "test", zipbuf, size) + require.NoError(t, err, "new package") + assert.Equal(t, "test", pkg.Name(), "pkg name") + assert.Equal(t, size, pkg.Size(), "pkg size") + + defer pkg.Close() +} + +func TestZipPackageValidate(t *testing.T) { + files := []zipFile{ + {filepath.Join("example", "readme.txt"), "This archive contains some text files."}, + {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, + {"todo.txt", "Get animal handling licence.\nWrite more examples."}, + } + b, err := newZipFile(files) + require.NoError(t, err, "create zip file") + fileSize := int64(len(b)) + unzipSize := func() (total int64) { + for _, file := range files { + total += int64(len(file.Body)) + } + return + }() + + tests := []struct { + Name string + MaxZipFileSize int64 + MaxZipFileUnzipSize int64 + MaxZipFileCount int + MaxZipFileDepth int + NoError bool + }{ + { + Name: "MaxZipFileSize1", + MaxZipFileSize: fileSize, + MaxZipFileUnzipSize: math.MaxInt64, + MaxZipFileCount: 100, + MaxZipFileDepth: 100, + NoError: true, + }, + { + Name: "MaxZipFileSize1", + MaxZipFileSize: fileSize - 1, + MaxZipFileUnzipSize: math.MaxInt64, + MaxZipFileCount: 100, + MaxZipFileDepth: 100, + NoError: false, + }, + { + Name: "MaxZipFileUnzipSize1", + MaxZipFileSize: math.MaxInt64, + MaxZipFileUnzipSize: unzipSize, + MaxZipFileCount: 100, + MaxZipFileDepth: 100, + NoError: true, + }, + { + Name: "MaxZipFileUnzipSize2", + MaxZipFileSize: math.MaxInt64, + MaxZipFileUnzipSize: unzipSize - 1, + MaxZipFileCount: 100, + MaxZipFileDepth: 100, + NoError: false, + }, + { + Name: "MaxZipFileCount", + MaxZipFileSize: math.MaxInt64, + MaxZipFileUnzipSize: math.MaxInt64, + MaxZipFileCount: len(files), + MaxZipFileDepth: 100, + NoError: true, + }, + { + Name: "MaxZipFileCount", + MaxZipFileSize: math.MaxInt64, + MaxZipFileUnzipSize: math.MaxInt64, + MaxZipFileCount: len(files) - 1, + MaxZipFileDepth: 100, + NoError: false, + }, + { + Name: "MaxZipFileCount", + MaxZipFileSize: math.MaxInt64, + MaxZipFileUnzipSize: math.MaxInt64, + MaxZipFileCount: 100, + MaxZipFileDepth: 2, + NoError: true, + }, + { + Name: "MaxZipFileCount", + MaxZipFileSize: math.MaxInt64, + MaxZipFileUnzipSize: math.MaxInt64, + MaxZipFileCount: 100, + MaxZipFileDepth: 1, + NoError: false, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + zipbuf := bytes.NewBuffer(b) + c := fakecontext.NewContext() + builder := &zipPackageBuilder{ + TempFileDir: os.TempDir(), + TempFilePattern: "function-repository-test", + MaxZipFileSize: test.MaxZipFileSize, + MaxZipFileUnzipSize: test.MaxZipFileUnzipSize, + MaxZipFileCount: test.MaxZipFileCount, + MaxZipFileDepth: test.MaxZipFileDepth, + Hasher: sha256.New, + MaxZipFileTimeout: 10000, + } + pkg, err := builder.NewPackage(c, "test", zipbuf, int64(zipbuf.Len())) + if test.NoError { + assert.NoError(t, err, "new package") + defer pkg.Close() + } else { + assert.Error(t, err, "new package") + } + }) + } +} + +func Test_mkdirUploadPathIfNotExists(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(os.Stat, func(_ string) (os.FileInfo, error) { return nil, errors.New("fake error! ") }) + assert.NotNil(t, mkdirUploadPathIfNotExists("test/path")) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/alias.go b/functionsystem/apps/meta_service/function_repo/router/alias.go new file mode 100644 index 0000000000000000000000000000000000000000..dad37755a336769d5d61e23576f1fe463c2f050d --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/alias.go @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "strconv" + + "meta_service/common/logger/log" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/utils" + "meta_service/function_repo/utils/constants" +) + +func regAliasHandler(r server.RouterGroup) { + r.POST("/:functionName/aliases", createFunctionAlias) + r.DELETE("/:functionName/aliases/:name", deleteFunctionAlias) + r.GET("/:functionName/aliases/:name", getFunctionAlias) + r.GET("/:functionName/aliases", getFunctionAliasesList) + r.PUT("/:functionName/aliases/:name", updateFunctionAlias) +} + +func createFunctionAlias(c server.Context) { + fName := c.Gin().Param("functionName") + info, err := service.ParseFunctionInfo(c, fName, "") + if err != nil { + log.GetLogger().Errorf("failed to get function query info, error: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + var req model.AliasRequest + if err := shouldBindJSON(c, &req); err != nil { + log.GetLogger().Errorf("failed to parse request, error: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + log.GetLogger().Infof("create alias req: %+v", req) + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionName, + Qualifier: req.Name, + }) + resp, err := service.CreateAlias(c, info.FunctionName, req) + if err != nil { + log.GetLogger().Errorf("failed to create alias, error: %s", err.Error()) + utils.BadRequest(c, err) + return + } + utils.WriteResponse(c, resp, err) +} + +func updateFunctionAlias(c server.Context) { + fName := c.Gin().Param("functionName") + aName := c.Gin().Param("name") + info, err := service.ParseFunctionInfo(c, fName, "") + if err != nil { + log.GetLogger().Errorf("failed to get function query info, error: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + var req model.AliasUpdateRequest + if err := shouldBindJSON(c, &req); err != nil { + log.GetLogger().Errorf("failed to parse request, error: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + log.GetLogger().Infof("update alias req: %+v", req) + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionName, + Qualifier: aName, + }) + + resp, err := service.UpdateAlias(c, info.FunctionName, aName, req) + utils.WriteResponse(c, resp, err) +} + +func deleteFunctionAlias(c server.Context) { + fName := c.Gin().Param("functionName") + aName := c.Gin().Param("name") + info, err := service.ParseFunctionInfo(c, fName, "") + if err != nil { + utils.BadRequest(c, err) + return + } + + req := model.AliasDeleteRequest{ + FunctionName: info.FunctionName, + AliasName: aName, + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionName, + Qualifier: aName, + }) + err = service.DeleteAlias(c, req) + utils.WriteResponse(c, nil, err) +} + +func getFunctionAlias(c server.Context) { + fname := c.Gin().Param("functionName") + aname := c.Gin().Param("name") + info, err := service.ParseFunctionInfo(c, fname, "") + if err != nil { + utils.BadRequest(c, err) + return + } + + req := model.AliasQueryRequest{ + FunctionName: info.FunctionName, + AliasName: aname, + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionName, + Qualifier: info.FunctionVersion, + }) + resp, err := service.GetAlias(c, req) + utils.WriteResponse(c, resp, err) +} + +func getFunctionAliasesList(c server.Context) { + fname := c.Gin().Param("functionName") + fversion := c.Gin().Query("functionVersion") + + pageIndexStr := c.Gin().DefaultQuery("pageIndex", constants.DefaultPageIndex) + pageIndex, err := strconv.Atoi(pageIndexStr) + if err != nil { + utils.BadRequest(c, err) + return + } + pageSizeStr := c.Gin().DefaultQuery("pageSize", constants.DefaultPageSize) + pageSize, err := strconv.Atoi(pageSizeStr) + if err != nil || pageSize > maxPageSize || pageSize < minPageSize { + utils.BadRequest(c, err) + return + } + + info, err := service.ParseFunctionInfo(c, fname, "") + if err != nil { + utils.BadRequest(c, err) + return + } + + req := model.AliasListQueryRequest{ + FunctionName: info.FunctionName, + FunctionVersion: fversion, + PageIndex: pageIndex, + PageSize: pageSize, + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionVersion, + Qualifier: "", + }) + resp, err := service.GetAliaseList(c, req) + utils.WriteResponse(c, resp, err) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/alias_test.go b/functionsystem/apps/meta_service/function_repo/router/alias_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0c366461d7176c1bdc2197cbe2552621baa5db2b --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/alias_test.go @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package router + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "testing" + + "meta_service/function_repo/errmsg" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" + + "meta_service/common/snerror" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" +) + +func Test_createFunctionAlias(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(service.CreateAlias, + func(ctx server.Context, fName string, req model.AliasRequest) ( + resp model.AliasCreateResponse, err error) { + return model.AliasCreateResponse{}, nil + }) + defer patches.Reset() + + for _, test := range []struct { + name string + method string + path string + body string + expectedCode int + expectedMsg string + }{ + { + "test upload function code", + "POST", + "/function-repository/v1/functions/0-test-aa/aliases", + ` + {"Name":"0-test-aa", + "FunctionVersion":"1", + "Description":"", + "RoutingConfig":{"1":20,"2":80}} + `, + http.StatusOK, + "", + }, + { + "test upload function code", + "POST", + "/function-repository/v1/functions/0-test-aa/aliases", + ``, + errmsg.InvalidJSONBody, + "", + }, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, test.method, test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_FunctionAlias(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(service.UpdateAlias, + func(ctx server.Context, fName, aName string, req model.AliasUpdateRequest) ( + resp model.AliasUpdateResponse, err error) { + return model.AliasUpdateResponse{}, nil + }) + patches.ApplyFunc(service.DeleteAlias, + func(ctx server.Context, req model.AliasDeleteRequest) (err error) { + return nil + }) + patches.ApplyFunc(service.GetAlias, + func(ctx server.Context, req model.AliasQueryRequest) (resp model.AliasQueryResponse, err error) { + return model.AliasQueryResponse{}, nil + }) + patches.ApplyFunc(service.GetAliaseList, + func(ctx server.Context, req model.AliasListQueryRequest) ( + resp model.AliasListQueryResponse, err error) { + return model.AliasListQueryResponse{}, nil + }) + defer patches.Reset() + + for _, test := range []struct { + name string + method string + path string + body string + expectedCode int + expectedMsg string + }{ + { + "test upload function code", + "PUT", + "/function-repository/v1/functions/0-test-aa/aliases/bb", + ` + {"RevisionID":"aa", + "FunctionVersion":"1", + "Description":"", + "RoutingConfig":{"1":20,"2":80}} + `, + http.StatusOK, + "", + }, + { + "test upload function code", + "PUT", + "/function-repository/v1/functions/0-test-aa/aliases/bb", + ``, + errmsg.InvalidJSONBody, + "", + }, + { + "test upload function code", + "DELETE", + "/function-repository/v1/functions/0-test-aa/aliases/bb", + ``, + http.StatusOK, + "", + }, + { + "test upload function code", + "GET", + "/function-repository/v1/functions/0-test-aa/aliases", + ``, + http.StatusOK, + "", + }, + { + "test upload function code", + "GET", + "/function-repository/v1/functions/0-test-aa/aliases/bb", + ``, + http.StatusOK, + "", + }, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, test.method, test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} diff --git a/functionsystem/apps/meta_service/function_repo/router/function.go b/functionsystem/apps/meta_service/function_repo/router/function.go new file mode 100644 index 0000000000000000000000000000000000000000..b6f61450b04182eaada82ccd0739249deca7f45d --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/function.go @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "errors" + "strconv" + "strings" + + common "meta_service/common/constants" + "meta_service/common/logger/log" + "meta_service/function_repo/config" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/utils" + "meta_service/function_repo/utils/constants" +) + +const ( + contentTypeMaxLen int = 100 + tokenSplit int = 2 +) + +func regFunctionHandlers(r server.RouterGroup) { + r.DELETE("/:functionName/code", deleteFunctionCode) + r.PUT("/:functionName/code", uploadFunctionCode) +} + +func deleteFunctionCode(c server.Context) { + fname := c.Gin().Param("functionName") + // not set default + qualifier := c.Gin().Query("qualifier") + versionNumber := c.Gin().Query("versionNumber") + kind := c.Gin().Query("kind") + info, err := service.ParseFunctionInfo(c, fname, qualifier) + if versionNumber != "" { + info.FunctionVersion = versionNumber + } + if err != nil { + log.GetLogger().Errorf("failed to get function query info :%s", err) + utils.BadRequest(c, err) + return + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionName, + Qualifier: info.FunctionVersion, + FunctionType: config.ComFunctionType, + }) + + err = service.DeleteResponse(c, info.FunctionName, info.FunctionVersion, kind) + utils.WriteResponse(c, nil, err) +} + +func getFunction(c server.Context) { + getFunctionInfo(c, config.ComFunctionType) +} + +func getFunctionInfo(c server.Context, functionType config.FunctionType) { + fname := c.Gin().Param("functionName") + qualifier := c.Gin().Query("qualifier") + versionNumber := c.Gin().Query("versionNumber") + kind := c.Gin().Query("kind") + log.GetLogger().Infof("get function info, fname: %s, qualifier: %s, versionNumber: %s, kind: %s", fname, + qualifier, versionNumber, kind) + info, err := service.ParseFunctionInfo(c, fname, qualifier) + if err != nil { + log.GetLogger().Errorf("failed to get function query info :%s", err) + utils.BadRequest(c, err) + return + } + if versionNumber != "" { + info.FunctionVersion = versionNumber + } + req := model.FunctionGetRequest{ + FunctionName: info.FunctionName, + FunctionVersion: info.FunctionVersion, + AliasName: info.AliasName, + Kind: kind, + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionVersion, + Qualifier: info.AliasName, + }) + c = initResourceInfo(c, info, functionType) + resp, err := service.GetFunction(c, req) + utils.WriteResponse(c, resp, err) +} + +func getFunctionList(c server.Context) { + fname := c.Gin().Query("functionName") + qualifier := c.Gin().Query("qualifier") + versionNumber := c.Gin().Query("versionNumber") + if versionNumber != "" { + // if versionNumber specified, use version only + qualifier = versionNumber + } + kind := c.Gin().Query("kind") + pageIndex, err := strconv.Atoi(c.Gin().DefaultQuery("pageIndex", constants.DefaultPageIndex)) + if err != nil { + log.GetLogger().Warnf("failed to get pageIndex") + } + pageSize, err := strconv.Atoi(c.Gin().DefaultQuery("pageSize", constants.DefaultPageSize)) + if err != nil { + log.GetLogger().Warnf("failed to get pageSize") + } + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: fname, + Qualifier: qualifier, + FunctionVersionNumber: versionNumber, + }) + log.GetLogger().Infof("get function list, fname: %s, qualifier: %s, kind: %s, pageIndex: %d, "+ + "pageSize: %d", + fname, qualifier, kind, pageIndex, pageSize) + resp, err := service.GetFunctionList(c, fname, qualifier, kind, pageIndex, pageSize) + utils.WriteResponse(c, resp, err) +} + +func uploadFunctionCode(c server.Context) { + uploadFunctionInfoCode(c, config.ComFunctionType) +} + +func uploadFunctionInfoCode(c server.Context, functionType config.FunctionType) { + fname := c.Gin().Param("functionName") + version := c.Gin().Request.Header.Get("X-Version") + kind := c.Gin().Request.Header.Get("X-Kind") + info, err := service.ParseFunctionInfo(c, fname, version) + if err != nil { + log.GetLogger().Errorf("failed to get function query info :%s", err) + utils.BadRequest(c, err) + return + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionName, + Qualifier: "", + FunctionType: functionType, + }) + + contentType := c.Gin().Request.Header.Get("Content-Type") + m, err := getFileInfo(contentType) + if err != nil { + log.GetLogger().Errorf("failed to get file info :%s", err) + utils.BadRequest(c, err) + return + } + + fileSize, err := strconv.ParseInt(m["file-size"], 10, 0) + if err != nil { + log.GetLogger().Errorf("failed to parse file size:%s", err) + utils.BadRequest(c, err) + return + } + req := model.FunctionCodeUploadRequest{ + RevisionID: m["revision-id"], + Kind: kind, + FileSize: fileSize, + } + err = utils.Validate(req) + if err != nil { + log.GetLogger().Errorf("failed to validate:%s", err) + utils.BadRequest(c, err) + return + } + resp, err := service.GetUploadFunctionCodeResponse(c, info, req) + utils.WriteResponse(c, resp, err) +} + +func initResourceInfo(c server.Context, info model.FunctionQueryInfo, + functionType config.FunctionType) server.Context { + resourceInfo := server.ResourceInfo{ + ResourceName: info.FunctionName, + } + if info.FunctionVersion != "" { + resourceInfo.Qualifier = info.FunctionVersion + } else { + resourceInfo.Qualifier = info.AliasName + } + resourceInfo.FunctionType = functionType + c.InitResourceInfo(resourceInfo) + return c +} + +func getFileInfo(contentType string) (map[string]string, error) { + if contentType == "" { + return nil, errors.New("the value of contentType is empty") + } + + tokens := strings.Split(contentType, ";") + + if len(tokens) > contentTypeMaxLen { + return nil, errors.New("the value of contentType is out of rang") + } + + m := make(map[string]string, common.DefaultMapSize) + for _, token := range tokens { + token = strings.TrimSpace(token) + if strings.HasPrefix(token, "application") { + continue + } + kv := strings.SplitN(token, "=", tokenSplit) + if len(kv) != tokenSplit { + continue // ignore + } + m[kv[0]] = kv[1] + } + return m, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/router/function_test.go b/functionsystem/apps/meta_service/function_repo/router/function_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ca8cc9b46773b32e6c6bce9ce58d3d4b8bfdd5c7 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/function_test.go @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" + + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" +) + +func Test_uploadFunctionCode(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(service.GetUploadFunctionCodeResponse, + func(ctx server.Context, info model.FunctionQueryInfo, + req model.FunctionCodeUploadRequest) (model.FunctionVersion, error) { + return model.FunctionVersion{}, nil + }) + defer patches.Reset() + + for _, test := range []struct { + name string + method string + path string + content string + body string + expectedCode int + expectedMsg string + }{ + {"test upload function code", "PUT", "/function-repository/v1/functions/0-test-aa/code", "application/vnd.yuanrong+attachment;file-size=0;revision-id=0", ``, http.StatusOK, ""}, + {"test upload function code", "PUT", "/function-repository/v1/functions/0-test-aa/code", "", ``, errmsg.InvalidUserParam, ""}, + {"test upload function code", "PUT", "/function-repository/v1/functions/0-test-aa/code", "application/vnd.yuanrong+attachment;file-size=a", ``, errmsg.InvalidUserParam, ""}, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + req := createRequest(t, test.method, test.path, body) + req.Header.Set("Content-Type", test.content) + rec := httptest.NewRecorder() + rec.Body = new(bytes.Buffer) + engine.ServeHTTP(rec, req) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} diff --git a/functionsystem/apps/meta_service/function_repo/router/interfacelog.go b/functionsystem/apps/meta_service/function_repo/router/interfacelog.go new file mode 100644 index 0000000000000000000000000000000000000000..4c5bb455b948555ccf37823922b858736d61a919 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/interfacelog.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package router +package router + +import ( + "strings" + "sync" + + "meta_service/common/logger" + "meta_service/common/logger/log" +) + +const ( + splitStr = "|" +) + +var ( + once sync.Once + interfaceLogger *logger.InterfaceLogger +) + +// InterfaceLog interface log +type interfaceLog struct { + httpMethod string + ip string + requestPath string + query string + bussinessID string + traceID string + retCode string + costTime string +} + +func initInterfaceLog() { + cfg := logger.InterfaceEncoderConfig{ModuleName: "function-repository"} + var err error + interfaceLogger, err = logger.NewInterfaceLogger("", "function-repository-interface", cfg) + if err != nil { + log.GetLogger().Errorf("failed to init interface log") + } +} + +func write(i interfaceLog) { + logStr := []string{i.httpMethod, i.ip, i.requestPath, i.query, i.bussinessID, i.traceID, i.retCode, i.costTime} + + once.Do(func() { + if interfaceLogger == nil { + initInterfaceLog() + } + }) + if interfaceLogger != nil { + interfaceLogger.Write(strings.Join(logStr, splitStr)) + } +} diff --git a/functionsystem/apps/meta_service/function_repo/router/layer.go b/functionsystem/apps/meta_service/function_repo/router/layer.go new file mode 100644 index 0000000000000000000000000000000000000000..d6a2d7ddb1f05e4089b613f03c02de89ae62a98d --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/layer.go @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "errors" + "net/http" + "strconv" + "time" + + "meta_service/common/constants" + "meta_service/common/logger/log" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/utils" +) + +const ( + minVersionNum = 1 + maxVersionNum = 1000000 + maxCompatibleNum = 64 + retryTimes = 5 + retryPeriodicTime = 500 * time.Millisecond +) + +func regLayerHandlers(r server.RouterGroup) { + // /:layerName/versions -> createLayer + // /update/:layerName -> updateLayer + r.POST("/:v1/:v2", createOrUpdateLayer) + + r.DELETE("/:layerName", deleteLayer) + r.DELETE("/:layerName/versions/:versionNumber", deleteLayerVersion) + r.GET("", getLayerList) + r.GET("/:layerName/versions/:versionNumber", getLayerVersion) + r.GET("/:layerName/versions", getLayerVersionList) +} + +func createOrUpdateLayer(c server.Context) { + v1 := c.Gin().Param("v1") + v2 := c.Gin().Param("v2") + + if v1 == "update" { + updateLayer(c, v2) + } else if v2 == "versions" { + createLayer(c, v1) + } else { + c.Gin().AbortWithStatus(http.StatusNotFound) + } +} + +func createLayer(c server.Context, lname string) { + compatibleRuntimes := c.Gin().Request.Header.Get(constants.HeaderCompatibleRuntimes) + description := c.Gin().Request.Header.Get(constants.HeaderDescription) + licenseInfo := c.Gin().Request.Header.Get(constants.HeaderLicenseInfo) + if lname == "" || len(lname) > paramMaxLength { + log.GetLogger().Errorf("invalid layerName length") + utils.BadRequest(c, errors.New("invalid layerName length")) + return + } + + log.GetLogger().Infof("create layer: %s", lname) + info, err := service.ParseLayerInfo(c, lname) + if err != nil { + log.GetLogger().Errorf("failed to get layer query info: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + contentType := c.Gin().Request.Header.Get("Content-Type") + m, err := getFileInfo(contentType) + if err != nil { + log.GetLogger().Errorf("failed to get file info :%s", err) + utils.BadRequest(c, err) + return + } + + fileSize, err := strconv.ParseInt(m["file-size"], 10, 0) + if err != nil { + log.GetLogger().Errorf("failed to parse file size:%s", err) + utils.BadRequest(c, err) + return + } + req := model.CreateLayerRequest{ + ZipFileSize: fileSize, + CompatibleRuntimes: compatibleRuntimes, + Description: description, + LicenseInfo: licenseInfo, + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.LayerName, + Qualifier: "", + }) + + var resp model.Layer + for i := 0; i < retryTimes; i++ { + resp, err = service.CreateLayer(c, info.LayerName, req) + if err == errmsg.EtcdTransactionFailedError { + log.GetLogger().Errorf("failed to create %s layer: %s, retry create", info.LayerName, err.Error()) + time.Sleep(time.Duration(i+1) * retryPeriodicTime) + continue + } + break + } + utils.WriteResponse(c, resp, err) +} + +func updateLayer(c server.Context, lname string) { + if lname == "" || len(lname) > paramMaxLength { + log.GetLogger().Errorf("invalid layerName length") + utils.BadRequest(c, errors.New("invalid layerName length")) + return + } + + log.GetLogger().Infof("update layer: %s", lname) + info, err := service.ParseLayerInfo(c, lname) + if err != nil { + log.GetLogger().Errorf("failed to get layer query info: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + var req model.UpdateLayerRequest + if err := shouldBindJSON(c, &req); err != nil { + log.GetLogger().Errorf("failed to bind json request: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: lname, + Qualifier: "", + }) + + var resp model.Layer + for i := 0; i < retryTimes; i++ { + resp, err = service.UpdateLayer(c, info.LayerName, req) + if err == errmsg.EtcdTransactionFailedError { + log.GetLogger().Errorf("failed to update %s layer: %s, retry update", info.LayerName, err.Error()) + time.Sleep(time.Duration(i+1) * retryPeriodicTime) + continue + } + break + } + utils.WriteResponse(c, resp, err) +} + +func getLayerQueryInfo(c server.Context) (model.LayerQueryInfo, bool) { + lname := c.Gin().Param("layerName") + if lname == "" || len(lname) > paramMaxLength { + log.GetLogger().Errorf("invalid layerName length") + utils.BadRequest(c, errors.New("invalid layerName length")) + return model.LayerQueryInfo{}, true + } + info, err := service.ParseLayerInfo(c, lname) + if err != nil { + log.GetLogger().Errorf("failed to get layer query info: %s", err.Error()) + utils.BadRequest(c, err) + return model.LayerQueryInfo{}, true + } + return info, false +} + +func deleteLayer(c server.Context) { + info, done := getLayerQueryInfo(c) + if done { + return + } + + log.GetLogger().Infof("delete layer: %s", info.LayerName) + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.LayerName, + Qualifier: "", + }) + + var err error + for i := 0; i < retryTimes; i++ { + err = service.DeleteLayer(c, info.LayerName) + if err == errmsg.EtcdTransactionFailedError { + log.GetLogger().Errorf("failed to delete %s layer: %s, retry delete", info.LayerName, err.Error()) + time.Sleep(time.Duration(i+1) * retryPeriodicTime) + continue + } + break + } + utils.WriteResponse(c, nil, err) +} + +func getLayerNameAndVersion(c server.Context) (string, int, bool) { + info, done := getLayerQueryInfo(c) + if done { + return "", 0, true + } + vstr := c.Gin().Param("versionNumber") + version, err := strconv.Atoi(vstr) + if err != nil { + log.GetLogger().Errorf("failed to parse version number: %s", err) + utils.BadRequest(c, err) + return "", 0, true + } + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.LayerName, + Qualifier: vstr, + }) + return info.LayerName, version, false +} + +func deleteLayerVersion(c server.Context) { + layerName, version, done := getLayerNameAndVersion(c) + if done { + return + } + + log.GetLogger().Infof("delete layer version: %s, %d", layerName, version) + if version < minVersionNum || version > maxVersionNum { + log.GetLogger().Errorf("invalid versionNumber") + utils.BadRequest(c, errors.New("invalid versionNumber")) + return + } + + var err error + for i := 0; i < retryTimes; i++ { + err = service.DeleteLayerVersion(c, layerName, version) + if err == errmsg.EtcdTransactionFailedError { + log.GetLogger().Infof("failed to delete %s layer version: %s, retry delete", layerName, err.Error()) + time.Sleep(time.Duration(i+1) * retryPeriodicTime) + continue + } + break + } + utils.WriteResponse(c, nil, err) +} + +func getLayerList(c server.Context) { + var req model.GetLayerListRequest + if err := utils.ShouldBindQuery(c, &req); err != nil { + log.GetLogger().Errorf("failed to bind json request: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + log.GetLogger().Infof("get layer list: %s, %d", req.LayerName, req.LayerVersion) + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: req.LayerName, + Qualifier: strconv.Itoa(req.LayerVersion), + }) + resp, err := service.GetLayerList(c, req) + utils.WriteResponse(c, resp, err) +} + +func getLayerVersion(c server.Context) { + layerName, version, done := getLayerNameAndVersion(c) + if done { + return + } + log.GetLogger().Infof("get layer version: %s, %d", layerName, version) + + // we allow version 0 as the "latest" version. + if version < 0 || version > maxVersionNum { + log.GetLogger().Errorf("invalid versionNumber") + utils.BadRequest(c, errors.New("invalid versionNumber")) + return + } + resp, err := service.GetLayerVersion(c, layerName, version) + utils.WriteResponse(c, resp, err) +} + +func getLayerVersionList(c server.Context) { + lname := c.Gin().Param("layerName") + if lname == "" || len(lname) > paramMaxLength { + log.GetLogger().Errorf("invalid layerName length") + utils.BadRequest(c, errors.New("invalid layerName length")) + return + } + + log.GetLogger().Infof("get layer version list: %s", lname) + runtime := c.Gin().Query("compatibleRuntime") + if len(runtime) > maxCompatibleNum { + log.GetLogger().Errorf("invalid compatibleRuntime length") + utils.BadRequest(c, errors.New("invalid compatibleRuntime length")) + return + } + + idx, size, err := utils.QueryPaging(c) + if err != nil { + utils.BadRequest(c, err) + return + } + + info, err := service.ParseLayerInfo(c, lname) + if err != nil { + log.GetLogger().Errorf("failed to get layer query info: %s", err.Error()) + utils.BadRequest(c, err) + return + } + + req := model.GetLayerVersionListRequest{ + LayerName: info.LayerName, + CompatibleRuntime: runtime, + PageIndex: idx, + PageSize: size, + } + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.LayerName, + Qualifier: "", + }) + resp, err := service.GetLayerVersionList(c, req) + utils.WriteResponse(c, resp, err) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/layer_test.go b/functionsystem/apps/meta_service/function_repo/router/layer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..452a55465cddd560b64bc2e91ebde75a1a1e9402 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/layer_test.go @@ -0,0 +1,537 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "archive/zip" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey" + "github.com/gin-gonic/gin" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "meta_service/common/constants" + "meta_service/common/snerror" + "meta_service/function_repo/model" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/test/fakecontext" + "meta_service/function_repo/utils" +) + +type mockPackage struct{} + +func (m *mockPackage) Name() string { return "" } + +func (m *mockPackage) FileName() string { return "" } + +func (m *mockPackage) Move() string { return "" } + +func (m *mockPackage) Signature() string { return "" } + +func (m *mockPackage) Size() int64 { return 64 } + +func (m *mockPackage) Close() error { return nil } + +type zipFile struct { + Name, Body string +} + +func newZipFile(files []zipFile) ([]byte, error) { + buf := new(bytes.Buffer) + + w := zip.NewWriter(buf) + for _, file := range files { + f, err := w.Create(file.Name) + if err != nil { + return nil, err + } + + _, err = f.Write([]byte(file.Body)) + if err != nil { + return nil, err + } + } + + err := w.Close() + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func Test_createLayer(t *testing.T) { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&http.Request{}), "ParseMultipartForm", + func(r *http.Request, maxMemory int64) error { + return nil + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, errors.New("mock err") + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, nil + }).ApplyFunc(pkgstore.NewPackage, func(c server.Context, name string, reader io.Reader, + n int64) (pkgstore.Package, error) { + return &mockPackage{}, nil + }) + defer patch.Reset() + for _, test := range []struct { + name string + path string + content string + body string + expectedCode int + expectedMsg string + }{ + {"test create success", "/function-repository/v1/layers/test-layer/versions", + "application/vnd.yuanrong+attachment;file-size=0", ``, http.StatusOK, ""}, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + req := createRequest(t, "POST", test.path, body) + req.Header.Set("Content-Type", test.content) + req.Header.Set(constants.HeaderCompatibleRuntimes, "[\"cpp11\"]") + req.Header.Set(constants.HeaderDescription, "description") + req.Header.Set(constants.HeaderLicenseInfo, "license") + req.Header.Set(constants.HeaderDataContentType, "mock-data") + req.MultipartForm = &multipart.Form{File: map[string][]*multipart.FileHeader{ + "file": []*multipart.FileHeader{{}}, + }} + rec := httptest.NewRecorder() + rec.Body = new(bytes.Buffer) + engine.ServeHTTP(rec, req) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_createLayer_and_updateLayer(t *testing.T) { + Convey("Test createLayer and updateLayer with err", t, func() { + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + Convey("with invalid layerName length err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patch.Reset() + c := fakecontext.NewMockContext() + c.Gin().Request = &http.Request{Header: make(map[string][]string)} + c.Gin().Request.Header.Set(constants.HeaderCompatibleRuntimes, "[\"cpp11\"]") + c.Gin().Request.Header.Set(constants.HeaderDescription, "description") + c.Gin().Request.Header.Set(constants.HeaderLicenseInfo, "license") + createLayer(c, "") + updateLayer(c, "") + }) + Convey("with service.ParseLayerInfo err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }).ApplyFunc(service.ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, errors.New("mock err") + }) + defer patch.Reset() + c := fakecontext.NewMockContext() + c.Gin().Request = &http.Request{Header: make(map[string][]string)} + c.Gin().Request.Header.Set(constants.HeaderCompatibleRuntimes, "[\"cpp11\"]") + c.Gin().Request.Header.Set(constants.HeaderDescription, "description") + c.Gin().Request.Header.Set(constants.HeaderLicenseInfo, "license") + createLayer(c, "aa") + updateLayer(c, "aa") + }) + Convey("with shouldBindJSON err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }).ApplyFunc(service.ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, nil + }).ApplyFunc(shouldBindJSON, func(c server.Context, v interface{}) error { + return errors.New("mock err") + }) + defer patch.Reset() + c := fakecontext.NewMockContext() + c.Gin().Request = &http.Request{Header: make(map[string][]string)} + c.Gin().Request.Header.Set(constants.HeaderCompatibleRuntimes, "[\"cpp11\"]") + c.Gin().Request.Header.Set(constants.HeaderDescription, "description") + c.Gin().Request.Header.Set(constants.HeaderLicenseInfo, "license") + createLayer(c, "aa") + updateLayer(c, "aa") + }) + }) +} + +func Test_getLayerVersionList(t *testing.T) { + Convey("Test getLayerVersionList", t, func() { + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + patchGin := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patchGin.Reset() + ginCtx.Params = []gin.Param{ + gin.Param{Key: "layerName", Value: ""}, + } + Convey("with invalid layerName length err", func() { + c := fakecontext.NewMockContext() + getLayerVersionList(c) + }) + Convey("with invalid compatibleRuntime length err", func() { + ginCtx.Params[0].Value = "aa" + ginCtx.Request = &http.Request{URL: &url.URL{}} + ginCtx.Request.URL.RawQuery = "compatibleRuntime=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa66" + c := fakecontext.NewMockContext() + getLayerVersionList(c) + }) + Convey("with queryPaging err", func() { + patch := gomonkey.ApplyFunc(utils.QueryPaging, func(c server.Context) (pageIndex, pageSize int, err error) { + return 0, 0, errors.New("mock err") + }) + defer patch.Reset() + ginCtx.Params[0].Value = "aa" + ginCtx.Request = &http.Request{URL: &url.URL{}} + ginCtx.Request.URL.RawQuery = "compatibleRuntime=go" + c := fakecontext.NewMockContext() + getLayerVersionList(c) + }) + Convey("with service.ParseLayerInfo err", func() { + patch := gomonkey.ApplyFunc(utils.QueryPaging, func(c server.Context) (pageIndex, pageSize int, err error) { + return 0, 0, nil + }).ApplyFunc(service.ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, errors.New("mock err") + }) + defer patch.Reset() + ginCtx.Params[0].Value = "aa" + ginCtx.Request = &http.Request{URL: &url.URL{}} + ginCtx.Request.URL.RawQuery = "compatibleRuntime=go" + c := fakecontext.NewMockContext() + getLayerVersionList(c) + }) + + }) +} + +func Test_deleteLayer(t *testing.T) { + Test_createLayer(t) + + type args struct { + c server.Context + } + for _, test := range []struct { + name string + path string + body string + expectedCode int + expectedMsg string + }{ + {"test delete success", "/function-repository/v1/layers/test-layer", "", http.StatusOK, ""}, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, "DELETE", test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_deleteLayerVersion(t *testing.T) { + Test_createLayer(t) + + type args struct { + c server.Context + } + for _, test := range []struct { + name string + path string + body string + expectedCode int + expectedMsg string + }{ + {"test delete version success", + "/function-repository/v1/layers/test-layer/versions/1", "", http.StatusOK, ""}, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, "DELETE", test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_deleteLayerVersion2(t *testing.T) { + Convey("Test deleteLayerVersion 2 with done", t, func() { + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyFunc(getLayerNameAndVersion, func(c server.Context) (string, int, bool) { + return "", 0, true + }) + defer patch.Reset() + deleteLayerVersion(ctx) + }) + Convey("Test deleteLayerVersion 2 with version out of range", t, func() { + ctx := fakecontext.NewMockContext() + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + patch := gomonkey.ApplyFunc(getLayerNameAndVersion, func(c server.Context) (string, int, bool) { + return "", 0, false + }).ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patch.Reset() + deleteLayerVersion(ctx) + }) +} + +func Test_getLayerVersion(t *testing.T) { + Convey("Test getLayerVersion with done", t, func() { + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyFunc(getLayerNameAndVersion, func(c server.Context) (string, int, bool) { + return "", 0, true + }) + defer patch.Reset() + getLayerVersion(ctx) + }) + Convey("Test getLayerVersion with version out of range", t, func() { + ctx := fakecontext.NewMockContext() + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + patch := gomonkey.ApplyFunc(getLayerNameAndVersion, func(c server.Context) (string, int, bool) { + return "", -1, false + }).ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patch.Reset() + getLayerVersion(ctx) + }) +} + +func Test_getLayerNameAndVersion(t *testing.T) { + Convey("Test getLayerNameAndVersion", t, func() { + ctx := fakecontext.NewMockContext() + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + mockDone := true + patch := gomonkey.ApplyFunc(getLayerQueryInfo, func(c server.Context) (model.LayerQueryInfo, bool) { + return model.LayerQueryInfo{}, mockDone + }).ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patch.Reset() + Convey("when done", func() { + getLayerNameAndVersion(ctx) + }) + Convey("when version atoi err", func() { + ginCtx.Params = []gin.Param{ + gin.Param{Key: "versionNumber", Value: "abc"}, + } + mockDone = false + getLayerNameAndVersion(ctx) + }) + }) +} + +func Test_getLayer(t *testing.T) { + Test_createLayer(t) + + type args struct { + c server.Context + } + for _, test := range []struct { + name string + path string + body string + expectedCode int + expectedMsg string + }{ + {"test get success", "/function-repository/v1/layers?ownerFlag=1", "", http.StatusOK, ""}, + {"test get version success", "/function-repository/v1/layers/test-layer/versions/1", "", http.StatusOK, ""}, + {"test get version list success", + "/function-repository/v1/layers/test-layer/versions", "", http.StatusOK, ""}, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, "GET", test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_updateLayer(t *testing.T) { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&http.Request{}), "ParseMultipartForm", + func(r *http.Request, maxMemory int64) error { + return nil + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, errors.New("mock err") + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, nil + }).ApplyFunc(pkgstore.NewPackage, func(c server.Context, name string, reader io.Reader, + n int64) (pkgstore.Package, error) { + return &mockPackage{}, nil + }) + defer patch.Reset() + + body := bytes.NewBuffer([]byte(``)) + engine := RegHandlers() + req := createRequest(t, "POST", "/function-repository/v1/layers/test-update-layer/versions", body) + req.Header.Set("Content-Type", "application/vnd.yuanrong+attachment;file-size=0") + req.Header.Set(constants.HeaderCompatibleRuntimes, "[\"cpp11\"]") + req.Header.Set(constants.HeaderDescription, "description") + req.Header.Set(constants.HeaderLicenseInfo, "license") + req.Header.Set(constants.HeaderDataContentType, "mock-data") + req.MultipartForm = &multipart.Form{File: map[string][]*multipart.FileHeader{ + "file": []*multipart.FileHeader{{}}, + }} + rec := httptest.NewRecorder() + rec.Body = new(bytes.Buffer) + engine.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + var resp struct { + Result model.Layer `json:"result"` + } + err := json.Unmarshal(rec.Body.Bytes(), &resp) + require.NoError(t, err) + + type args struct { + c server.Context + } + for _, test := range []struct { + name string + path string + body string + expectedCode int + expectedMsg string + }{ + {"test update success", "/function-repository/v1/layers/update/test-update-layer", `{ + "description": "new description", + "licenseInfo": "MIT", + "lastUpdateTime": "` + resp.Result.UpdateTime.Format(time.RFC3339Nano) + `", + "version": 1 +}`, http.StatusOK, ""}, + } { + fmt.Println(test.body) + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, "POST", test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_getLayerQueryInfo(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return c + }).ApplyFunc(service.ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, errors.New("mock err") + }) + defer patch.Reset() + + Convey("Test getLayerQueryInfo", t, func() { + c.Params = []gin.Param{ + gin.Param{Key: "layerName", Value: ""}, + } + _, fail := getLayerQueryInfo(ctx) + So(fail, ShouldEqual, true) + }) + Convey("Test getLayerQueryInfo 2", t, func() { + c.Params = []gin.Param{ + gin.Param{Key: "layerName", Value: "abc"}, + } + _, fail := getLayerQueryInfo(ctx) + So(fail, ShouldEqual, true) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/middleware.go b/functionsystem/apps/meta_service/function_repo/router/middleware.go new file mode 100644 index 0000000000000000000000000000000000000000..24b8a6b822629825d8cf339a19ef91e12eba44b1 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/middleware.go @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package router +package router + +import ( + "strconv" + "strings" + "time" + + "meta_service/common/logger/log" + "meta_service/function_repo/server" + "meta_service/function_repo/utils" +) + +const ( + xBusinessID = "x-business-id" + xTenantID = "x-tenant-id" + xProductID = "x-product-id" + maxContentTypeLength = 100 + uploadCodeContentType = "application/vnd.yuanrong+attachment" + uploadCodeFileSize = "file-size" + uploadCodeRevisionID = "revision-id" +) + +const ( + defaultUserID = "12345678901234561234567890123456" + defaultTenantID = "12345678901234561234567890123456" + defaultPrivilege = "1111" + defaultAppID = "yrk" + defaultAppSecret = "12CFV18835434FDGEEF39BD6YRE45D46" + // HeaderBusinessID - + HeaderBusinessID = "X-Business-ID" + // HeaderTenantID - + HeaderTenantID = "X-Tenant-ID" + // HeaderPrivilege is Privilege id + HeaderPrivilege = "X-Privilege" + // HeaderUserID is User ID + HeaderUserID = "X-User-Id" + // AppIDKey is context key of appID + AppIDKey = "appID" + // AppSecretKey is context key of appSecret + AppSecretKey = "appSecret" +) + +// SetRequestHead adding a Tenant ID to a Request Header +func SetRequestHead() server.HandlerFunc { + return func(c server.Context) { + c.Gin().Request.Header.Set(HeaderUserID, defaultUserID) + + if c.Gin().Request.Header.Get(HeaderTenantID) == "" { + c.Gin().Request.Header.Set(HeaderTenantID, defaultTenantID) + } + + c.Gin().Request.Header.Set(HeaderPrivilege, defaultPrivilege) + c.Gin().Request.Header.Set(HeaderBusinessID, defaultAppID) + + c.Gin().Set(AppIDKey, defaultAppID) + c.Gin().Set(AppSecretKey, defaultAppSecret) + + c.Gin().Next() + } +} + +// LogMiddle middle of log +func LogMiddle() server.HandlerFunc { + return func(c server.Context) { + start := time.Now() + path := c.Gin().Request.URL.Path + query := c.Gin().Request.URL.RawQuery + c.Gin().Next() + latency := time.Since(start) + if len(c.Gin().Errors) > 0 { + for _, e := range c.Gin().Errors.Errors() { + log.GetLogger().Error(e) + } + } + header := c.Gin().Request.Header + write(interfaceLog{ + httpMethod: c.Gin().Request.Method, + ip: c.Gin().ClientIP(), + requestPath: path, + query: query, + bussinessID: header.Get("x-business-id"), + traceID: header.Get("x-trace-id"), + retCode: strconv.Itoa(c.Gin().Writer.Status()), + costTime: latency.String(), + }) + } +} + +func isUploadCode(contentType string) (bool, string) { + if len(contentType) > maxContentTypeLength { + return false, "" + } + + ct := strings.ReplaceAll(contentType, " ", "") + fields := strings.Split(ct, ";") + + result := false + info := "" + for _, f := range fields { + if f == uploadCodeContentType { + result = true + } else if strings.HasPrefix(f, uploadCodeFileSize) { + info = f + } else if strings.HasPrefix(f, uploadCodeRevisionID) { + info = info + "&" + f + } + } + return result, info +} + +// InitTenantInfoMiddle: Init Tenant Info +func InitTenantInfoMiddle() server.HandlerFunc { + return func(c server.Context) { + header := c.Gin().Request.Header + info := server.TenantInfo{ + BusinessID: header.Get(xBusinessID), + TenantID: header.Get(xTenantID), + ProductID: header.Get(xProductID), + } + if err := utils.Validate(info); err != nil { + utils.BadRequest(c, err) + c.Gin().Abort() + } else { + c.InitTenantInfo(info) + c.Gin().Next() + } + } +} diff --git a/functionsystem/apps/meta_service/function_repo/router/middleware_test.go b/functionsystem/apps/meta_service/function_repo/router/middleware_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0e8cf86c2b7a084c91053618ab14ba0511ea4547 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/middleware_test.go @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package router +package router + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsUploadCode(t *testing.T) { + for _, test := range []struct { + contentType string + is bool + info string + }{ + {"application/vnd.yuanrong+attachment;file-size=683;revision-id=12345678901234567890" + + "12345678901234567890123456789012345678901234567890123456789012345678901234567890", false, ""}, + {"", false, ""}, + {"application/json", false, ""}, + {"application/vnd.yuanrong+attachment;file-size=683;revision-id=1234567890", + true, "file-size=683&revision-id=1234567890"}, + } { + ok, info := isUploadCode(test.contentType) + assert.Equal(t, test.is, ok) + assert.Equal(t, test.info, info) + } +} diff --git a/functionsystem/apps/meta_service/function_repo/router/podpool.go b/functionsystem/apps/meta_service/function_repo/router/podpool.go new file mode 100644 index 0000000000000000000000000000000000000000..ca1a430b0c4de72e8d301a371938a7067ccd958f --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/podpool.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "strconv" + + "meta_service/common/logger/log" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/utils" +) + +func regPodPoolHandlers(r server.RouterGroup) { + r.POST("", createPodPool) + r.DELETE("", deletePodPool) + r.PUT("/:id", updatePodPool) + r.GET("", getPodPool) +} + +func createPodPool(c server.Context) { + var req model.PodPoolCreateRequest + if err := shouldBindJSON(c, &req); err != nil { + log.GetLogger().Errorf("failed to bind json :%s", err) + utils.BadRequest(c, err) + return + } + log.GetLogger().Debugf("start to create pod pools, pod pool num: %d", len(req.Pools)) + resp, err := service.CreatePodPool(c, req) + utils.WriteResponseWithMsg(c, resp, err) +} + +func deletePodPool(c server.Context) { + id := c.Gin().Query("id") + group := c.Gin().Query("group") + log.GetLogger().Debugf("start to delete pod pool, pod pool id: %s, group: %s", id, group) + err := service.DeletePodPool(c, id, group) + utils.WriteResponse(c, nil, err) +} + +func updatePodPool(c server.Context) { + var req model.PodPoolUpdateRequest + if err := shouldBindJSON(c, &req); err != nil { + log.GetLogger().Errorf("failed to bind json :%s", err) + utils.BadRequest(c, err) + return + } + req.ID = c.Gin().Param("id") + log.GetLogger().Debugf("start to update pod pool, id： %s, size: %d", req.ID, req.Size) + err := service.UpdatePodPool(c, req) + utils.WriteResponse(c, nil, err) +} + +func getPodPool(c server.Context) { + id := c.Gin().Query("id") + group := c.Gin().Query("group") + limit := c.Gin().Query("limit") + limitInt, err := strconv.Atoi(limit) + if err != nil { + log.GetLogger().Errorf("failed to convert string: %s to int, err: %s", limit, err) + utils.BadRequest(c, err) + return + } + offset := c.Gin().Query("offset") + offsetInt, err := strconv.Atoi(offset) + if err != nil { + log.GetLogger().Errorf("failed to convert string: %s to int, err: %s", limit, err) + utils.BadRequest(c, err) + return + } + req := model.PodPoolGetRequest{ + ID: id, + Group: group, + Limit: limitInt, + Offset: offsetInt, + } + log.GetLogger().Debugf("start to get pod pool, id： %s, group: %s, limit: %d, offset: %d", + req.ID, req.Group, req.Limit, req.Offset) + resp, err := service.GetPodPool(c, req) + utils.WriteResponse(c, resp, err) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/podpool_test.go b/functionsystem/apps/meta_service/function_repo/router/podpool_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6d9b2231e018da683a1da0ad2f716ea16e445bed --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/podpool_test.go @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" +) + +func mockCleanEtcd(id string) { + ctx := server.NewContext(&gin.Context{Request: &http.Request{}}) + txn := storage.NewTxn(ctx) + storage.DeletePodPool(txn, id) + txn.Commit() +} + +func Test_createPodPool(t *testing.T) { + type args struct { + c server.Context + } + for i, test := range []struct { + name string + path string + body string + expectedCode int + expectedMsg string + }{ + {"test bind failed", "/function-repository/v1/podpools", "{\n" + + "\t\"pools\": [[]\n" + + "}", 121052, "request body is not a valid JSON object"}, + {"test create success", "/function-repository/v1/podpools", "{\n" + + "\t\"pools\": [{\"id\":\"pool1\",\"size\":1,\"group\":\"group1\",\"image\":\"image1\"," + + "\"init_image\":\"init1\",\"reuse\":true,\"labels\":{\"p1\":\"p1\"},\"environment\":{\"p1\":\"p1\"}," + + "\"resources\":{\"requests\":{\"CPU\":\"500\"},\"limits\":{\"CPU\":\"600\"}},\"volumes\":\"volumes\"," + + "\"volume_mounts\":\"volumes\",\"topology_spread_constraints\":\"[{}]\"," + + "\"affinities\":\"affinity1\"}]\n" + + "}", http.StatusOK, ""}, + {"test create name error length", "/function-repository/v1/podpools", "{\n" + + "\t\"pools\": [{\"id\":\"123e4567-e89b-12d3-a456-4266554400001234\",\"size\":1,\"group\":\"group1\",\"image\":\"image1\",\"init_image\":\"init1\",\"reuse\":true,\"labels\":{\"p1\":\"p1\"},\"environment\":{\"p1\":\"p1\"},\"resources\":{\"requests\":{\"CPU\":\"500\"},\"limits\":{\"CPU\":\"600\"}},\"volumes\":\"volumes\",\"volume_mounts\":\"volumes\",\"affinities\":\"affinity1\"}]\n" + + "}", http.StatusOK, ""}, + {"test create failed", "/function-repository/v1/podpools", "{\n" + + "\t\"pools\": [{\"id\":\"\",\"size\":1,\"group\":\"group1\",\"image\":\"image1\",\"init_image\":\"init1\",\"reuse\":true,\"labels\":{\"p1\":\"p1\"},\"environment\":{\"p1\":\"p1\"},\"annotations\":{\"p1\":\"p1\"},\"resources\":{\"requests\":{\"CPU\":\"500\"},\"limits\":{\"CPU\":\"600\"}},\"volumes\":\"volumes\",\"volume_mounts\":\"volumes\",\"pod_anti_affinities\":\"affinity1\"},{\"id\":\"\",\"size\":2,\"group\":\"group2\",\"image\":\"image2\",\"init_image\":\"init2\",\"reuse\":true,\"labels\":{\"p2\":\"p2\"},\"environment\":{\"p2\":\"p2\"},\"annotations\":{\"p2\":\"p2\"},\"resources\":{\"requests\":{\"CPU\":\"600\"},\"limits\":{\"CPU\":\"800\"}},\"volumes\":\"volumes2\",\"volume_mounts\":\"volumes2\",\"affinities\":\"affinity2\"}]\n" + + "}", http.StatusOK, ""}, + } { + if i == 1 { + mockCleanEtcd("pool1") + } + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, "POST", test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_deletePodPool(t *testing.T) { + type args struct { + c server.Context + } + tests := []struct { + name string + path string + delBody string + createBody string + expectedCode int + expectedMsg string + }{ + {"test invalid param", "/function-repository/v1/podpools", "", "", + errmsg.InvalidUserParam, "id and group cannot be empty at the same time. check input parameters"}, + {"test delete failed", "/function-repository/v1/podpools?id=pool3", "", "", + 4120, "pool does not exist or has been deleted. check input parameters"}, + {"test delete failed2", "/function-repository/v1/podpools?group=group3", "", "", + 4120, "pool does not exist or has been deleted. check input parameters"}, + {"test delete succeed", "/function-repository/v1/podpools?id=pool4", "", "{\n" + + "\t\"pools\": [{\"id\":\"pool4\",\"size\":1,\"group\":\"group4\",\"image\":\"image1\",\"init_image\":\"init1\",\"reuse\":true,\"labels\":{\"p1\":\"p1\"},\"environment\":{\"p1\":\"p1\"},\"annotations\":{\"p1\":\"p1\"},\"resources\":{\"requests\":{\"CPU\":\"500\"},\"limits\":{\"CPU\":\"600\"}},\"volumes\":\"volumes\",\"volume_mounts\":\"volumes\",\"pod_affinities\":\"affinity1\"}]\n" + + "}", + http.StatusOK, ""}, + } + for i := 0; i < 3; i++ { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + body := bytes.NewBuffer([]byte(tt.delBody)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, "DELETE", tt.path, body) + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, tt.expectedCode, badResponse.Code) + if tt.expectedMsg != "" { + assert.Equal(t, tt.expectedMsg, badResponse.Message) + } + }) + } + + tt := tests[3] + t.Run(tt.name, func(t *testing.T) { + mockCleanEtcd("pool4") + body := bytes.NewBuffer([]byte(tt.createBody)) + engine := RegHandlers() + // create + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/podpools", body) + // delete + body = bytes.NewBuffer([]byte(tt.delBody)) + _, rec = routerRequest(t, engine, "DELETE", tt.path, body) + fmt.Printf("%v\n", rec) + assert.Equal(t, tt.expectedCode, rec.Code) + }) +} + +func Test_getPodPool(t *testing.T) { + tests := []struct { + name string + method string + path string + body string + expectedCode int + expectedMsg string + }{ + { + "test get pod pool with empty id and group", + "GET", + "/function-repository/v1/podpools?limit=5&offset=0", + ``, + http.StatusOK, + "", + }, + { + "test get pod pool with id failed", + "GET", + "/function-repository/v1/podpools?id=pool_get&limit=5&offset=0", + ``, + http.StatusOK, + "", + }, + { + "test get pod pool with group failed", + "GET", + "/function-repository/v1/podpools?group=group_get1&limit=5&offset=0", + ``, + http.StatusOK, + "pod pool group: group_get1 may not exist, try get by id. check input parameters", + }, + { + "test get pod pool with id succeed", + "GET", + "/function-repository/v1/podpools?id=pool_get2&limit=5&offset=0", + "{\n" + + "\t\"pools\": [{\"id\":\"pool_get2\",\"size\":1,\"group\":\"group4\",\"image\":\"image1\",\"init_image\":\"init1\",\"reuse\":true,\"labels\":{\"p1\":\"p1\"},\"environment\":{\"p1\":\"p1\"},\"annotations\":{\"p1\":\"p1\"},\"resources\":{\"requests\":{\"CPU\":\"500\"},\"limits\":{\"CPU\":\"600\"}},\"volumes\":\"volumes\",\"volume_mounts\":\"volumes\",\"pod_affinities\":\"affinity1\"}]\n" + + "}", + http.StatusOK, + "", + }, + } + for i := 0; i < 3; i++ { + tt := tests[i] + body := bytes.NewBuffer([]byte(tt.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, tt.method, tt.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, tt.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, tt.expectedCode, badResponse.Code) + if tt.expectedMsg != "" { + assert.Equal(t, tt.expectedMsg, badResponse.Message) + } + } + } + + tt := tests[3] + t.Run(tt.name, func(t *testing.T) { + body := bytes.NewBuffer([]byte(tt.body)) + engine := RegHandlers() + // create + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/podpools", body) + // get + body = bytes.NewBuffer([]byte("")) + _, rec = routerRequest(t, engine, "GET", tt.path, body) + fmt.Printf("%v\n", rec) + assert.Equal(t, tt.expectedCode, rec.Code) + }) +} + +func Test_updatePodPool(t *testing.T) { + type args struct { + c server.Context + } + for _, test := range []struct { + name string + path string + body string + expectedCode int + expectedMsg string + }{ + {"test bind failed", "/function-repository/v1/podpools/pool_update1", "{\"size\": [[]}", 121052, "request body is not a valid JSON object"}, + {"test update failed", "/function-repository/v1/podpools/pool_update1", "{\"size\": 5}", 4120, + "pool does not exist or has been deleted. check input parameters"}, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, "PUT", test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} diff --git a/functionsystem/apps/meta_service/function_repo/router/publish.go b/functionsystem/apps/meta_service/function_repo/router/publish.go new file mode 100644 index 0000000000000000000000000000000000000000..ff1109c0b8faecd864fabd55086e976af93d524f --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/publish.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "errors" + + "meta_service/common/logger/log" + "meta_service/function_repo/config" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/utils" +) + +func regPublishHandlers(r server.RouterGroup) { + r.POST("/:functionName/versions", publishFunction) +} + +func publishFunction(c server.Context) { + funcName := c.Gin().Param("functionName") + if funcName == "" { + utils.BadRequest(c, errors.New("empty function name")) + return + } + req := model.PublishRequest{} + if err := shouldBindJSON(c, &req); err != nil { + log.GetLogger().Errorf("failed to bind parameters when publish the function: %s", err.Error()) + utils.BadRequest(c, err) + return + } + info, err := service.ParseFunctionInfo(c, funcName, "") + if err != nil { + log.GetLogger().Errorf("failed to get function query info: %s", err.Error()) + utils.BadRequest(c, err) + return + } + c.InitResourceInfo(server.ResourceInfo{ + ResourceName: info.FunctionName, + Qualifier: "", + FunctionType: config.ComFunctionType, + }) + if err = service.CheckFunctionVersion(c, info.FunctionName); err != nil { + log.GetLogger().Errorf("failed to check function versions: %s", err.Error()) + utils.BadRequest(c, err) + return + } + resp, err := service.PublishFunction(c, info.FunctionName, req) + if err != nil { + log.GetLogger().Errorf("failed to publish function: %s", err.Error()) + utils.BadRequest(c, err) + return + } + utils.WriteResponse(c, resp, err) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/publish_test.go b/functionsystem/apps/meta_service/function_repo/router/publish_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8799ad540d8bc79d23dd77cef2de5bd9cbbb2dd8 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/publish_test.go @@ -0,0 +1,79 @@ +package router + +import ( + "bytes" + "errors" + "net/http" + "testing" + + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" +) + +func Test_publishFunction(t *testing.T) { + engine := RegHandlers() + // POST /function-repository/v1/functions/:functionName/versions + Convey("Test publishFunction 1", t, func() { + body := bytes.NewBuffer([]byte("")) + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/functions//versions", body) + So(rec.Code, ShouldEqual, http.StatusInternalServerError) + }) + Convey("Test publishFunction 2", t, func() { + body := bytes.NewBuffer([]byte("")) + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/functions/abc123/versions", body) + So(rec.Code, ShouldEqual, http.StatusInternalServerError) + }) + Convey("Test publishFunction 3 with ParseFunctionInfo err", t, func() { + patch := gomonkey.ApplyFunc(service.ParseFunctionInfo, func(ctx server.Context, + queryInfo, qualifier string) (model.FunctionQueryInfo, error) { + return model.FunctionQueryInfo{}, errors.New("mock err") + }) + defer patch.Reset() + body := bytes.NewBuffer([]byte(`{"revisionId":"mock-rev-id","description":"mock-desc"}`)) + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/functions/abc123/versions", body) + So(rec.Code, ShouldEqual, http.StatusInternalServerError) + }) + patches := gomonkey.ApplyFunc(service.ParseFunctionInfo, func(ctx server.Context, + queryInfo, qualifier string) (model.FunctionQueryInfo, error) { + return model.FunctionQueryInfo{}, nil + }) + defer patches.Reset() + Convey("Test publishFunction 4 with CheckFunctionVersion err", t, func() { + patch := gomonkey.ApplyFunc(service.CheckFunctionVersion, func(ctx server.Context, functionName string) error { + return errors.New("mock err") + }) + defer patch.Reset() + body := bytes.NewBuffer([]byte(`{"revisionId":"mock-rev-id","description":"mock-desc"}`)) + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/functions/abc123/versions", body) + So(rec.Code, ShouldEqual, http.StatusInternalServerError) + }) + Convey("Test publishFunction 5 with PublishFunction err", t, func() { + patch := gomonkey.ApplyFunc(service.CheckFunctionVersion, func(ctx server.Context, functionName string) error { + return nil + }).ApplyFunc(service.PublishFunction, func( + ctx server.Context, funcName string, req model.PublishRequest) (model.PublishResponse, error) { + return model.PublishResponse{}, errors.New("mock err") + }) + defer patch.Reset() + body := bytes.NewBuffer([]byte(`{"revisionId":"mock-rev-id","description":"mock-desc"}`)) + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/functions/abc123/versions", body) + So(rec.Code, ShouldEqual, http.StatusInternalServerError) + }) + Convey("Test publishFunction 6", t, func() { + patch := gomonkey.ApplyFunc(service.CheckFunctionVersion, func(ctx server.Context, functionName string) error { + return nil + }).ApplyFunc(service.PublishFunction, func( + ctx server.Context, funcName string, req model.PublishRequest) (model.PublishResponse, error) { + return model.PublishResponse{}, nil + }) + defer patch.Reset() + body := bytes.NewBuffer([]byte(`{"revisionId":"mock-rev-id","description":"mock-desc"}`)) + _, rec := routerRequest(t, engine, "POST", "/function-repository/v1/functions/abc123/versions", body) + So(rec.Code, ShouldEqual, http.StatusOK) + }) + +} diff --git a/functionsystem/apps/meta_service/function_repo/router/router_test.go b/functionsystem/apps/meta_service/function_repo/router/router_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1d0026cf86d2646e0968a32dca25041d2a04eaf7 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/router_test.go @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "testing" + + "meta_service/function_repo/initialize" + "meta_service/function_repo/server" + "meta_service/function_repo/test" +) + +// TestMain init router test +func TestMain(m *testing.M) { + if !initialize.Initialize(test.ConfigPath, "/home/sn/config/log.json") { + fmt.Println("failed to initialize") + os.Exit(1) + } + test.ResetETCD() + result := m.Run() + test.ResetETCD() + os.Exit(result) +} + +func routerRequest(t *testing.T, router server.Engine, method string, path string, body *bytes.Buffer) (*http.Request, + *httptest.ResponseRecorder) { + req := createRequest(t, method, path, body) + rec := httptest.NewRecorder() + rec.Body = new(bytes.Buffer) + router.ServeHTTP(rec, req) + return req, rec +} + +func routerRequestWithQueries(t *testing.T, router server.Engine, method string, path string, + body *bytes.Buffer, queries map[string]string) (*http.Request, + *httptest.ResponseRecorder) { + req := createRequest(t, method, path, body) + query := req.URL.Query() + for k, v := range queries { + query.Add(k, v) + } + req.URL.RawQuery = query.Encode() + rec := httptest.NewRecorder() + rec.Body = new(bytes.Buffer) + router.ServeHTTP(rec, req) + return req, rec +} + +func createRequest(t *testing.T, method, path string, body io.Reader) *http.Request { + // HACK: derive content-length since protocol/http does not add content-length + // if it's not present. + if body != nil { + buf := &bytes.Buffer{} + _, err := io.Copy(buf, body) + if err != nil { + t.Fatalf("Test: Could not copy %s request body to %s: %v", method, path, err) + } + body = buf + } + + req, err := http.NewRequest(method, "http://127.0.0.1:8080"+path, body) + if err != nil { + t.Fatalf("Test: Could not create %s request to %s: %v", method, path, err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-business-id", "yrk") + req.Header.Set("x-tenant-id", "i1fe539427b24702acc11fbb4e134e17") + req.Header.Set("x-product-id", "") + + return req +} diff --git a/functionsystem/apps/meta_service/function_repo/router/run.go b/functionsystem/apps/meta_service/function_repo/router/run.go new file mode 100644 index 0000000000000000000000000000000000000000..21d74ddb13d766283d1b63b97ce74acfa8f53d2a --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/run.go @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package router implements an api server with routers registered +package router + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "meta_service/common/utils" + "meta_service/function_repo/server" +) + +const ( + paramMaxLength int = 256 +) + +// Run starts the server +func Run(addr string, stopCh <-chan struct{}) { + r := RegHandlers() + if err := r.Run(addr, stopCh); err != nil { + utils.ProcessBindErrorAndExit(err) + } +} + +// RegHandlers registering route +func RegHandlers() server.Engine { + gin.SetMode(gin.ReleaseMode) + r := server.New() + rg := r.Group("/function-repository/v1") + rg.Use(LogMiddle()) + rg.Use(InitTenantInfoMiddle()) + regFunctionHandlers(rg.Group("functions")) + regPublishHandlers(rg.Group("functions")) + regAliasHandler(rg.Group("functions")) + regLayerHandlers(rg.Group("layers")) + regTriggerHandlers(rg.Group("triggers")) + regServiceHandlers(rg.Group("snService")) + regPodPoolHandlers(rg.Group("podpools")) + r.Group("/healthz").GET("", func(c server.Context) { + c.Gin().Data(http.StatusOK, "", nil) + }) + return r +} diff --git a/functionsystem/apps/meta_service/function_repo/router/service.go b/functionsystem/apps/meta_service/function_repo/router/service.go new file mode 100644 index 0000000000000000000000000000000000000000..a79b4995d69b07e3c14a54eada3f34781bfd44f1 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/service.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "errors" + + "meta_service/common/logger/log" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/utils" +) + +const ( + maxServiceIDLen = 16 +) + +func regServiceHandlers(r server.RouterGroup) { + r.GET("/:snServiceId", getService) + r.GET("", getService) +} + +func getService(c server.Context) { + id := c.Gin().Param("snServiceId") + kind := c.Gin().Query("kind") + if len(id) > maxServiceIDLen { + log.GetLogger().Errorf("invalid serviceId length: %d", len(id)) + utils.BadRequest(c, errors.New("invalid serviceId length")) + return + } + + resp, err := service.GetServiceID(c, id, kind) + utils.WriteResponse(c, resp, err) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/service_test.go b/functionsystem/apps/meta_service/function_repo/router/service_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0fd57c74f5234db2a88a2e6a5c90679c99e10009 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/service_test.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "net/http/httptest" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/gin-gonic/gin" + . "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/test/fakecontext" +) + +func TestGetService(t *testing.T) { + Convey("Test getService", t, func() { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + c.Params = []gin.Param{ + gin.Param{Key: "snServiceId", Value: "12345678901234567890"}, + } + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", + func(*fakecontext.Context) *gin.Context { + return c + }).ApplyFunc(service.GetServiceID, func(ctx server.Context, id, _ string) (model.ServiceGetResponse, error) { + return model.ServiceGetResponse{}, nil + }) + defer patch.Reset() + getService(ctx) + + c.Params = []gin.Param{ + gin.Param{Key: "snServiceId", Value: "123"}, + } + getService(ctx) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/trigger.go b/functionsystem/apps/meta_service/function_repo/router/trigger.go new file mode 100644 index 0000000000000000000000000000000000000000..993d4613b4b4dcca8e496e7ecd87180b2f453c05 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/trigger.go @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "errors" + + "meta_service/common/logger/log" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/utils" +) + +const ( + maxTriggerLen = 64 + maxFunctionIDLen = 255 +) + +func regTriggerHandlers(r server.RouterGroup) { + r.POST("", createTrigger) + r.DELETE("/:triggerId", deleteTrigger) + r.DELETE("/:triggerId/:functionId", deleteTriggers) + r.GET("/:triggerId", getTrigger) + r.GET("", getTriggerList) + r.PUT("/:triggerId", updateTrigger) +} + +func checkTriggerID(tid string) error { + if tid == "" || len(tid) > maxTriggerLen { + log.GetLogger().Errorf("invalid triggerid length: %d", len(tid)) + return errmsg.NewParamError("invalid triggerid length: %d", len(tid)) + } + return nil +} + +func checkFunctionID(fid string) error { + if fid == "" || len(fid) > maxFunctionIDLen { + log.GetLogger().Errorf("invalid functionid length: %d", len(fid)) + return errmsg.NewParamError("invalid functionid length: %d", len(fid)) + } + return nil +} + +func createTrigger(c server.Context) { + var req model.TriggerCreateRequest + err := shouldBindJSON(c, &req) + if err != nil { + log.GetLogger().Errorf("invalid param in request") + utils.BadRequest(c, err) + return + } + resp, err := service.CreateTrigger(c, req) + utils.WriteResponse(c, resp, err) +} + +func deleteTrigger(c server.Context) { + tid := c.Gin().Param("triggerId") + if tid == "function" { + deleteTriggers(c) + return + } + + if err := checkTriggerID(tid); err != nil { + utils.BadRequest(c, errmsg.NewParamError("invalid triggerid length: %d", len(tid))) + return + } + err := service.DeleteTriggerByID(c, tid) + utils.WriteResponse(c, nil, err) +} + +func deleteTriggers(c server.Context) { + tid := c.Gin().Param("triggerId") + if tid != "function" { + log.GetLogger().Errorf("invalid param in request") + utils.BadRequest(c, errors.New("invalid param in request")) + return + } + fid := c.Gin().Param("functionId") + + if err := checkFunctionID(fid); err != nil { + utils.BadRequest(c, errmsg.NewParamError("invalid functionid length: %d", len(fid))) + return + } + + err := service.DeleteTriggerByFuncID(c, fid) + utils.WriteResponse(c, nil, err) +} + +func getTrigger(c server.Context) { + tid := c.Gin().Param("triggerId") + if err := checkTriggerID(tid); err != nil { + utils.BadRequest(c, errmsg.NewParamError("invalid triggerid length: %d", len(tid))) + return + } + resp, err := service.GetTrigger(c, tid) + utils.WriteResponse(c, resp, err) +} + +func getTriggerList(c server.Context) { + idx, size, err := utils.QueryPaging(c) + if err != nil { + log.GetLogger().Errorf("invalid page info") + utils.BadRequest(c, err) + return + } + + fid := c.Gin().Query("funcId") + if err := checkFunctionID(fid); err != nil { + utils.BadRequest(c, errmsg.NewParamError("invalid functionid length: %d", len(fid))) + return + } + resp, err := service.GetTriggerList(c, idx, size, fid) + utils.WriteResponse(c, resp, err) +} + +func updateTrigger(c server.Context) { + var req model.TriggerUpdateRequest + if err := shouldBindJSON(c, &req); err != nil { + log.GetLogger().Errorf("invalid param in request") + utils.BadRequest(c, err) + return + } + + tid := c.Gin().Param("triggerId") + if err := checkTriggerID(tid); err != nil { + utils.BadRequest(c, errmsg.NewParamError("invalid triggerid length: %d", len(tid))) + return + } + req.TriggerID = tid + resp, err := service.UpdateTrigger(c, req) + utils.WriteResponse(c, resp, err) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/trigger_test.go b/functionsystem/apps/meta_service/function_repo/router/trigger_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3e938ff456a79ad7aaa68d6605bac93d646e5711 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/trigger_test.go @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/gin-gonic/gin" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/service" + "meta_service/function_repo/test/fakecontext" + "meta_service/function_repo/utils" +) + +func TestTrigger(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(service.CreateTrigger, + func(ctx server.Context, req model.TriggerCreateRequest) (model.TriggerResponse, error) { + return model.TriggerResponse{}, nil + }) + patches.ApplyFunc(service.UpdateTrigger, + func(ctx server.Context, req model.TriggerUpdateRequest) (model.TriggerResponse, error) { + return model.TriggerResponse{}, nil + }) + patches.ApplyFunc(service.DeleteTriggerByID, + func(ctx server.Context, tid string) (err error) { + return nil + }) + patches.ApplyFunc(service.DeleteTriggerByFuncID, + func(ctx server.Context, fid string) error { + return nil + }) + patches.ApplyFunc(service.GetTrigger, + func(ctx server.Context, tid string) (model.TriggerResponse, error) { + return model.TriggerResponse{}, nil + }) + patches.ApplyFunc(service.GetTriggerList, + func(ctx server.Context, pageIndex, pageSize int, fid string) (model.TriggerListGetResponse, error) { + return model.TriggerListGetResponse{}, nil + }) + defer patches.Reset() + + for _, test := range []struct { + name string + method string + path string + body string + expectedCode int + expectedMsg string + }{ + {"test trigger create", "POST", "/function-repository/v1/triggers", `{ + "functionid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggertype": "HTTP", + "spec": { + "funcid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggerid": "", + "triggertype": "HTTP", + "httpmethod": "PUT", + "resourceid": "iamsff", + "authflag": false, + "authalgorithm": "", + "triggerurl": "", + "appid": "", + "appsecret": "" + } +}`, http.StatusOK, ""}, + {"test trigger create2", "POST", "/function-repository/v1/triggers", + `{`, 121052, "request body is not a valid JSON object"}, + {"test trigger create3", "POST", "/function-repository/v1/triggers", `{ + "functionid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggertype": "HTTP", + "spec": "" +}`, 4104, "json: cannot unmarshal string into Go value of type model.HTTPTriggerSpec. check input parameters"}, + {"test trigger create4", "POST", "/function-repository/v1/triggers", `{ + "functionid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggertype": "unkown" +}`, 4104, "invalid triggertype: unkown. check input parameters"}, + {"test trigger delete", "DELETE", "/function-repository/v1/triggers/1f05e8e5-ce0d-4c4a-903a-eae27b8f9d04", + "", + http.StatusOK, ""}, + {"test trigger delete", "DELETE", "/function-repository/v1/triggers/function/sn:cn:businessid:tenantid@productid:function:test:$latest", + "", + http.StatusOK, ""}, + {"test trigger get", "GET", "/function-repository/v1/triggers/1f05e8e5-ce0d-4c4a-903a-eae27b8f9d04", + "", + http.StatusOK, ""}, + {"test trigger get list", "GET", "/function-repository/v1/triggers?funcId=sn:cn:businessid:tenantid@productid:function:test:$latest", + "", + http.StatusOK, ""}, + {"test update trigger", "PUT", "/function-repository/v1/triggers/1f05e8e5-ce0d-4c4a-903a-eae27b8f9d04", + "abc123", errmsg.InvalidJSONBody, "request body is not a valid JSON object"}, + {"test update trigger 2", "PUT", "/function-repository/v1/triggers/66666666666666666666666666666666666666666666666666666666666666666", + `{ + "functionid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggertype": "HTTP", + "spec": { + "funcid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggerid": "", + "triggertype": "HTTP", + "httpmethod": "PUT", + "resourceid": "iamsff", + "authflag": false, + "authalgorithm": "", + "triggerurl": "", + "appid": "", + "appsecret": "" + } +}`, errmsg.InvalidUserParam, "invalid triggerid length: 65. check input parameters"}, + {"test update trigger 3", "PUT", "/function-repository/v1/triggers/1f05e8e5-ce0d-4c4a-903a-eae27b8f9d04", + `{ + "functionid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggertype": "HTTP", + "spec": { + "funcid": "sn:cn:businessid:tenantid@productid:function:test:$latest", + "triggerid": "", + "triggertype": "HTTP", + "httpmethod": "PUT", + "resourceid": "iamsff", + "authflag": false, + "authalgorithm": "", + "triggerurl": "", + "appid": "", + "appsecret": "" + } +}`, http.StatusOK, ""}, + } { + body := bytes.NewBuffer([]byte(test.body)) + engine := RegHandlers() + _, rec := routerRequest(t, engine, test.method, test.path, body) + if rec.Code == http.StatusOK { + assert.Equal(t, test.expectedCode, rec.Code) + } else { + fmt.Printf("%v\n", rec) + badResponse := &snerror.BadResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), badResponse); err != nil { + t.Errorf("%s", err) + } + assert.Equal(t, test.expectedCode, badResponse.Code) + if test.expectedMsg != "" { + assert.Equal(t, test.expectedMsg, badResponse.Message) + } + } + } +} + +func Test_checkFunctionID(t *testing.T) { + err := checkFunctionID("") + assert.NotNil(t, err) +} + +func Test_getTriggerList(t *testing.T) { + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + patchGin := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patchGin.Reset() + Convey("Test getTriggerList", t, func() { + Convey("with queryPaging err", func() { + patch := gomonkey.ApplyFunc(utils.QueryPaging, func(c server.Context) (pageIndex, pageSize int, err error) { + return 0, 0, errors.New("mock err") + }) + defer patch.Reset() + c := fakecontext.NewMockContext() + getTriggerList(c) + }) + Convey("with checkFunctionID err", func() { + patch := gomonkey.ApplyFunc(checkFunctionID, func(fid string) error { + return errors.New("mock err") + }) + defer patch.Reset() + ginCtx.Request = &http.Request{URL: &url.URL{}} + ginCtx.Request.URL.RawQuery = "funcId=a1" + c := fakecontext.NewMockContext() + getTriggerList(c) + }) + }) +} + +func Test_deleteTriggers(t *testing.T) { + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + patchGin := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patchGin.Reset() + + Convey("Test deleteTriggers with triggerId invalid param err", t, func() { + ginCtx.Params = []gin.Param{ + gin.Param{Key: "triggerId", Value: "mock-fu"}, + } + c := fakecontext.NewMockContext() + deleteTriggers(c) + }) + Convey("Test deleteTriggers with checkFunctionID err", t, func() { + ginCtx.Params = []gin.Param{ + {Key: "triggerId", Value: "function"}, + {Key: "functionId", Value: ""}, + } + c := fakecontext.NewMockContext() + deleteTriggers(c) + }) +} + +func Test_deleteTrigger(t *testing.T) { + w := httptest.NewRecorder() + ginCtx, _ := gin.CreateTestContext(w) + patchGin := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return ginCtx + }) + defer patchGin.Reset() + + Convey("Test deleteTrigger with triggerId invalid param err", t, func() { + ginCtx.Params = []gin.Param{ + gin.Param{Key: "triggerId", Value: "function"}, + } + c := fakecontext.NewMockContext() + deleteTrigger(c) + }) + Convey("Test deleteTrigger with checkTriggerID err", t, func() { + ginCtx.Params = []gin.Param{ + gin.Param{Key: "triggerId", Value: "function"}, + } + c := fakecontext.NewMockContext() + deleteTrigger(c) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/utils.go b/functionsystem/apps/meta_service/function_repo/router/utils.go new file mode 100644 index 0000000000000000000000000000000000000000..ee1182fa6417f1a4e3b5eacd2e1cbd08fc55fd6f --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/utils.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "encoding/json" + + "meta_service/common/logger/log" + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" + "meta_service/function_repo/utils" +) + +const ( + minPageIndex = 1 + maxPageSize = 1000 + minPageSize = 1 +) + +func shouldBindJSON(c server.Context, v interface{}) error { + if c.Gin().Request == nil || c.Gin().Request.Body == nil { + return errmsg.New(errmsg.InvalidJSONBody) + } + decoder := json.NewDecoder(c.Gin().Request.Body) + if err := decoder.Decode(v); err != nil { + log.GetLogger().Errorf("failed to decode body: %s", err.Error()) + if e, ok := err.(snerror.SNError); ok { + return e + } + return errmsg.New(errmsg.InvalidJSONBody) + } + return utils.Validate(v) +} diff --git a/functionsystem/apps/meta_service/function_repo/router/utils_test.go b/functionsystem/apps/meta_service/function_repo/router/utils_test.go new file mode 100644 index 0000000000000000000000000000000000000000..a719261c625b17721cecdc111d1cf41e11933021 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/router/utils_test.go @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package router + +import ( + "errors" + "net/http/httptest" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/gin-gonic/gin" + . "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/test/fakecontext" + "meta_service/function_repo/utils" +) + +func Test_queryPaging(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + ctx := fakecontext.NewMockContext() + mockPageIndex := "0" + mockPageSize := "0" + patch := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return c + }).ApplyMethod(reflect.TypeOf(c), "Query", func(c *gin.Context, key string) string { + if key == "pageIndex" { + return mockPageIndex + } + return mockPageSize + }) + defer patch.Reset() + Convey("Test queryPaging", t, func() { + mockPageIndex = "0" + _, _, err := utils.QueryPaging(ctx) + So(err, ShouldNotBeNil) + + mockPageIndex = "xxx" + _, _, err = utils.QueryPaging(ctx) + So(err, ShouldNotBeNil) + + mockPageIndex = "1" + + mockPageSize = "0" + _, _, err = utils.QueryPaging(ctx) + So(err, ShouldNotBeNil) + + mockPageSize = "xxx" + _, _, err = utils.QueryPaging(ctx) + So(err, ShouldNotBeNil) + + mockPageSize = "1" + _, _, err = utils.QueryPaging(ctx) + So(err, ShouldBeNil) + }) + +} + +func Test_shouldBindQuery(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "Gin", func( + *fakecontext.Context) *gin.Context { + return c + }).ApplyMethod(reflect.TypeOf(c), "ShouldBindQuery", func(c *gin.Context, obj interface{}) error { + return errors.New("mock err") + }) + defer patch.Reset() + Convey("Test shouldBindQuery", t, func() { + err := utils.ShouldBindQuery(ctx, 1) + So(err, ShouldNotBeNil) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/server/gin.go b/functionsystem/apps/meta_service/function_repo/server/gin.go new file mode 100644 index 0000000000000000000000000000000000000000..76dc1fd8a06eda6504ab155276a584a0f33d3371 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/server/gin.go @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "context" + "net/http" + "os" + "time" + + "github.com/gin-gonic/gin" + + "meta_service/common/healthcheck" + "meta_service/common/logger/log" + "meta_service/common/reader" + commontls "meta_service/common/tls" + "meta_service/function_repo/config" +) + +const ( + graceExitTime time.Duration = 30 +) + +type engine struct { + engine *gin.Engine + srv *http.Server +} + +// New returns a new http server engine powered by gin +func New() Engine { + return engine{ + engine: gin.New(), + } +} + +// Group implements Engine +func (e engine) Group(relativePath string, handlers ...HandlerFunc) RouterGroup { + return newRouterGroup(e.engine.Group(relativePath, makeHandlersChain(handlers)...)) +} + +// Run implements Engine +func (e engine) Run(addr string, stopCh <-chan struct{}) error { + e.srv = &http.Server{ + Addr: addr, + Handler: e.engine, + } + go func() { + if stopCh != nil { + <-stopCh + } + log.GetLogger().Warnf("received termination signal to shutdown") + e.GraceExit() + }() + var passPhase []byte + var err error + if config.RepoCfg.MutualTLSConfig.TLSEnable && config.RepoCfg.MutualTLSConfig.PwdFile != "" { + passPhase, err = reader.ReadFileWithTimeout(config.RepoCfg.MutualTLSConfig.PwdFile) + if err != nil { + log.GetLogger().Errorf("failed to read file PwdFile: %s", err.Error()) + return err + } + } + + if config.RepoCfg.MutualTLSConfig.TLSEnable { + ip := config.RepoCfg.ServerCfg.IP + if len(ip) == 0 { + ip = os.Getenv("POD_IP") + } + go healthcheck.StartServeTLS(ip, config.RepoCfg.MutualTLSConfig.ModuleCertFile, + config.RepoCfg.MutualTLSConfig.ModuleKeyFile, string(passPhase), nil) + } + if config.RepoCfg.MutualTLSConfig.TLSEnable { + e.srv.TLSConfig = commontls.NewTLSConfig( + commontls.BuildServerTLSConfOpts(config.RepoCfg.MutualTLSConfig)...) + err = e.srv.ListenAndServeTLS("", "") + } else { + err = e.srv.ListenAndServe() + } + if err != nil { + log.GetLogger().Errorf("failed to serve, err: %s", err.Error()) + return err + } + return nil +} + +// GraceExit graceful exit when receive a SIGTERM signal +func (e engine) GraceExit() { + if e.srv == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), graceExitTime*time.Second) + defer cancel() + if err := e.srv.Shutdown(ctx); err != nil { + log.GetLogger().Errorf("failed to shutdown repository: %s", err.Error()) + } + log.GetLogger().Info("graceful exit") +} + +type routerGroup struct { + rg *gin.RouterGroup +} + +func newRouterGroup(rg *gin.RouterGroup) RouterGroup { + return routerGroup{rg} +} + +// GET implements RouterGroup +func (rg routerGroup) GET(relativePath string, handlers ...HandlerFunc) { + rg.rg.GET(relativePath, makeHandlersChain(handlers)...) +} + +// POST implements RouterGroup +func (rg routerGroup) POST(relativePath string, handlers ...HandlerFunc) { + rg.rg.POST(relativePath, makeHandlersChain(handlers)...) +} + +// DELETE implements RouterGroup +func (rg routerGroup) DELETE(relativePath string, handlers ...HandlerFunc) { + rg.rg.DELETE(relativePath, makeHandlersChain(handlers)...) +} + +// PUT implements RouterGroup +func (rg routerGroup) PUT(relativePath string, handlers ...HandlerFunc) { + rg.rg.PUT(relativePath, makeHandlersChain(handlers)...) +} + +// Group implements RouterGroup +func (rg routerGroup) Group(relativePath string, handlers ...HandlerFunc) RouterGroup { + return routerGroup{ + rg.rg.Group(relativePath, makeHandlersChain(handlers)...), + } +} + +// Use implements RouterGroup +func (rg routerGroup) Use(middleware ...HandlerFunc) { + rg.rg.Use(makeHandlersChain(middleware)...) +} + +type ctx struct { + ginCtx *gin.Context +} + +// NewContext - +func NewContext(c *gin.Context) Context { + return ctx{c} +} + +// Gin implements Context +func (c ctx) Gin() *gin.Context { + return c.ginCtx +} + +// Context implements Context +func (c ctx) Context() context.Context { + return c.Gin().Request.Context() +} + +// InitTenantInfo implements Context +func (c ctx) InitTenantInfo(info TenantInfo) { + c.ginCtx.Set("tenant", info) +} + +// TenantInfo implements Context +func (c ctx) TenantInfo() (TenantInfo, error) { + raw, exist := c.ginCtx.Get("tenant") + if !exist { + return TenantInfo{}, ErrNoTenantInfo + } + info, ok := raw.(TenantInfo) + + var err error + if !ok { + err = ErrNoTenantInfo + } + return info, err +} + +// InitResourceInfo implements Context +func (c ctx) InitResourceInfo(info ResourceInfo) { + c.ginCtx.Set("resource", info) +} + +// ResourceInfo implements Context +func (c ctx) ResourceInfo() ResourceInfo { + raw, exist := c.ginCtx.Get("resource") + if !exist { + return ResourceInfo{} + } + info, ok := raw.(ResourceInfo) + + if !ok { + info = ResourceInfo{} + } + return info +} + +func makeHandlersChain(handlers []HandlerFunc) []gin.HandlerFunc { + chain := make([]gin.HandlerFunc, 0, len(handlers)) + for _, handler := range handlers { + chain = append(chain, func(c *gin.Context) { + handler(NewContext(c)) + }) + } + return chain +} + +// ServeHTTP A Handler responds to an HTTP request +func (e engine) ServeHTTP(w http.ResponseWriter, req *http.Request) { + e.engine.ServeHTTP(w, req) +} diff --git a/functionsystem/apps/meta_service/function_repo/server/gin_test.go b/functionsystem/apps/meta_service/function_repo/server/gin_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1c2187331f0f24d225093def328dc1c9356f4134 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/server/gin_test.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/stretchr/testify/assert" + + "meta_service/common/tls" + "meta_service/function_repo/config" +) + +var ( + repoCfg = &config.Configs{ + MutualTLSConfig: tls.MutualTLSConfig{ + TLSEnable: true, + RootCAFile: "fake/path", + ModuleKeyFile: "fake/path/to/key", + ModuleCertFile: "fake/path/to/cert", + ServerName: "test", + }, + } + e = engine{} +) + +func Test_engine_Run(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyGlobalVar(&config.RepoCfg, repoCfg) + assert.NotNil(t, e.Run("test")) +} diff --git a/functionsystem/apps/meta_service/function_repo/server/interface.go b/functionsystem/apps/meta_service/function_repo/server/interface.go new file mode 100644 index 0000000000000000000000000000000000000000..0d6966d444157f26e42892d9c19c42eb2ec10f13 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/server/interface.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package server wraps useful methods creating a http server +package server + +import ( + "context" + "net/http" + + "github.com/gin-gonic/gin" + + "meta_service/common/snerror" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" +) + +var ( + // ErrNoTenantInfo means tenant info does not exist. + ErrNoTenantInfo = snerror.New(errmsg.NoTenantInfo, "tenantInfo does not exist") +) + +// TenantInfo has tenant information including business ID, tenant ID and product ID. +type TenantInfo struct { + BusinessID, TenantID, ProductID string +} + +// ResourceInfo includes basic information useful for logging. +type ResourceInfo struct { + ResourceName, Qualifier, FunctionVersionNumber string + FunctionType config.FunctionType +} + +// Engine defines a http server engine +type Engine interface { + Group(relativePath string, handlers ...HandlerFunc) RouterGroup + Run(addr string, stopCh <-chan struct{}) error + GraceExit() + ServeHTTP(w http.ResponseWriter, req *http.Request) +} + +// RouterGroup is used internally to configure router, a RouterGroup is associated with a prefix and an array of +// handlers (middleware). +type RouterGroup interface { + GET(relativePath string, handlers ...HandlerFunc) + POST(relativePath string, handlers ...HandlerFunc) + DELETE(relativePath string, handlers ...HandlerFunc) + PUT(relativePath string, handlers ...HandlerFunc) + Group(relativePath string, handlers ...HandlerFunc) RouterGroup + Use(handlers ...HandlerFunc) +} + +// HandlerFunc defines the handler used by middleware as return value. +type HandlerFunc func(Context) + +// Context defines information whose life cycle is within a http request +type Context interface { + Gin() *gin.Context + Context() context.Context + InitTenantInfo(TenantInfo) + TenantInfo() (TenantInfo, error) + InitResourceInfo(ResourceInfo) + ResourceInfo() ResourceInfo +} diff --git a/functionsystem/apps/meta_service/function_repo/service/alias.go b/functionsystem/apps/meta_service/function_repo/service/alias.go new file mode 100644 index 0000000000000000000000000000000000000000..dc42bd5fc39707d1840567d6cf977f08c5debce5 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/alias.go @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "time" + + "meta_service/common/logger/log" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/storage/publish" + "meta_service/function_repo/utils" +) + +const ( + totalWeight = 100 +) + +// CheckFuncVersion check if alias's function version exist +func CheckFuncVersion(txn *storage.Txn, fName, fVersion string) error { + isExist, err := storage.IsFuncVersionExist(txn, fName, fVersion) + if err != nil { + log.GetLogger().Errorf("failed to check function version, error: %s", err.Error()) + return err + } + if !isExist { + return errmsg.New(errmsg.FunctionVersionNotFound, fName, fVersion) + } + return nil +} + +// CheckAliasName check if alias's name exist +func CheckAliasName(txn *storage.Txn, fName, aliasName string) error { + isExist, err := storage.IsAliasNameExist(txn, fName, aliasName) + if err != nil { + log.GetLogger().Errorf("failed to check alias name, error: %s", err.Error()) + return err + } + if isExist { + return errmsg.New(errmsg.AliasNameAlreadyExists) + } + // aliasName doesn't exist, check successfully + return nil +} + +// CheckFunctionAliasNum check if total alias number whether is out of range +func CheckFunctionAliasNum(txn *storage.Txn, funcName string) error { + number, err := storage.GetAliasNumByFunctionName(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to get alias num by function name version, error: %s", err.Error()) + return err + } + if uint(number) >= config.RepoCfg.FunctionCfg.AliasMax { + return errmsg.New(errmsg.AliasOutOfLimit, config.RepoCfg.FunctionCfg.AliasMax) + } + return nil +} + +// CheckRevisionID check if revisionID is the same +func CheckRevisionID(txn *storage.Txn, fName, aName, revisionID string) (storage.AliasValue, error) { + alias, err := storage.GetAlias(txn, fName, aName) + if err != nil { + if err == errmsg.KeyNotFoundError { + return alias, errmsg.New(errmsg.AliasNameNotFound, utils.RemoveServiceID(fName), aName) + } + log.GetLogger().Errorf("failed to get alias, error: %s", err.Error()) + return alias, err + } + + if revisionID != alias.RevisionID { + return storage.AliasValue{}, errmsg.New(errmsg.RevisionIDError) + } + return alias, nil +} + +// CheckRoutingConfig check if routingConfig is illegal +func CheckRoutingConfig(txn *storage.Txn, fName string, routingConfig map[string]int) error { + sum := 0 + for k, v := range routingConfig { + if err := CheckFuncVersion(txn, fName, k); err != nil { + return errmsg.New(errmsg.FunctionVersionNotFound, fName, k) + } + sum += v + } + if sum != totalWeight { + return errmsg.New(errmsg.TotalRoutingWeightNotOneHundred) + } + return nil +} + +func buildAlias(fName string, req model.AliasRequest) storage.AliasValue { + revisionID := utils.GetUTCRevisionID() + alias := storage.AliasValue{ + Name: req.Name, + FunctionName: fName, + FunctionVersion: req.FunctionVersion, + RevisionID: revisionID, + Description: req.Description, + RoutingConfig: req.RoutingConfig, + CreateTime: time.Now(), + UpdateTime: time.Now(), + } + return alias +} + +func buildAliasResponse(ctx server.Context, alias storage.AliasValue) (model.AliasResponse, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return model.AliasResponse{}, err + } + aliasURN := utils.BuildAliasURN(t.BusinessID, t.TenantID, t.ProductID, alias.FunctionName, alias.Name) + response := model.AliasResponse{ + AliasURN: aliasURN, + Name: alias.Name, + FunctionVersion: alias.FunctionVersion, + Description: alias.Description, + RevisionID: alias.RevisionID, + RoutingConfig: alias.RoutingConfig, + } + return response, nil +} + +func buildAliasListResponse(ctx server.Context, aliases []storage.AliasValue) (model.AliasListQueryResponse, error) { + resp := model.AliasListQueryResponse{} + resp.Total = len(aliases) + resp.Aliases = make([]model.AliasResponse, resp.Total) + for i, v := range aliases { + aliasResp, err := buildAliasResponse(ctx, v) + if err != nil { + return model.AliasListQueryResponse{}, err + } + resp.Aliases[i] = aliasResp + } + return resp, nil +} + +func buildAliasEtcd(ctx server.Context, alias storage.AliasValue) (publish.AliasEtcd, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return publish.AliasEtcd{}, err + } + + aliasURN := utils.BuildAliasURN(t.BusinessID, t.TenantID, t.ProductID, alias.FunctionName, alias.Name) + funcURN := utils.BuildFunctionURN(t.BusinessID, t.TenantID, t.ProductID, alias.FunctionName) + funcVerURN := utils.BuildFunctionVersionURN( + t.BusinessID, t.TenantID, t.ProductID, alias.FunctionName, alias.FunctionVersion) + + aliasEtcd := publish.AliasEtcd{ + AliasURN: aliasURN, + FunctionURN: funcURN, + FunctionVersionURN: funcVerURN, + Name: alias.Name, + FunctionVersion: alias.FunctionVersion, + RevisionID: alias.RevisionID, + Description: alias.Description, + } + aliasEtcd.RoutingConfig = make([]publish.AliasRoutingEtcd, len(alias.RoutingConfig)) + i := 0 + for k, v := range alias.RoutingConfig { + funcVerURN = utils.BuildFunctionVersionURN(t.BusinessID, t.TenantID, t.ProductID, alias.FunctionName, k) + aliasEtcd.RoutingConfig[i].FunctionVersionURN = funcVerURN + aliasEtcd.RoutingConfig[i].Weight = v + i++ + } + + return aliasEtcd, nil +} + +func mergeAlias(req model.AliasUpdateRequest, alias storage.AliasValue) storage.AliasValue { + alias.FunctionVersion = req.FunctionVersion + if req.Description != "" { + alias.Description = req.Description + } + if len(req.RoutingConfig) != 0 { + alias.RoutingConfig = req.RoutingConfig + } + alias.UpdateTime = time.Now() + return alias +} + +// CreateAlias create alias implement +func CreateAlias(ctx server.Context, fName string, req model.AliasRequest) ( + resp model.AliasCreateResponse, err error) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + if err = CheckFuncVersion(txn, fName, req.FunctionVersion); err != nil { + log.GetLogger().Errorf("failed to check function version, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + if err = CheckAliasName(txn, fName, req.Name); err != nil { + log.GetLogger().Errorf("failed to check alias name, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + if err = CheckFunctionAliasNum(txn, fName); err != nil { + log.GetLogger().Errorf("failed to check function alias number, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + if err = CheckRoutingConfig(txn, fName, req.RoutingConfig); err != nil { + log.GetLogger().Errorf("failed to check routing configuration, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + + alias := buildAlias(fName, req) + err = storage.CreateAlias(txn, alias) + if err != nil { + log.GetLogger().Errorf("failed to save alias, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + + aliasEtcd, err := buildAliasEtcd(ctx, alias) + if err != nil { + log.GetLogger().Errorf("failed to build alias for etcd watch, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + + err = publish.CreateAliasEtcd(txn, fName, aliasEtcd) + if err != nil { + log.GetLogger().Errorf("failed to save aliasEtcd, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + + // commit transaction to storage + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit create alias transaction, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + + aliasRes, err := buildAliasResponse(ctx, alias) + if err != nil { + log.GetLogger().Errorf("failed to build alias response, error: %s", err.Error()) + return model.AliasCreateResponse{}, err + } + + response := model.AliasCreateResponse{ + AliasResponse: aliasRes, + } + return response, nil +} + +// UpdateAlias update alias implement +func UpdateAlias(ctx server.Context, fName, aName string, req model.AliasUpdateRequest) ( + resp model.AliasUpdateResponse, err error) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + alias, err := CheckRevisionID(txn, fName, aName, req.RevisionID) + if err != nil { + log.GetLogger().Errorf("failed to check alias, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + if err = CheckFuncVersion(txn, fName, req.FunctionVersion); err != nil { + log.GetLogger().Errorf("failed to check function version, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + if req.RoutingConfig != nil && len(req.RoutingConfig) != 0 { + if err = CheckRoutingConfig(txn, fName, req.RoutingConfig); err != nil { + log.GetLogger().Errorf("failed to check routing configuration, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + } + + updateAlias := mergeAlias(req, alias) + err = storage.CreateAlias(txn, updateAlias) + if err != nil { + log.GetLogger().Errorf("failed to save alias, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + + aliasEtcd, err := buildAliasEtcd(ctx, updateAlias) + if err != nil { + log.GetLogger().Errorf("failed to build alias for etcd watch, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + err = publish.CreateAliasEtcd(txn, fName, aliasEtcd) + if err != nil { + log.GetLogger().Errorf("failed to save aliasEtcd, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + + // commit transaction to storage + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit update alias transaction, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + + aliasRes, err := buildAliasResponse(ctx, updateAlias) + if err != nil { + log.GetLogger().Errorf("failed to build alias response, error: %s", err.Error()) + return model.AliasUpdateResponse{}, err + } + updateRes := model.AliasUpdateResponse{ + AliasResponse: aliasRes, + } + + return updateRes, nil +} + +// DeleteAlias delete alias implement +func DeleteAlias(ctx server.Context, req model.AliasDeleteRequest) (err error) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + alias, err := storage.GetAlias(txn, req.FunctionName, req.AliasName) + if err != nil { + log.GetLogger().Errorf(" failed to get alias, error: %s", err.Error()) + return err + } + + var routingVers []string + for k := range alias.RoutingConfig { + routingVers = append(routingVers, k) + } + + if req.AliasName != "" { + err = storage.DeleteAlias(txn, req.FunctionName, req.AliasName, routingVers) + } else { + err = storage.DeleteAliasByFunctionName(txn, req.FunctionName) + } + if err != nil { + return err + } + + // commit transaction to storage + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit delete alias transaction, error: %s", err.Error()) + return err + } + + return nil +} + +// GetAlias get single alias information +func GetAlias(ctx server.Context, req model.AliasQueryRequest) (resp model.AliasQueryResponse, err error) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + alias, err := storage.GetAlias(txn, req.FunctionName, req.AliasName) + if err != nil { + if err == errmsg.KeyNotFoundError { + return model.AliasQueryResponse{}, errmsg.New( + errmsg.AliasNameNotFound, utils.RemoveServiceID(req.FunctionName), req.AliasName) + } + log.GetLogger().Errorf("failed to get alias, error: %s", err.Error()) + return model.AliasQueryResponse{}, err + } + + // commit transaction to storage + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit get alias transaction, error: %s", err.Error()) + return model.AliasQueryResponse{}, err + } + + aliasRes, err := buildAliasResponse(ctx, alias) + if err != nil { + log.GetLogger().Errorf("failed to build alias response, error: %s", err.Error()) + return model.AliasQueryResponse{}, err + } + getAliasRes := model.AliasQueryResponse{ + AliasResponse: aliasRes, + } + + return getAliasRes, nil +} + +// GetAliaseList get aliases list +func GetAliaseList(ctx server.Context, req model.AliasListQueryRequest) ( + resp model.AliasListQueryResponse, err error) { + aliases, err := storage.GetAliasesByPage(ctx, req.FunctionName, req.PageIndex, req.PageSize) + if err != nil { + log.GetLogger().Errorf("failed to get alias list, error: %s", err.Error()) + return model.AliasListQueryResponse{}, err + } + + aliasesResp, err := buildAliasListResponse(ctx, aliases) + if err != nil { + log.GetLogger().Errorf("failed to build aliases response, error: %s", err.Error()) + return model.AliasListQueryResponse{}, err + } + + return aliasesResp, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/service/alias_test.go b/functionsystem/apps/meta_service/function_repo/service/alias_test.go new file mode 100644 index 0000000000000000000000000000000000000000..eb838e970d3f9f74846b4e179b3653fcef3f3a29 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/alias_test.go @@ -0,0 +1,549 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "errors" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/storage/publish" + "meta_service/function_repo/test/fakecontext" +) + +func TestCreateAlias(t *testing.T) { + type args struct { + ctx server.Context + req model.AliasRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test create alias", args{fakecontext.NewMockContext(), + model.AliasRequest{ + Name: "test", + FunctionVersion: "", + Description: "des", + RoutingConfig: nil, + }}, + model.FunctionVersion{}, false}, + } + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + _, err := CreateAlias(tt.args.ctx, "name", tt.args.req) + assert.NotEqual(t, err, nil) + }) +} + +func createAliasPatches(setErr int) [10]*gomonkey.Patches { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(CheckFuncVersion, func(txn *storage.Txn, fName, fVersion string) error { + if setErr == 0 { + return errors.New("mock err CheckFuncVersion") + } + return nil + }), + gomonkey.ApplyFunc(CheckAliasName, func(txn *storage.Txn, fName, aliasName string) error { + if setErr == 1 { + return errors.New("mock err CheckAliasName") + } + return nil + }), + gomonkey.ApplyFunc(CheckFunctionAliasNum, func(txn *storage.Txn, funcName string) error { + if setErr == 2 { + return errors.New("mock err CheckFunctionAliasNum") + } + return nil + }), + gomonkey.ApplyFunc(CheckRoutingConfig, func(*storage.Txn, string, map[string]int) error { + if setErr == 3 { + return errors.New("mock err CheckRoutingConfig") + } + return nil + }), + gomonkey.ApplyFunc(storage.CreateAlias, func(txn *storage.Txn, alias storage.AliasValue) error { + if setErr == 4 { + return errors.New("mock err storage.CreateAlias") + } + return nil + }), + gomonkey.ApplyFunc(buildAliasEtcd, func(server.Context, storage.AliasValue) (publish.AliasEtcd, error) { + if setErr == 5 { + return publish.AliasEtcd{}, errors.New("mock err buildAliasEtcd") + } + return publish.AliasEtcd{}, nil + }), + gomonkey.ApplyFunc(publish.CreateAliasEtcd, func(*storage.Txn, string, publish.AliasEtcd) error { + if setErr == 6 { + return errors.New("mock err publish.CreateAliasEtcd") + } + return nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&storage.Txn{}), "Commit", func(tx *storage.Txn) error { + if setErr == 7 { + return errors.New("mock err Commit") + } + return nil + }), + gomonkey.ApplyFunc(buildAliasResponse, func(server.Context, storage.AliasValue) (model.AliasResponse, error) { + if setErr == 8 { + return model.AliasResponse{}, errors.New("mock err buildAliasResponse") + } + return model.AliasResponse{}, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&storage.Txn{}), "Cancel", func(tx *storage.Txn) { + return + }), + } + return patches +} + +func TestCreateAlias2(t *testing.T) { + req := model.AliasRequest{ + Name: "test", + FunctionVersion: "", + Description: "des", + RoutingConfig: nil, + } + Convey("Test CreateAlias 2", t, func() { + ctx := fakecontext.NewMockContext() + for i := 0; i <= 9; i++ { + fmt.Println("case ", i) + patches := createAliasPatches(i) + _, err := CreateAlias(ctx, "mock-name", req) + if i == 9 { + So(err, ShouldBeNil) + } else { + So(err, ShouldNotBeNil) + } + for j, _ := range patches { + patches[j].Reset() + } + } + + }) +} + +func TestUpdateAlias(t *testing.T) { + type args struct { + ctx server.Context + req model.AliasUpdateRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test update alias", args{fakecontext.NewMockContext(), + model.AliasUpdateRequest{ + FunctionVersion: "", + Description: "des", + RevisionID: "rev", + RoutingConfig: nil, + }}, + model.FunctionVersion{}, false}, + } + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + _, err := UpdateAlias(tt.args.ctx, "name", "alias", tt.args.req) + assert.NotEqual(t, err, nil) + }) +} + +func TestUpdateAlias2(t *testing.T) { + patch := gomonkey.ApplyFunc(storage.GetAlias, func(txn *storage.Txn, funcName, aliasName string) ( + storage.AliasValue, error) { + return storage.AliasValue{}, errmsg.KeyNotFoundError + }) + defer patch.Reset() + + req := model.AliasUpdateRequest{ + FunctionVersion: "", + Description: "des", + RevisionID: "rev", + RoutingConfig: nil, + } + ctx := fakecontext.NewMockContext() + _, err := UpdateAlias(ctx, "name", "alias", req) + assert.NotEqual(t, errmsg.EtcdInternalError, err) + + patch = gomonkey.ApplyFunc(storage.GetAlias, func(txn *storage.Txn, funcName, aliasName string) ( + storage.AliasValue, error) { + return storage.AliasValue{}, errmsg.EtcdInternalError + }) + defer patch.Reset() + _, err = UpdateAlias(ctx, "name", "alias", req) + assert.Equal(t, errmsg.EtcdInternalError, err) + + patch = gomonkey.ApplyFunc(storage.GetAlias, func(txn *storage.Txn, funcName, aliasName string) ( + storage.AliasValue, error) { + return storage.AliasValue{RevisionID: "rev2"}, nil + }) + defer patch.Reset() + _, err = UpdateAlias(ctx, "name", "alias", req) + assert.NotNil(t, err) + + patch = gomonkey.ApplyFunc(storage.GetAlias, func(txn *storage.Txn, funcName, aliasName string) ( + storage.AliasValue, error) { + return storage.AliasValue{RevisionID: "rev"}, nil + }) + defer patch.Reset() + patch2 := gomonkey.ApplyFunc(storage.IsFuncVersionExist, func(txn *storage.Txn, funcName, funcVersion string) (bool, error) { + return false, errmsg.EtcdInternalError + }) + defer patch2.Reset() + _, err = UpdateAlias(ctx, "name", "alias", req) + assert.Equal(t, errmsg.EtcdInternalError, err) + + patch2 = gomonkey.ApplyFunc(storage.IsFuncVersionExist, func(txn *storage.Txn, funcName, funcVersion string) (bool, error) { + return false, nil + }) + defer patch2.Reset() + _, err = UpdateAlias(ctx, "name", "alias", req) + assert.NotNil(t, err) + + req = model.AliasUpdateRequest{ + FunctionVersion: "", + Description: "des", + RevisionID: "rev", + RoutingConfig: map[string]int{"1": 50, "2": 50}, + } + patch2 = gomonkey.ApplyFunc(storage.IsFuncVersionExist, func(txn *storage.Txn, funcName, funcVersion string) (bool, error) { + return true, nil + }) + defer patch2.Reset() + patch3 := gomonkey.ApplyFunc(storage.CreateAlias, func(txn *storage.Txn, alias storage.AliasValue) error { + return nil + }) + defer patch3.Reset() + patch4 := gomonkey.ApplyFunc(publish.CreateAliasEtcd, func(txn *storage.Txn, funcName string, aliasEtcd publish.AliasEtcd) error { + return errmsg.EtcdInternalError + }) + defer patch4.Reset() + _, err = UpdateAlias(ctx, "name", "alias", req) + assert.Equal(t, errmsg.EtcdInternalError, err) +} + +func TestDeleteAlias(t *testing.T) { + type args struct { + ctx server.Context + req model.AliasDeleteRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test delete alias", args{fakecontext.NewMockContext(), + model.AliasDeleteRequest{ + FunctionName: "", + AliasName: "des", + }}, + model.FunctionVersion{}, false}, + } + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + err := DeleteAlias(tt.args.ctx, tt.args.req) + assert.NotEqual(t, err, nil) + }) +} + +func TestGetAlias(t *testing.T) { + type args struct { + ctx server.Context + req model.AliasQueryRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test delete alias", args{fakecontext.NewMockContext(), + model.AliasQueryRequest{ + FunctionName: "", + AliasName: "des", + }}, + model.FunctionVersion{}, false}, + } + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + _, err := GetAlias(tt.args.ctx, tt.args.req) + assert.NotEqual(t, err, nil) + }) +} + +func TestBuildAlias(t *testing.T) { + Convey("Test buildAlias", t, func() { + aliasValue := buildAlias("fakeName", model.AliasRequest{ + Name: "", + FunctionVersion: "", + Description: "abc", + RoutingConfig: nil, + }) + So(aliasValue.Description, ShouldEqual, "abc") + }) +} + +func TestBuildAliasResponse(t *testing.T) { + Convey("Test buildAliasResponse with ctx nil", t, func() { + ctx := fakecontext.NewContext() + _, err := buildAliasResponse(ctx, storage.AliasValue{}) + So(err, ShouldNotBeNil) + }) + Convey("Test buildAliasResponse", t, func() { + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyGlobalVar(&config.RepoCfg, new(config.Configs)) + defer patch.Reset() + config.RepoCfg.URNCfg.Prefix = "p" + config.RepoCfg.URNCfg.Zone = "z" + _, err := buildAliasResponse(ctx, storage.AliasValue{}) + So(err, ShouldBeNil) + }) +} + +func TestCheckAliasName(t *testing.T) { + Convey("Test CheckAliasName with IsAliasNameExist err", t, func() { + txn := storage.Txn{} + patch := gomonkey.ApplyFunc(storage.IsAliasNameExist, func(txn *storage.Txn, fName, aName string) (bool, error) { + return false, errors.New("mock err") + }) + defer patch.Reset() + err := CheckAliasName(&txn, "mockFName", "mockAliasName") + So(err, ShouldNotBeNil) + }) + Convey("Test CheckAliasName with IsExist", t, func() { + txn := storage.Txn{} + patch := gomonkey.ApplyFunc(storage.IsAliasNameExist, func(txn *storage.Txn, fName, aName string) (bool, error) { + return true, nil + }) + defer patch.Reset() + err := CheckAliasName(&txn, "mockFName", "mockAliasName") + So(err, ShouldNotBeNil) + }) + Convey("Test CheckAliasName with not IsExist", t, func() { + txn := storage.Txn{} + patch := gomonkey.ApplyFunc(storage.IsAliasNameExist, func(txn *storage.Txn, fName, aName string) (bool, error) { + return false, nil + }) + defer patch.Reset() + err := CheckAliasName(&txn, "mockFName", "mockAliasName") + So(err, ShouldBeNil) + }) +} + +func TestCheckFunctionAliasNum(t *testing.T) { + Convey("Test CheckFunctionAliasNum with GetAliasNumByFunctionName err", t, func() { + txn := storage.Txn{} + patch := gomonkey.ApplyGlobalVar(&config.RepoCfg, new(config.Configs)) + patch = patch.ApplyFunc(storage.GetAliasNumByFunctionName, func(txn storage.Transaction, funcName string) (int, error) { + return 0, errors.New("mock err") + }) + defer patch.Reset() + err := CheckFunctionAliasNum(&txn, "mockName") + So(err, ShouldNotBeNil) + }) + Convey("Test CheckFunctionAliasNum with number > AliasMax", t, func() { + txn := storage.Txn{} + patch := gomonkey.ApplyGlobalVar(&config.RepoCfg, new(config.Configs)) + patch = patch.ApplyFunc(storage.GetAliasNumByFunctionName, func(txn storage.Transaction, funcName string) (int, error) { + return 1, nil + }) + defer patch.Reset() + err := CheckFunctionAliasNum(&txn, "mockName") + So(err, ShouldNotBeNil) + }) + Convey("Test CheckFunctionAliasNum", t, func() { + txn := storage.Txn{} + patch := gomonkey.ApplyGlobalVar(&config.RepoCfg, new(config.Configs)) + config.RepoCfg.FunctionCfg.AliasMax = 1 + patch = patch.ApplyFunc(storage.GetAliasNumByFunctionName, func(txn storage.Transaction, funcName string) (int, error) { + return 0, nil + }) + defer patch.Reset() + err := CheckFunctionAliasNum(&txn, "mockName") + So(err, ShouldBeNil) + }) +} + +func TestGetAliaseList(t *testing.T) { + Convey("Test GetAliaseList with GetAliasesByPage err", t, func() { + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyFunc(storage.GetAliasesByPage, + func(ctx server.Context, funcName string, pageIndex, pageSize int) ([]storage.AliasValue, error) { + return nil, errors.New("mock err") + }) + defer patch.Reset() + _, err := GetAliaseList(ctx, model.AliasListQueryRequest{}) + So(err, ShouldNotBeNil) + }) + Convey("Test GetAliaseList with buildAliasListResponse err", t, func() { + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyFunc(storage.GetAliasesByPage, + func(ctx server.Context, funcName string, pageIndex, pageSize int) ([]storage.AliasValue, error) { + return []storage.AliasValue{{}}, nil + }).ApplyFunc(buildAliasResponse, func(server.Context, storage.AliasValue) (model.AliasResponse, error) { + return model.AliasResponse{}, errors.New("mock err") + }) + defer patch.Reset() + _, err := GetAliaseList(ctx, model.AliasListQueryRequest{}) + So(err, ShouldNotBeNil) + }) + Convey("Test GetAliaseList", t, func() { + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyFunc(storage.GetAliasesByPage, + func(ctx server.Context, funcName string, pageIndex, pageSize int) ([]storage.AliasValue, error) { + return []storage.AliasValue{{}}, nil + }).ApplyFunc(buildAliasResponse, func(server.Context, storage.AliasValue) (model.AliasResponse, error) { + return model.AliasResponse{}, nil + }) + defer patch.Reset() + _, err := GetAliaseList(ctx, model.AliasListQueryRequest{}) + So(err, ShouldBeNil) + }) +} + +func TestDeleteAlias2(t *testing.T) { + Convey("Test DeleteAlias2", t, func() { + patches := gomonkey.NewPatches() + txn := &storage.Txn{} + patches.ApplyFunc(storage.NewTxn, func(c server.Context) *storage.Txn { + return txn + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Cancel", func(_ *storage.Txn) {}) + patches.ApplyFunc(storage.GetAlias, func(*storage.Txn, string, string) (storage.AliasValue, error) { + return storage.AliasValue{ + RoutingConfig: map[string]int{"abc": 123}, + }, nil + }) + defer patches.Reset() + req := model.AliasDeleteRequest{} + ctx := fakecontext.NewMockContext() + Convey("with AliasName != nil", func() { + patch := gomonkey.ApplyFunc(storage.DeleteAlias, func(*storage.Txn, string, string, []string) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return nil + }) + defer patch.Reset() + req.AliasName = "abc" + err := DeleteAlias(ctx, req) + So(err, ShouldBeNil) + }) + Convey("with AliasName == nil and DeleteAliasByFunctionName err", func() { + patch := gomonkey.ApplyFunc(storage.DeleteAliasByFunctionName, func(storage.Transaction, string) error { + return errors.New("mock err") + }) + defer patch.Reset() + req.AliasName = "" + err := DeleteAlias(ctx, req) + So(err, ShouldNotBeNil) + }) + Convey("with AliasName == nil", func() { + patch := gomonkey.ApplyFunc(storage.DeleteAliasByFunctionName, func(storage.Transaction, string) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return errors.New("mock err") + }) + defer patch.Reset() + req.AliasName = "" + err := DeleteAlias(ctx, req) + So(err, ShouldNotBeNil) + }) + + }) +} + +func TestGetAlias2(t *testing.T) { + Convey("Test GetAlias", t, func() { + ctx := fakecontext.NewMockContext() + txn := &storage.Txn{} + patches := gomonkey.NewPatches() + patches.ApplyMethod(reflect.TypeOf(txn), "Cancel", func(_ *storage.Txn) {}) + defer patches.Reset() + Convey("with GetAlias err", func() { + patch := gomonkey.ApplyFunc(storage.GetAlias, + func(*storage.Txn, string, string) (storage.AliasValue, error) { + return storage.AliasValue{}, errors.New("mock err") + }) + defer patch.Reset() + req := model.AliasQueryRequest{} + _, err := GetAlias(ctx, req) + So(err, ShouldNotBeNil) + }) + Convey("with Commit err", func() { + patch := gomonkey.ApplyFunc(storage.GetAlias, + func(*storage.Txn, string, string) (storage.AliasValue, error) { + return storage.AliasValue{}, nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return errors.New("mock err") + }) + defer patch.Reset() + req := model.AliasQueryRequest{} + _, err := GetAlias(ctx, req) + So(err, ShouldNotBeNil) + }) + Convey("with buildAliasResponse err", func() { + patch := gomonkey.ApplyFunc(storage.GetAlias, + func(*storage.Txn, string, string) (storage.AliasValue, error) { + return storage.AliasValue{}, nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", + func(_ *storage.Txn) error { + return nil + }).ApplyFunc(buildAliasResponse, + func(ctx server.Context, alias storage.AliasValue) (model.AliasResponse, error) { + return model.AliasResponse{}, errors.New("mock err") + }) + defer patch.Reset() + req := model.AliasQueryRequest{} + _, err := GetAlias(ctx, req) + So(err, ShouldNotBeNil) + }) + Convey("with no err", func() { + patch := gomonkey.ApplyFunc(storage.GetAlias, + func(*storage.Txn, string, string) (storage.AliasValue, error) { + return storage.AliasValue{}, nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", + func(_ *storage.Txn) error { + return nil + }).ApplyFunc(buildAliasResponse, + func(ctx server.Context, alias storage.AliasValue) (model.AliasResponse, error) { + return model.AliasResponse{}, nil + }) + defer patch.Reset() + req := model.AliasQueryRequest{} + _, err := GetAlias(ctx, req) + So(err, ShouldBeNil) + }) + + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/cron.go b/functionsystem/apps/meta_service/function_repo/service/cron.go new file mode 100644 index 0000000000000000000000000000000000000000..21b988b752109fe694230d49452eec3f1ca1703c --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/cron.go @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "meta_service/common/logger/log" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/storage" +) + +const ( + cronDuration = 24 * time.Hour +) + +// CRON runs CRON jobs. A typical CRON job involves cleaning up invalid resources / unwanted status left by a previous +// function-repository session. +func CRON(stopCh <-chan struct{}) { + wait.Until(func() { + deleteUncontrolledPackages() + }, cronDuration, stopCh) + log.GetLogger().Warnf("received termination signal to shutdown") +} + +func deleteUncontrolledPackages() { + tuples, err := storage.GetUncontrolledList() + if err != nil { + log.GetLogger().Warnf("failed to get uncontrolled list: %s", err.Error()) + return + } + for _, tuple := range tuples { + if err := pkgstore.Delete(tuple.Key.BucketID, tuple.Key.ObjectID); err != nil { + log.GetLogger().Warnf("failed to delete package from uncontrolled list, bucketID: %s, objectID: %s", + tuple.Key.BucketID, tuple.Key.ObjectID) + continue + } + if err := storage.RemoveUncontrolledByKey(tuple.Key); err != nil { + log.GetLogger().Warnf("failed to remove uncontrolled by key: %s", err.Error()) + continue + } + log.GetLogger().Debugf("delete uncontrolled package %+v successfully", tuple) + } +} diff --git a/functionsystem/apps/meta_service/function_repo/service/cron_test.go b/functionsystem/apps/meta_service/function_repo/service/cron_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d6636ef93f0068ad44cbf2855358cb791c29b8cf --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/cron_test.go @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service is processing service codes +package service + +import ( + "testing" + "time" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/storage" +) + +func Test_deleteUncontrolledPackages(t *testing.T) { + patch := gomonkey.ApplyFunc(storage.GetUncontrolledList, func() ([]storage.UncontrolledObjectTuple, error) { + t := make([]storage.UncontrolledObjectTuple, 1) + t1 := storage.UncontrolledObjectTuple{ + Key: storage.UncontrolledObjectKey{ + BucketID: "bID1", + ObjectID: "obj1", + }, + Value: storage.UncontrolledObjectValue{ + CreateTime: time.Time{}, + }, + } + t = append(t, t1) + return t, nil + }) + defer patch.Reset() + + deleteUncontrolledPackages() +} + +func TestCRON(t *testing.T) { + Convey("Test CRON", t, func() { + ch := make(chan struct{}) + go CRON(ch) + ch <- struct{}{} + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/function.go b/functionsystem/apps/meta_service/function_repo/service/function.go new file mode 100644 index 0000000000000000000000000000000000000000..972431ac4fb603af7eddc0e674622b9dd9cfda69 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/function.go @@ -0,0 +1,1146 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service is processing service codes +package service + +import ( + "encoding/json" + "errors" + "io" + "strconv" + + common "meta_service/common/constants" + "meta_service/common/functionhandler" + "meta_service/common/logger/log" + "meta_service/common/snerror" + "meta_service/common/timeutil" + "meta_service/common/urnutils" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/storage/publish" + "meta_service/function_repo/utils" + "meta_service/function_repo/utils/constants" +) + +const ( + formatIntBase int = 10 + nilMapValue = "{}" +) + +type uncontrolledInfo struct { + ctx server.Context + bucketID string + objectID string +} + +type functionInstanceAttribute struct { + minInstance int64 + maxInstance int64 + concurrentNum int +} + +// CreateFunctionInfo create function +func CreateFunctionInfo(ctx server.Context, req model.FunctionCreateRequest, isAdmin bool) (model.FunctionVersion, + error, +) { + // start transaction + txn := storage.GetTxnByKind(ctx, req.Kind) + defer txn.Cancel() + + version := urnutils.GetFunctionVersion(req.Kind) + f, err := storage.GetFunctionVersion(txn, req.Name, version) + if err == nil { + log.GetLogger().Errorf("function name %s is exist", f.Function.Name) + return model.FunctionVersion{}, errmsg.New(errmsg.FunctionNameExist) + } + err = checkFunctionLayerList(storage.GetTxnByKind(ctx, ""), req.Layers, req.Runtime) + if err != nil { + log.GetLogger().Errorf("failed to check layer :%s", err.Error()) + return model.FunctionVersion{}, err + } + functionVersion, err := buildFunctionVersion(req) + if err != nil { + log.GetLogger().Errorf("failed to build function version when creating :%s", err.Error()) + return model.FunctionVersion{}, err + } + functionVersion.FunctionVersion.Package.CodeUploadType = req.CodeUploadType + err = storage.CreateFunctionVersion(txn, functionVersion) + if err != nil { + log.GetLogger().Errorf("failed to create function when saving function: %s", err.Error()) + return model.FunctionVersion{}, err + } + err = storage.CreateFunctionStatus(txn, functionVersion.Function.Name, + functionVersion.FunctionVersion.Version) + if err != nil { + log.GetLogger().Errorf("failed to create function status when saving function: %s", err.Error()) + return model.FunctionVersion{}, err + } + if req.CodeUploadType != common.S3StorageType || !isAdmin { + err = publish.SavePublishFuncVersion(txn, functionVersion) + if err != nil { + log.GetLogger().Errorf("failed to create function mete data when saving function: %s", err.Error()) + return model.FunctionVersion{}, err + } + } + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to create function when committing: %s", err.Error()) + return model.FunctionVersion{}, err + } + return buildFuncResult(ctx, functionVersion) +} + +func buildFuncResult(ctx server.Context, + functionVersion storage.FunctionVersionValue, +) (model.FunctionVersion, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to create function can not get tenant info: %s", err.Error()) + return model.FunctionVersion{}, err + } + key := storage.FunctionVersionKey{ + TenantInfo: t, + FunctionName: functionVersion.Function.Name, + FunctionVersion: functionVersion.FunctionVersion.Version, + } + ret := buildFunctionVersionModel(key, functionVersion) + return ret, nil +} + +func checkFunctionLayerList(txn storage.Transaction, urns []string, runtime string) error { + layerMap := make(map[string]string, len(urns)) + for _, urn := range urns { + info, err := ParseLayerInfo(txn.GetCtx(), urn) + if err != nil { + log.GetLogger().Errorf("failed to get layer query info :%s", err.Error()) + return err + } + val, err := storage.GetLayerVersionTx(txn, info.LayerName, info.LayerVersion) + if err != nil { + log.GetLogger().Errorf("failed to get layer version :%s", err.Error()) + return err + } + + var found bool + for _, rt := range val.CompatibleRuntimes { + if rt == runtime { + found = true + break + } + } + // not exit + if !found { + return errmsg.NewParamError("runtime %s of layer [%s] is not compatible with "+ + "function's runtime [%s]", val.CompatibleRuntimes, info.LayerName, runtime) + } + + // repetitive + if layerMap[urn] == "" { + layerMap[urn] = urn + } else { + return errmsg.NewParamError( + "the URN list of layer's version is not unique, layer name is [%s]", info.LayerName) + } + } + return nil +} + +// UpdateFunctionInfo update function +func UpdateFunctionInfo(ctx server.Context, f model.FunctionQueryInfo, + fv model.FunctionUpdateRequest, isAdmin bool) ( + model.FunctionVersion, error, +) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + funcVersion, err := storage.GetFunctionByFunctionNameAndVersion(ctx, f.FunctionName, f.FunctionVersion, "") + if err != nil { + log.GetLogger().Errorf("failed to get function by function name and version :%s", err.Error()) + return model.FunctionVersion{}, err + } + if funcVersion.FunctionVersion.RevisionID != fv.RevisionID { + log.GetLogger().Errorf("revisionId is not the same as latest versions %s ,%s", + funcVersion.FunctionVersion.RevisionID, fv.RevisionID) + return model.FunctionVersion{}, errmsg.New(errmsg.RevisionIDError) + } + err = checkFunctionLayerList(txn, fv.Layers, funcVersion.FunctionVersion.Runtime) + if err != nil { + log.GetLogger().Errorf("failed to check layer :%s", err.Error()) + return model.FunctionVersion{}, err + } + err = buildUpdateFunctionVersion(fv, &funcVersion) + if err != nil { + log.GetLogger().Errorf("failed to build update function version :%s", err.Error()) + return model.FunctionVersion{}, err + } + funcVersion.FunctionVersion.Package.CodeUploadType = fv.CodeUploadType + err = storage.UpdateFunctionVersion(txn, funcVersion) + if err != nil { + log.GetLogger().Errorf("failed to update function version :%s", err.Error()) + return model.FunctionVersion{}, err + } + if fv.CodeUploadType != common.S3StorageType || !isAdmin { + err = publish.SavePublishFuncVersion(txn, funcVersion) + if err != nil { + log.GetLogger().Errorf("failed to create function mete data when saving function: %s", err.Error()) + return model.FunctionVersion{}, err + } + } + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit when update :%s", err.Error()) + return model.FunctionVersion{}, err + } + return buildFuncResult(ctx, funcVersion) +} + +func buildBasicUpdateFunctionVersion(request model.FunctionUpdateRequest, + fv *storage.FunctionVersionValue, +) { + fv.FunctionVersion.Handler = getChangeStringValue(fv.FunctionVersion.Handler, request.Handler) + fv.FunctionVersion.CPU = getChangeInt64Value(fv.FunctionVersion.CPU, request.CPU) + fv.FunctionVersion.Memory = getChangeInt64Value(fv.FunctionVersion.Memory, request.Memory) + fv.FunctionVersion.Timeout = getChangeInt64Value(fv.FunctionVersion.Timeout, request.Timeout) + fv.Function.Description = getChangeStringValue(fv.Function.Description, request.Description) + fv.FunctionVersion.HookHandler = getHookHandler(request.Kind, fv.FunctionVersion.Runtime, request.Handler, + request.HookHandler) + if request.ExtendedHandler != nil { + fv.FunctionVersion.ExtendedHandler = request.ExtendedHandler + } + if request.ExtendedTimeout != nil { + fv.FunctionVersion.ExtendedTimeout = request.ExtendedTimeout + } + fv.FunctionVersion.Package.StorageType = request.StorageType + fv.FunctionVersion.Package.CodePath = request.CodePath + fv.FunctionVersion.CacheInstance = request.CacheInstance + fv.FunctionVersion.Device = request.Device + poolLabel := utils.GetPoolLabels(request.ResourceAffinitySelectors) + if len(poolLabel) == 0 { + poolLabel = utils.GetPoolLabels(request.SchedulePolicies) + } + fv.FunctionVersion.PoolLabel = poolLabel + fv.FunctionVersion.PoolID = request.PoolID + fv.FunctionVersion.Package.BucketID = request.S3CodePath.BucketID + fv.FunctionVersion.Package.ObjectID = request.S3CodePath.ObjectID + fv.FunctionVersion.Package.BucketUrl = request.S3CodePath.BucketUrl + fv.FunctionVersion.Package.Token = request.S3CodePath.Token + fv.FunctionVersion.Package.Signature = request.S3CodePath.Sha512 +} + +func buildUpdateFunctionVersion(request model.FunctionUpdateRequest, + fv *storage.FunctionVersionValue, +) error { + buildBasicUpdateFunctionVersion(request, fv) + if request.CustomResources != nil { + bytes, err := json.Marshal(request.CustomResources) + if err != nil { + log.GetLogger().Errorf("failed to get marshal customResources :%s", err.Error()) + return err + } + fv.FunctionVersion.CustomResources = getChangeStringValue(fv.FunctionVersion.CustomResources, string(bytes)) + } + if request.Environment != nil { + env, err := utils.EncodeEnv(request.Environment) + if err != nil { + log.GetLogger().Errorf("failed to get encode env info :%s", err.Error()) + return err + } + fv.FunctionVersion.Environment = getChangeStringValue(fv.FunctionVersion.Environment, env) + } + if request.ConcurrentNum != "" { + concurrentNum, err := strconv.Atoi(request.ConcurrentNum) + if err != nil { + log.GetLogger().Errorf("failed to get conversion str concurrentNum %s :%s", concurrentNum, err.Error()) + return err + } + fv.FunctionVersion.ConcurrentNum = getChangeIntValue(fv.FunctionVersion.ConcurrentNum, concurrentNum) + } + if request.MinInstance != "" { + minInstance, err := strconv.ParseInt(request.MinInstance, 10, 64) + if err != nil { + log.GetLogger().Errorf("failed to get conversion str minInstance %s :%s", minInstance, err.Error()) + return err + } + fv.FunctionVersion.MinInstance = minInstance + } + if request.MaxInstance != "" { + maxInstance, err := strconv.ParseInt(request.MaxInstance, 10, 64) + if err != nil { + log.GetLogger().Errorf("failed to get conversion str maxInstance %s :%s", maxInstance, err.Error()) + return err + } + fv.FunctionVersion.MaxInstance = maxInstance + } + err := updateFunctionLayer(request, fv) + if err != nil { + log.GetLogger().Errorf("failed to get layers %s :%s", request.Layers, err.Error()) + return err + } + fv.FunctionVersion.RevisionID = utils.GetUTCRevisionID() + fv.Function.UpdateTime = utils.NowTimeF() + return nil +} + +func updateFunctionLayer(request model.FunctionUpdateRequest, fv *storage.FunctionVersionValue) error { + if len(request.Layers) == 0 { + fv.FunctionLayer = nil + return nil + } + functionLayer := make([]storage.FunctionLayer, len(request.Layers), len(request.Layers)) + for index, data := range request.Layers { + name, versionStr := utils.InterceptNameAndVersionFromLayerURN(data) + version, err := strconv.Atoi(versionStr) + if err != nil { + log.GetLogger().Errorf("failed to conversion str version %s ", versionStr) + return err + } + functionLayer[index] = storage.FunctionLayer{ + Name: name, + Version: version, + Order: index, + } + } + fv.FunctionLayer = functionLayer + return nil +} + +func getChangeStringValue(ov, v string) string { + if v != "" { + return v + } + return ov +} + +func getChangeIntValue(ov, v int) int { + if v != 0 { + return v + } + return ov +} + +func getChangeInt64Value(ov, v int64) int64 { + if v != 0 { + return v + } + return ov +} + +// build FunctionVersion struct +func buildFunctionVersion(request model.FunctionCreateRequest) (storage.FunctionVersionValue, error) { + functionLayer := make([]storage.FunctionLayer, len(request.Layers), len(request.Layers)) + for index, data := range request.Layers { + name, versionStr := utils.InterceptNameAndVersionFromLayerURN(data) + version, err := strconv.Atoi(versionStr) + if err != nil { + log.GetLogger().Errorf("failed to conversion str version %s ", versionStr) + return storage.FunctionVersionValue{}, err + } + functionLayer[index] = storage.FunctionLayer{ + Name: name, + Version: version, + Order: index, + } + } + var env string + var err error = nil + if request.Environment != nil { + env, err = utils.EncodeEnv(request.Environment) + if err != nil { + log.GetLogger().Errorf("failed to encode env %s ", err.Error()) + return storage.FunctionVersionValue{}, err + } + } + var customResources []byte + if request.CustomResources != nil { + customResources, err = json.Marshal(request.CustomResources) + if err != nil { + log.GetLogger().Warnf("failed to marshal customResources %s ", err) + return storage.FunctionVersionValue{}, err + } + } + return functionVersionResult(request, env, functionLayer, string(customResources)) +} + +func functionVersionResult(request model.FunctionCreateRequest, env string, + functionLayer []storage.FunctionLayer, customResources string, +) (storage.FunctionVersionValue, error) { + w, err := tranWorkerParams(request) + if err != nil { + log.GetLogger().Errorf("failed to tran worker params :%s", err.Error()) + return storage.FunctionVersionValue{}, err + } + function := storage.Function{ + Name: request.Name, + Description: request.Description, + Tag: request.Tags, + CreateTime: utils.NowTimeF(), + } + version := getFunctionVersion(request, env, w, customResources) + value := storage.FunctionVersionValue{ + Function: function, + FunctionVersion: version, + FunctionLayer: functionLayer, + } + return value, nil +} + +func getHookHandler(kind, runtime, handler string, hookMap map[string]string) map[string]string { + hookHandler := functionhandler.FunctionHookHandlerInfo{} + if len(hookMap) > 0 { + hookHandler.InitHandler = hookMap[functionhandler.InitHandler] + hookHandler.CallHandler = hookMap[functionhandler.CallHandler] + hookHandler.CheckpointHandler = hookMap[functionhandler.CheckpointHandler] + hookHandler.RecoverHandler = hookMap[functionhandler.RecoverHandler] + hookHandler.ShutdownHandler = hookMap[functionhandler.ShutdownHandler] + hookHandler.SignalHandler = hookMap[functionhandler.SignalHandler] + } + mapBuilder := functionhandler.GetBuilder(kind, runtime, handler) + if mapBuilder != nil { + return mapBuilder.HookHandler(runtime, hookHandler) + } + return map[string]string{} +} + +func getFunctionVersion(request model.FunctionCreateRequest, env string, + w functionInstanceAttribute, customResources string, +) storage.FunctionVersion { + poolLabel := utils.GetPoolLabels(request.ResourceAffinitySelectors) + if len(poolLabel) == 0 { + poolLabel = utils.GetPoolLabels(request.SchedulePolicies) + } + version := storage.FunctionVersion{ + Package: storage.Package{ + StorageType: request.StorageType, + LocalPackage: storage.LocalPackage{ + CodePath: request.CodePath, + }, + S3Package: storage.S3Package{ + BucketID: request.S3CodePath.BucketID, + ObjectID: request.S3CodePath.ObjectID, + BucketUrl: request.S3CodePath.BucketUrl, + Token: request.S3CodePath.Token, + Signature: request.S3CodePath.Sha512, + }, + }, + RevisionID: utils.GetUTCRevisionID(), + Handler: request.Handler, + CPU: request.CPU, + Memory: request.Memory, + Runtime: utils.GetRuntimeName(request.Kind, request.Runtime), + Timeout: request.Timeout, + Version: utils.GetDefaultVersion(), + Environment: env, + CustomResources: customResources, + Description: utils.GetDefaultVersion(), + PublishTime: utils.NowTimeF(), + MinInstance: w.minInstance, + MaxInstance: w.maxInstance, + ConcurrentNum: w.concurrentNum, + CacheInstance: request.CacheInstance, + HookHandler: getHookHandler(request.Kind, request.Runtime, request.Handler, request.HookHandler), + ExtendedHandler: request.ExtendedHandler, + ExtendedTimeout: request.ExtendedTimeout, + Device: request.Device, + PoolLabel: poolLabel, + PoolID: request.PoolID, + } + if request.Kind == common.Faas { + version.Kind = common.Faas + version.Version = utils.GetDefaultFaaSVersion() + version.Description = utils.GetDefaultFaaSVersion() + version.FuncName = urnutils.GetPureFaaSFunctionName(request.Name) + version.Service = urnutils.GetPureFaaSService(request.Name) + version.IsBridgeFunction = false + version.EnableAuthInHeader = false + } + return version +} + +func tranWorkerParams(request model.FunctionCreateRequest) (functionInstanceAttribute, error) { + minInstance, err := strconv.ParseInt(request.MinInstance, 10, 64) + if err != nil { + log.GetLogger().Errorf("failed to convert str %s :%s ", "minInstance", request.MinInstance) + return functionInstanceAttribute{}, err + } + maxInstance, err := strconv.ParseInt(request.MaxInstance, 10, 64) + if err != nil { + log.GetLogger().Errorf("failed to convert str %s :%s ", "maxInstance", request.MaxInstance) + return functionInstanceAttribute{}, err + } + concurrentNum, err := strconv.Atoi(request.ConcurrentNum) + if err != nil { + log.GetLogger().Errorf("failed to convert str %s :%s ", "concurrentNum", request.ConcurrentNum) + return functionInstanceAttribute{}, err + } + return functionInstanceAttribute{minInstance, maxInstance, concurrentNum}, nil +} + +func deletePackage(txn storage.Transaction, ctx server.Context, fvs []storage.FunctionVersionValue) (err error) { + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit delete function :%s", err.Error()) + return err + } + + err = deleteFunctionPackage(ctx, fvs) + if err != nil { + log.GetLogger().Errorf("failed to delete function package :%s", err.Error()) + return err + } + return nil +} + +// DeleteResponseForMeta delete function info +func DeleteResponseForMeta(ctx server.Context, tenantInfo server.TenantInfo, funcName string, + funVersion, kind string, +) (response model.FunctionDeleteResponse, err error) { + // start transaction + txn := storage.GetTxnByKind(ctx, kind) + defer txn.Cancel() + + // check whether the function exists + fvs, err := storage.GetFunctionVersions(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to get function version :%s", err.Error()) + return model.FunctionDeleteResponse{}, err + } + if len(fvs) > 0 { + // use real kind type + kind = fvs[0].FunctionVersion.Kind + } + if utils.IsLatestVersion(funVersion) { + err = deleteAllFunctions(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to delete functions :%s", err.Error()) + return model.FunctionDeleteResponse{}, err + } + publish.DeleteAllPublishFunction(txn, funcName, kind, tenantInfo) + publish.DeleteTraceChainInfo(txn, funcName, tenantInfo) + err := publish.DeleteAliasByFuncNameEtcd(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to delete alias by function name :%s", err.Error()) + return model.FunctionDeleteResponse{}, err + } + } else { + err = deleteFunctionByVersion(txn, funcName, funVersion) + publish.DeletePublishFunction(txn, funcName, tenantInfo, funVersion, kind) + if err != nil { + log.GetLogger().Errorf("failed to delete publishings :%s", err.Error()) + return model.FunctionDeleteResponse{}, err + } + } + // storage.FunctionVersion->model.FunctionValue + fvLen := len(fvs) + modelFvs := make([]model.FunctionVersion, fvLen) + for i := 0; i < fvLen; i++ { + modelFvs[i], err = buildFuncResult(ctx, fvs[i]) + if err != nil { + return model.FunctionDeleteResponse{}, err + } + } + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit delete function :%s", err.Error()) + return model.FunctionDeleteResponse{}, err + } + + return model.FunctionDeleteResponse{ + Total: len(modelFvs), + Versions: modelFvs, + }, nil +} + +// DeleteResponse delete function info +func DeleteResponse(ctx server.Context, funcName string, + funVersion, kind string, +) error { + // start transaction + txn := storage.GetTxnByKind(ctx, kind) + defer txn.Cancel() + + // check whether the function exists + fvs, err := storage.GetFunctionVersions(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to get function version :%s", err.Error()) + return err + } + + err = canDelFunctionPackage(txn, funVersion, fvs) + if err != nil { + log.GetLogger().Errorf("failed to check function package :%s", err.Error()) + return err + } + return deletePackage(txn, ctx, fvs) +} + +func canDelFunctionPackage(txn storage.Transaction, fv string, fvs []storage.FunctionVersionValue) error { + for index, data := range fvs { + var bucketID, objectID string + // if the default function version is used, delete all functions + if fv != utils.GetDefaultVersion() && fv != utils.GetDefaultFaaSVersion() { + if data.FunctionVersion.Version != fv { + fvs[index].Function.Name = constants.NilStringValue + continue + } + + bucketID = data.FunctionVersion.Package.BucketID + objectID = data.FunctionVersion.Package.ObjectID + + referred, err := storage.IsObjectReferred(txn, bucketID, objectID) + if err != nil { + log.GetLogger().Errorf("failed to check object referred :%s", err.Error()) + return err + } + if referred { + fvs[index].Function.Name = constants.NilStringValue + continue + } + } + err := storage.AddUncontrolledTx(txn, bucketID, objectID) + if err != nil { + log.GetLogger().Errorf("failed to add uncontrolled") + return err + } + } + return nil +} + +func deleteFunctionPackage(ctx server.Context, fvs []storage.FunctionVersionValue) error { + for _, v := range fvs { + if v.Function.Name == constants.NilStringValue { + continue + } + if v.FunctionVersion.Package.CodeUploadType != common.S3StorageType { + continue + } + bucketID := v.FunctionVersion.Package.BucketID + objectID := v.FunctionVersion.Package.ObjectID + err := pkgstore.Delete(bucketID, objectID) + if err != nil { + log.GetLogger().Warnf("failed to delete bucket file, bucketId is %s, objectId is %s,err：%s", + bucketID, objectID, err.Error()) + } + // remove uncontrolled + err = storage.RemoveUncontrolled(ctx, bucketID, objectID) + if err != nil { + log.GetLogger().Warnf("failed to remove uncontrolled package :%s ", err.Error()) + } + } + return nil +} + +func deleteFunctionByVersion(txn storage.Transaction, funcName string, funVersion string) error { + _, err := storage.GetAliasNumByFunctionName(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to get alias num by function name version :%s", err.Error()) + return err + } + exist, err := storage.AliasRoutingExist(txn, funcName, funVersion) + if err != nil { + log.GetLogger().Errorf("failed to check alias routing :%s", err.Error()) + return err + } + if exist { + log.GetLogger().Errorf("failed to check function alias route info "+ + "there is other alias list use the version %s", funVersion) + return snerror.NewWithFmtMsg(errmsg.FunctionVersionDeletionForbidden, + errmsg.ErrorMessage(errmsg.FunctionVersionDeletionForbidden)) + } + + err = storage.DeleteFunctionVersion(txn, funcName, funVersion) + if err != nil { + log.GetLogger().Errorf("failed to delete function version :%s", err.Error()) + return err + } + err = storage.DeleteFunctionStatus(txn, funcName, funVersion) + if err != nil { + log.GetLogger().Errorf("failed to delete function status :%s", err.Error()) + return err + } + + if err := DeleteTriggerByFuncNameVersion(txn, funcName, funVersion); err != nil { + log.GetLogger().Errorf("failed to delete trigger register info, error: %s", err.Error()) + return err + } + + return nil +} + +func deleteAllFunctions(txn storage.Transaction, funcName string) error { + err := storage.DeleteFunctionVersions(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to delete function versions :%s", err.Error()) + return err + } + err = storage.DeleteFunctionStatuses(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to delete function status :%s", err.Error()) + return err + } + // delete alias + err = storage.DeleteAliasByFunctionName(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to delete alias by function name :%s", err.Error()) + return err + } + + err = DeleteTriggerByFuncName(txn, funcName) + if err != nil { + log.GetLogger().Errorf("failed to delete trigger info by function name, error:%s", err.Error()) + return err + } + + return nil +} + +// GetFunction return function info +func GetFunction(ctx server.Context, req model.FunctionGetRequest) (resp model.FunctionGetResponse, err error) { + if req.AliasName != "" { + return getFunctionByAlias(ctx, req.FunctionName, req.AliasName, req.Kind) + } + return getFunctionByVersion(ctx, req.FunctionName, req.FunctionVersion, req.Kind) +} + +// GetFunctionVersionList return function version list info +func GetFunctionVersionList(ctx server.Context, f string, pi int, ps int) (resp model.FunctionVersionListGetResponse, + err error, +) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + tuples, total, err := storage.GetFunctionVersionList(ctx, f, "", pi, ps) + if err != nil { + log.GetLogger().Errorf("failed to get function version list :%s", err.Error()) + return model.FunctionVersionListGetResponse{}, err + } + versions := make([]model.FunctionVersion, len(tuples), len(tuples)) + for i, tuple := range tuples { + versions[i] = buildFunctionVersionModel(tuple.Key, tuple.Value) + } + return model.FunctionVersionListGetResponse{ + Versions: versions, + Total: total, + }, nil +} + +// GetFunctionList return function list info +func GetFunctionList(c server.Context, f string, q string, k string, + pi int, ps int, +) (model.FunctionListGetResponse, error) { + // start transaction + txn := storage.GetTxnByKind(c, k) + defer txn.Cancel() + tuples, total, err := storage.GetFunctionList(c, f, q, k, pi, ps) + if err != nil { + log.GetLogger().Errorf("failed to get function version list :%s", err.Error()) + return model.FunctionListGetResponse{}, nil + } + versions := make([]model.FunctionVersion, len(tuples), len(tuples)) + for i, tuple := range tuples { + versions[i] = buildFunctionVersionModel(tuple.Key, tuple.Value) + } + return model.FunctionListGetResponse{ + Total: total, + FunctionVersion: versions, + }, nil +} + +func getFunctionByVersion(c server.Context, f string, + v string, kind string, +) (model.FunctionGetResponse, error) { + version := v + if version == "" { + if kind == constants.Faas { + version = utils.GetDefaultFaaSVersion() + } else { + version = utils.GetDefaultVersion() + } + } + var fv storage.FunctionVersionValue + var err error + fv, err = storage.GetFunctionByFunctionNameAndVersion(c, f, version, kind) + if err != nil { + log.GetLogger().Errorf("failed to get function by function name and version :%s", err.Error()) + return model.FunctionGetResponse{}, err + } + return buildGetFunctionResponse(c, fv, "") +} + +func getFunctionByAlias(c server.Context, f string, a string, + kind string, +) (model.FunctionGetResponse, error) { + version := storage.GetFunctionByFunctionNameAndAlias(c, f, a) + return getFunctionByVersion(c, f, version, kind) +} + +func buildGetFunctionResponse(c server.Context, version storage.FunctionVersionValue, + dl string, +) (model.FunctionGetResponse, error) { + t, err := c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error :%s", err.Error()) + return model.FunctionGetResponse{}, err + } + dateTime := buildTime(version) + var function model.Function + function = buildFunction(dateTime, t, version) + + env, err := utils.DecodeEnv(version.FunctionVersion.Environment) + if err != nil { + log.GetLogger().Errorf("failed to decode env :%s", err.Error()) + return model.FunctionGetResponse{}, err + } + var layerURNs []string + for _, v := range version.FunctionLayer { + layer, err := GetLayerVersion(c, v.Name, v.Version) + if err != nil { + log.GetLogger().Errorf("failed to get layer info :%s", err.Error()) + return model.FunctionGetResponse{}, err + } + layerURNs = append(layerURNs, layer.LayerVersionURN) + } + functionVersionURN := switchBuildFuncVersionURN(t, version) + v := buildFunctionVersionInfo(function, functionVersionURN, version, layerURNs, env) + return model.FunctionGetResponse{ + FunctionVersion: v, + Created: version.Function.CreateTime, + }, nil +} + +func switchBuildFuncVersionURN(t server.TenantInfo, version storage.FunctionVersionValue) string { + return utils.BuildFunctionVersionURNWithTenant(t, version.Function.Name, version.FunctionVersion.Version) +} + +func buildFunctionVersionInfo(function model.Function, functionVersionURN string, + version storage.FunctionVersionValue, layerURNs []string, env map[string]string, +) model.FunctionVersion { + v := model.FunctionVersion{ + Function: function, + FunctionVersionURN: functionVersionURN, + RevisionID: version.FunctionVersion.RevisionID, + CodeSize: version.FunctionVersion.Package.Size, + CodeSha256: version.FunctionVersion.Package.Signature, + BucketID: version.FunctionVersion.Package.BucketID, + ObjectID: version.FunctionVersion.Package.ObjectID, + Handler: version.FunctionVersion.Handler, + Layers: layerURNs, + CPU: version.FunctionVersion.CPU, + Memory: version.FunctionVersion.Memory, + Runtime: version.FunctionVersion.Runtime, + Timeout: version.FunctionVersion.Timeout, + VersionNumber: version.FunctionVersion.Version, + VersionDesc: version.FunctionVersion.Description, + Environment: env, + StatefulFlag: version.FunctionVersion.StatefulFlag, + LastModified: version.Function.UpdateTime, + Published: version.FunctionVersion.PublishTime, + MinInstance: version.FunctionVersion.MinInstance, + MaxInstance: version.FunctionVersion.MaxInstance, + ConcurrentNum: version.FunctionVersion.ConcurrentNum, + Status: version.FunctionVersion.Status, + InstanceNum: version.FunctionVersion.InstanceNum, + Device: version.FunctionVersion.Device, + } + if version.FunctionVersion.CustomResources != "" { + err := json.Unmarshal([]byte(version.FunctionVersion.CustomResources), &v.CustomResources) + if err != nil { + // ignore customResource unmarshal error + log.GetLogger().Warnf("failed to unmarshal customResources :%s", err.Error()) + } + } + return v +} + +func buildFunction(dateTime model.DateTime, t server.TenantInfo, + version storage.FunctionVersionValue, +) model.Function { + function := model.Function{ + DateTime: dateTime, + FunctionURN: utils.BuildFunctionURNWithTenant(t, version.Function.Name), + FunctionName: version.Function.Name, + TenantID: t.TenantID, + BusinessID: t.BusinessID, + ProductID: t.ProductID, + ReversedConcurrency: version.Function.ReversedConcurrency, + Description: version.Function.Description, + Tag: version.Function.Tag, + } + return function +} + +func buildTime(version storage.FunctionVersionValue) model.DateTime { + dateTime := model.DateTime{ + CreateTime: version.Function.CreateTime, + UpdateTime: version.Function.UpdateTime, + } + return dateTime +} + +// GetUploadFunctionCodeResponse is upload function package +func GetUploadFunctionCodeResponse(ctx server.Context, info model.FunctionQueryInfo, + req model.FunctionCodeUploadRequest, +) (model.FunctionVersion, error) { + name := utils.RemoveServiceID(info.FunctionName) + "-" + + strconv.FormatInt(timeutil.NowUnixMillisecond(), formatIntBase) + tempFile, err := getUploadFile(ctx) + if err != nil { + log.GetLogger().Errorf("failed to get package :%s", err.Error()) + return model.FunctionVersion{}, err + } + pkg, err := pkgstore.NewPackage(ctx, name, tempFile, req.FileSize) + if err != nil { + log.GetLogger().Errorf("failed to new package :%s", err.Error()) + return model.FunctionVersion{}, err + } + defer pkg.Close() + + // start transaction + txn := storage.GetTxnByKind(ctx, req.Kind) + defer txn.Cancel() + + fv, err := storage.GetFunctionVersion(txn, info.FunctionName, info.FunctionVersion) + if err != nil { + log.GetLogger().Errorf("failed to get function name by function name and version :%s", err.Error()) + return model.FunctionVersion{}, err + } + if fv.FunctionVersion.RevisionID != req.RevisionID { + return model.FunctionVersion{}, errmsg.New(errmsg.RevisionIDError) + } + + uploader, err := pkgstore.NewUploader(ctx, pkg) + if err != nil { + log.GetLogger().Errorf("failed to new uploader :%s", err.Error()) + return model.FunctionVersion{}, err + } + + uInfo := newUncontrolled(ctx, uploader.BucketID(), uploader.ObjectID()) + if err = storage.AddUncontrolled(ctx, uploader.BucketID(), uploader.ObjectID()); err != nil { + log.GetLogger().Errorf("failed to add uncontrolled :%s", err.Error()) + return model.FunctionVersion{}, err + } + var rollback bool + defer removeUncontrolledInfos(uInfo, rollback) + if err = upload(uploader); err != nil { + return model.FunctionVersion{}, err + } + oldBucketID, oldObjectID := updateFunctionPackageInfo(&fv, pkg, uploader) + resp, err := getUploadFunctionCodeHelper(txn, fv, oldBucketID, oldObjectID) + if err != nil { + return failUploadFunctionCodeHelper(ctx, uploader, rollback, err) + } + + return resp, nil +} + +func getUploadFile(ctx server.Context) (io.Reader, error) { + contentType := ctx.Gin().Request.Header.Get(common.HeaderDataContentType) + ctx.Gin().Request.Header.Set("Content-Type", contentType) + err := ctx.Gin().Request.ParseMultipartForm(common.MaxUploadMemorySize) + if err != nil { + return nil, err + } + if len(ctx.Gin().Request.MultipartForm.File["file"]) < 1 { + log.GetLogger().Errorf("failed to get MultipartForm file") + return nil, errors.New("multipartForm is empty") + } + tmpFile, err := ctx.Gin().Request.MultipartForm.File["file"][0].Open() + if err != nil { + return nil, err + } + return tmpFile, nil +} + +func failUploadFunctionCodeHelper(ctx server.Context, uploader pkgstore.Uploader, + rollback bool, failErr error, +) (model.FunctionVersion, error) { + log.GetLogger().Errorf("failed to get upload function code :%s", failErr.Error()) + if e := uploader.Rollback(); e == nil { + storage.RemoveUncontrolled(ctx, uploader.BucketID(), uploader.ObjectID()) + } else { + rollback = true + } + return model.FunctionVersion{}, failErr +} + +func removeUncontrolledInfos(v uncontrolledInfo, rollback bool) { + if !rollback { + err := storage.RemoveUncontrolled(v.ctx, v.bucketID, v.objectID) + if err != nil { + log.GetLogger().Warnf("failed to remove uncontrolled :%s", err.Error()) + } + } +} + +func newUncontrolled(ctx server.Context, bucketID string, objectID string) uncontrolledInfo { + return uncontrolledInfo{ + ctx: ctx, + bucketID: bucketID, + objectID: objectID, + } +} + +func updateFunctionPackageInfo(fv *storage.FunctionVersionValue, pkg pkgstore.Package, + uploader pkgstore.Uploader, +) (string, string) { + oldBucketID := fv.FunctionVersion.Package.BucketID + oldObjectID := fv.FunctionVersion.Package.ObjectID + fv.FunctionVersion.Package.Signature = pkg.Signature() + fv.FunctionVersion.Package.BucketID = uploader.BucketID() + fv.FunctionVersion.Package.ObjectID = uploader.ObjectID() + fv.FunctionVersion.Package.Size = pkg.Size() + return oldBucketID, oldObjectID +} + +func upload(uploader pkgstore.Uploader) error { + if err := uploader.Upload(); err != nil { + log.GetLogger().Errorf("failed to upload function package :%s", err.Error()) + return err + } + return nil +} + +func getUploadFunctionCodeHelper(txn storage.Transaction, fv storage.FunctionVersionValue, + oldBucketID, oldObjectID string, +) (model.FunctionVersion, error) { + err := storage.UpdateFunctionVersion(txn, fv) + if err != nil { + log.GetLogger().Errorf("failed to update function version :%s", err.Error()) + return model.FunctionVersion{}, err + } + + if err := publish.SavePublishFuncVersion(txn, fv); err != nil { + log.GetLogger().Errorf("failed to savePublishFuncVersion :%s", err.Error()) + return model.FunctionVersion{}, err + } + + // tries to delete old + if oldBucketID != "" && oldObjectID != "" { + ref, err := storage.IsObjectReferred(txn, oldBucketID, oldObjectID) + if err != nil { + log.GetLogger().Warnf("failed to check object referred :%s", err.Error()) + } + if err == nil && !ref { + err = pkgstore.Delete(oldBucketID, oldObjectID) + if err != nil { + log.GetLogger().Errorf("failed to delete function package ：%s", err.Error()) + } + } + } + if err := txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit upload info :%s") + return model.FunctionVersion{}, err + } + + tenantInfo, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenantInfo :%s", err.Error()) + return model.FunctionVersion{}, err + } + key := storage.FunctionVersionKey{ + TenantInfo: tenantInfo, + FunctionName: fv.Function.Name, + FunctionVersion: fv.FunctionVersion.Version, + } + return buildFunctionVersionModel(key, fv), nil +} + +func buildEnvironment(env *map[string]string, v storage.FunctionVersionValue) { + if len(v.FunctionVersion.Environment) != 0 { + value, err := utils.DecryptETCDValue(v.FunctionVersion.Environment) + if err != nil { + log.GetLogger().Warnf("failed to decrypt ETCD value :%s", err.Error()) + } + if value != nilMapValue { + err = json.Unmarshal([]byte(value), &env) + if err != nil { + log.GetLogger().Warnf("failed to unmarshal env when Convert2Model :%s", err.Error()) + } + } + } +} + +func buildFunctionVersionModel(key storage.FunctionVersionKey, + v storage.FunctionVersionValue, +) model.FunctionVersion { + env := make(map[string]string, common.DefaultMapSize) + buildEnvironment(&env, v) + funcVer := buildFunctionVersionEntity(key, v, env) + funcVer.FuncLayer = make([]model.FunctionLayer, len(v.FunctionLayer), len(v.FunctionLayer)) + for i, l := range v.FunctionLayer { + funcVer.FuncLayer[i].Name = l.Name + funcVer.FuncLayer[i].Version = l.Version + funcVer.FuncLayer[i].Order = l.Order + } + return funcVer +} + +func functionEntity(key storage.FunctionVersionKey, + v storage.FunctionVersionValue, +) (string, model.Function) { + var function model.Function + function = model.Function{ + FunctionName: v.Function.Name, + TenantID: key.TenantID, + BusinessID: key.BusinessID, + ProductID: key.ProductID, + ReversedConcurrency: v.Function.ReversedConcurrency, + Description: v.Function.Description, + Tag: v.Function.Tag, + FunctionURN: utils.BuildFunctionURN(key.BusinessID, key.TenantID, key.ProductID, v.Function.Name), + DateTime: model.DateTime{ + CreateTime: v.Function.CreateTime, + UpdateTime: v.Function.UpdateTime, + }, + } + functionVersionURN := utils.BuildFunctionVersionURN(key.BusinessID, key.TenantID, key.ProductID, + v.Function.Name, v.FunctionVersion.Version) + return functionVersionURN, function +} + +func buildFunctionVersionEntity(key storage.FunctionVersionKey, v storage.FunctionVersionValue, + env map[string]string, +) model.FunctionVersion { + functionVersionURN, function := functionEntity(key, v) + funcVer := model.FunctionVersion{ + Function: function, + CodeSize: v.FunctionVersion.Package.Size, + CodeSha256: v.FunctionVersion.Package.Signature, + RevisionID: v.FunctionVersion.RevisionID, + Handler: v.FunctionVersion.Handler, + CPU: v.FunctionVersion.CPU, + Memory: v.FunctionVersion.Memory, + Runtime: v.FunctionVersion.Runtime, + Timeout: v.FunctionVersion.Timeout, + VersionNumber: v.FunctionVersion.Version, + Environment: env, + BucketID: v.FunctionVersion.Package.BucketID, + ObjectID: v.FunctionVersion.Package.ObjectID, + VersionDesc: v.FunctionVersion.Description, + StatefulFlag: v.FunctionVersion.StatefulFlag, + Published: v.FunctionVersion.PublishTime, + MinInstance: v.FunctionVersion.MinInstance, + MaxInstance: v.FunctionVersion.MaxInstance, + ConcurrentNum: v.FunctionVersion.ConcurrentNum, + FunctionVersionURN: functionVersionURN, + Device: v.FunctionVersion.Device, + } + return funcVer +} diff --git a/functionsystem/apps/meta_service/function_repo/service/function_test.go b/functionsystem/apps/meta_service/function_repo/service/function_test.go new file mode 100644 index 0000000000000000000000000000000000000000..19342b4a8c2591de04dac46a0f892c8c92e88eb6 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/function_test.go @@ -0,0 +1,1134 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "os" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "meta_service/common/constants" + common "meta_service/common/constants" + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/storage/publish" + "meta_service/function_repo/test/fakecontext" + "meta_service/function_repo/utils" +) + +func TestCreateFunctionInfo(t *testing.T) { + type args struct { + ctx server.Context + req model.FunctionCreateRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test create success", args{fakecontext.NewMockContext(), model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 500, + Timeout: 500, + MinInstance: "10", + MaxInstance: "10", + ConcurrentNum: "10", + Handler: "main", + Kind: "faas", + Environment: nil, + SchedulePolicies: []model.ResourceAffinitySelector{ + {Group: "rg1"}, + }, + PoolID: "pool1", + }, + Name: "successFunc", + Runtime: "java1.8", + Tags: nil, + }}, model.FunctionVersion{}, false}, + {"test layer error", args{fakecontext.NewMockContext(), model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 500, + Timeout: 500, + MinInstance: "10", + MaxInstance: "10", + ConcurrentNum: "10", + Layers: []string{"aa", "bb"}, + Handler: "main", + Environment: nil, + }, + Name: "layererror", + Runtime: "java1.8", + Tags: nil, + }}, model.FunctionVersion{ + Status: constants.FunctionStatusAvailable, + }, true}, + {"test create name exist", args{fakecontext.NewMockContext(), model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 500, + Timeout: 500, + MinInstance: "10", + MaxInstance: "10", + ConcurrentNum: "10", + Handler: "main", + Environment: nil, + }, + Name: "successFunc", + Runtime: "java1.8", + Tags: nil, + }}, model.FunctionVersion{}, false}, + } + // test success + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + got, _ := CreateFunctionInfo(tt.args.ctx, tt.args.req, false) + if got.RevisionID == "" { + t.Logf("RevisionID %s", got.RevisionID) + t.Errorf("CreateFunctionInfo() got = %v", got) + } + if got.FunctionName != tt.args.req.Name { + t.Errorf("CreateFunctionInfo got function %s", got.FunctionName) + } + if got.VersionNumber == "" { + t.Errorf("CreateFunctionInfo got version empty %s", got.VersionNumber) + } + assert.Equal(t, got.Runtime, "java8") + }) + // test layer + tt = tests[1] + t.Run(tt.name, func(t *testing.T) { + _, err := CreateFunctionInfo(tt.args.ctx, tt.args.req, false) + t.Logf("err: %s", err) + assert.NotEqual(t, err, nil) + }) + // test name exist + tt = tests[2] + t.Run(tt.name, func(t *testing.T) { + _, err := CreateFunctionInfo(tt.args.ctx, tt.args.req, false) + _, err = CreateFunctionInfo(tt.args.ctx, tt.args.req, false) + t.Logf("err: %s", err) + assert.NotEqual(t, err, nil) + }) +} + +func TestUpdateFunctionInfo(t *testing.T) { + type args struct { + ctx server.Context + f model.FunctionQueryInfo + fv model.FunctionUpdateRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + { + "test update success", + args{ + fakecontext.NewMockContext(), + model.FunctionQueryInfo{ + FunctionName: "0-test-successFunc", + FunctionVersion: "$latest", + AliasName: "", + }, + model.FunctionUpdateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 100, + Timeout: 100, + Layers: nil, + MinInstance: "10", + MaxInstance: "10", + ConcurrentNum: "10", + Handler: "main", + Environment: nil, + Description: "test update success", + }, + RevisionID: "", + }}, + model.FunctionVersion{}, + false, + }, + } + // test success + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + mockCreateFunction(t) + // get RevisionID not same + _, err := UpdateFunctionInfo(tt.args.ctx, tt.args.f, tt.args.fv, false) + assert.NotEqual(t, err, nil) + // test success + getFunc := mockGetFunc(t, tt.args.ctx, tt.args.f.FunctionName) + tt.args.fv.RevisionID = getFunc.RevisionID + got, err := UpdateFunctionInfo(tt.args.ctx, tt.args.f, tt.args.fv, false) + if (err != nil) != tt.wantErr { + t.Errorf("UpdateFunctionInfo() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.NotEqual(t, got.RevisionID, nil) + // test update value + getFunc = mockGetFunc(t, tt.args.ctx, tt.args.f.FunctionName) + assert.NotEqual(t, createReq.Timeout, getFunc.Timeout) + assert.Equal(t, int64(100), getFunc.Timeout) + assert.Nil(t, err) + err = DeleteResponse(tt.args.ctx, "0-test-successFunc", utils.GetDefaultVersion(), "") + txn := storage.GetTxnByKind(tt.args.ctx, "") + defer txn.Cancel() + deleteAllFunctions(txn, "0-test-successFunc") + assert.Nil(t, err) + }) +} + +func mockCreateFunction(t *testing.T) { + _, err := CreateFunctionInfo(fakecontext.NewMockContext(), createReq, false) + if err != nil { + t.Errorf("CreateFunctionInfo err %v", err) + } +} + +func mockGetFunc(t *testing.T, ctx server.Context, name string) model.FunctionVersion { + rep, err := GetFunction(ctx, model.FunctionGetRequest{ + FunctionName: name, + FunctionVersion: "", + AliasName: "", + }) + if err != nil { + t.Errorf("MockGetFunc err %s", err) + } + return rep.FunctionVersion +} + +var createReq = model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 500, + Timeout: 500, + MinInstance: "10", + MaxInstance: "10", + ConcurrentNum: "10", + Handler: "main", + Environment: nil, + }, + Name: "0-test-successFunc", + Runtime: "java1.8", + Tags: nil, +} + +func TestGetFunctionVersionList(t *testing.T) { + type args struct { + ctx server.Context + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test get version success", args{fakecontext.NewMockContext()}, + model.FunctionVersion{}, false}, + } + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + _, err := GetFunctionVersionList(tt.args.ctx, "name", 3, 4) + assert.Equal(t, err, nil) + }) +} + +func TestGetFunctionList(t *testing.T) { + type args struct { + ctx server.Context + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test get func success", args{fakecontext.NewMockContext()}, + model.FunctionVersion{}, false}, + } + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + _, err := GetFunctionList(tt.args.ctx, "name", "", "", 4, 1024) + assert.Equal(t, err, nil) + }) +} + +func getZipBody(t *testing.T) *http.Request { + zip, err := base64.StdEncoding.DecodeString("UEsDBBQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEAAAAc3JjL1BLAQI/ABQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEACQAAAAAAAAAEAAAAAAAAABzcmMvCgAgAAAAAAABABgA/3JrC45F2AHLUrYMjkXYAf9yawuORdgBUEsFBgAAAAABAAEAVgAAACIAAAAAAA==") + assert.Nil(t, err) + return &http.Request{Body: ioutil.NopCloser(bytes.NewReader(zip))} +} + +func TestGetUploadFunctionCodeResponse(t *testing.T) { + ctx := fakecontext.NewMockContext() + getFunc := mockGetFunc(t, ctx, "0-test-successFunc") + patches := gomonkey.NewPatches() + patches.ApplyFunc(getUploadFile, func(_ server.Context) (io.Reader, error) { + zip, _ := base64.StdEncoding.DecodeString("UEsDBBQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEAAAAc3JjL1BLAQI/ABQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEACQAAAAAAAAAEAAAAAAAAABzcmMvCgAgAAAAAAABABgA/3JrC45F2AHLUrYMjkXYAf9yawuORdgBUEsFBgAAAAABAAEAVgAAACIAAAAAAA==") + return ioutil.NopCloser(bytes.NewReader(zip)), nil + }) + defer patches.Reset() + RevisionID := getFunc.RevisionID + _, err := GetUploadFunctionCodeResponse(ctx, model.FunctionQueryInfo{ + FunctionName: "0-test-successFunc", + FunctionVersion: "$latest", + AliasName: ""}, + model.FunctionCodeUploadRequest{RevisionID: RevisionID, + FileSize: 142}, + ) + assert.Nil(t, err) + err = DeleteResponse(ctx, "0-test-successFunc", utils.GetDefaultVersion(), "") + txn := storage.GetTxnByKind(ctx, "") + defer txn.Cancel() + deleteAllFunctions(txn, "0-test-successFunc") + assert.Nil(t, err) +} + +func TestDeleteResponse(t *testing.T) { + type args struct { + ctx server.Context + req model.FunctionCreateRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test delete success", args{fakecontext.NewMockContext(), model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 500, + Timeout: 500, + MinInstance: "10", + MaxInstance: "10", + ConcurrentNum: "10", + Handler: "main", + Environment: nil, + }, + Name: "deleteFunc", + Runtime: "java1.8", + Tags: nil, + }}, model.FunctionVersion{}, false}, + } + // del + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + got, _ := CreateFunctionInfo(tt.args.ctx, tt.args.req, false) + if got.RevisionID == "" { + t.Logf("RevisionID %s", got.RevisionID) + t.Errorf("CreateFunctionInfo() got = %v", got) + } + err := DeleteResponse(tt.args.ctx, "deleteFunc", utils.GetDefaultVersion(), "") + txn := storage.GetTxnByKind(tt.args.ctx, "") + defer txn.Cancel() + deleteAllFunctions(txn, "deleteFunc") + assert.Equal(t, err, nil) + }) +} + +func TestGetUploadFunctionCodeResponse2(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(getUploadFile, func(_ server.Context) (io.Reader, error) { + zip, _ := base64.StdEncoding.DecodeString("UEsDBBQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEAAAAc3JjL1BLAQI/ABQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEACQAAAAAAAAAEAAAAAAAAABzcmMvCgAgAAAAAAABABgA/3JrC45F2AHLUrYMjkXYAf9yawuORdgBUEsFBgAAAAABAAEAVgAAACIAAAAAAA==") + return ioutil.NopCloser(bytes.NewReader(zip)), nil + }) + defer patches.Reset() + ctx := fakecontext.NewMockContext() + getFunc := mockGetFunc(t, ctx, "0-test-successFunc") + zip, err := base64.StdEncoding.DecodeString("") + assert.Nil(t, err) + ctx.Gin().Request = &http.Request{Body: ioutil.NopCloser(bytes.NewReader(zip))} + RevisionID := getFunc.RevisionID + _, err = GetUploadFunctionCodeResponse(ctx, model.FunctionQueryInfo{ + FunctionName: "0-test-successFunc", + FunctionVersion: "$latest", + AliasName: ""}, + model.FunctionCodeUploadRequest{RevisionID: RevisionID, + FileSize: 0}, + ) + assert.NotNil(t, err) + assert.Equal(t, errmsg.ZipFileError, err.(snerror.SNError).Code()) + err = DeleteResponse(ctx, "0-test-successFunc", utils.GetDefaultVersion(), "") + txn := storage.GetTxnByKind(ctx, "") + defer txn.Cancel() + deleteAllFunctions(txn, "0-test-successFunc") + assert.Nil(t, err) +} + +func TestGetUploadFunctionCodeResponse3(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(getUploadFile, func(_ server.Context) (io.Reader, error) { + zip, _ := base64.StdEncoding.DecodeString("UEsDBBQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEAAAAc3JjL1BLAQI/ABQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEACQAAAAAAAAAEAAAAAAAAABzcmMvCgAgAAAAAAABABgA/3JrC45F2AHLUrYMjkXYAf9yawuORdgBUEsFBgAAAAABAAEAVgAAACIAAAAAAA==") + return ioutil.NopCloser(bytes.NewReader(zip)), nil + }) + defer patches.Reset() + ctx := fakecontext.NewMockContext() + + ctx.Gin().Request = getZipBody(t) + getFunc := mockGetFunc(t, ctx, "0-test-successFunc") + RevisionID := getFunc.RevisionID + _, err := GetUploadFunctionCodeResponse(ctx, model.FunctionQueryInfo{ + FunctionName: "0-test-successFunc", + FunctionVersion: "$latest", + AliasName: ""}, + model.FunctionCodeUploadRequest{RevisionID: RevisionID, + FileSize: 142}, + ) + assert.Nil(t, err) +} + +func TestGetUploadFunctionCodeResponse4(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(getUploadFile, func(_ server.Context) (io.Reader, error) { + zip, _ := base64.StdEncoding.DecodeString("UEsDBBQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEAAAAc3JjL1BLAQI/ABQAAAAAAEpwgVQAAAAAAAAAAAAAAAAEACQAAAAAAAAAEAAAAAAAAABzcmMvCgAgAAAAAAABABgA/3JrC45F2AHLUrYMjkXYAf9yawuORdgBUEsFBgAAAAABAAEAVgAAACIAAAAAAA==") + return ioutil.NopCloser(bytes.NewReader(zip)), nil + }) + defer patches.Reset() + ctx := fakecontext.NewMockContext() + ctx.Gin().Request = getZipBody(t) + _, err := GetUploadFunctionCodeResponse(ctx, model.FunctionQueryInfo{ + FunctionName: "0-test-successFunc", + FunctionVersion: "$latest", + AliasName: ""}, + model.FunctionCodeUploadRequest{RevisionID: "RevisionID", + FileSize: 142}, + ) + assert.NotNil(t, err) + assert.Equal(t, errmsg.RevisionIDError, err.(snerror.SNError).Code()) + err = DeleteResponse(ctx, "0-test-successFunc", utils.GetDefaultVersion(), "") + txn := storage.GetTxnByKind(ctx, "") + defer txn.Cancel() + deleteAllFunctions(txn, "0-test-successFunc") + assert.Nil(t, err) +} + +type mockUploaderBuilder struct{} + +// NewUploader implements UploaderBuilder +func (mockUploaderBuilder) NewUploader(c server.Context, pkg pkgstore.Package) (pkgstore.Uploader, error) { + return mockUploader{}, nil +} + +type mockUploader struct { + mockRollbackErr error +} + +// BucketID implements Uploader +func (mockUploader) BucketID() string { + return "" +} + +// ObjectID implements Uploader +func (mockUploader) ObjectID() string { + return "" +} + +// Upload implements Uploader +func (mockUploader) Upload() error { + return nil +} + +// Rollback implements Uploader +func (mu mockUploader) Rollback() error { + return mu.mockRollbackErr +} + +func TestFailUploadFunctionCodeHelper(t *testing.T) { + Convey("test failUploadFunctionCodeHelper", t, func() { + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyFunc(storage.RemoveUncontrolled, func(ctx server.Context, bucketID, objectID string) error { + return nil + }) + defer patch.Reset() + noopUploader := mockUploader{} + noopUploader.mockRollbackErr = nil + _, err := failUploadFunctionCodeHelper(ctx, noopUploader, false, errors.New("mock err")) + So(err, ShouldNotBeNil) + noopUploader.mockRollbackErr = errors.New("mock err") + _, err = failUploadFunctionCodeHelper(ctx, noopUploader, false, errors.New("mock err")) + So(err, ShouldNotBeNil) + }) +} + +func TestGetUploadFile(t *testing.T) { + Convey("Test getUploadFile with ParseMultipartForm err", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&http.Request{}), "ParseMultipartForm", + func(r *http.Request, maxMemory int64) error { + return errors.New("mock err") + }) + defer patch.Reset() + ctx := fakecontext.NewMockContext() + ctx.Gin().Request = &http.Request{} + ctx.Gin().Request.Header = make(map[string][]string) + ctx.Gin().Request.Header.Set(common.HeaderDataContentType, "mock-data") + _, err := getUploadFile(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("Test getUploadFile with multipartForm is empty err", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&http.Request{}), "ParseMultipartForm", + func(r *http.Request, maxMemory int64) error { + return nil + }) + defer patch.Reset() + ctx := fakecontext.NewMockContext() + ctx.Gin().Request = &http.Request{} + ctx.Gin().Request.Header = make(map[string][]string) + ctx.Gin().Request.Header.Set(common.HeaderDataContentType, "mock-data") + ctx.Gin().Request.MultipartForm = &multipart.Form{File: map[string][]*multipart.FileHeader{ + "file": []*multipart.FileHeader{}, + }} + _, err := getUploadFile(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("Test getUploadFile with Open err", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&http.Request{}), "ParseMultipartForm", + func(r *http.Request, maxMemory int64) error { + return nil + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, errors.New("mock err") + }) + defer patch.Reset() + ctx := fakecontext.NewMockContext() + ctx.Gin().Request = &http.Request{} + ctx.Gin().Request.Header = make(map[string][]string) + ctx.Gin().Request.Header.Set(common.HeaderDataContentType, "mock-data") + ctx.Gin().Request.MultipartForm = &multipart.Form{File: map[string][]*multipart.FileHeader{ + "file": []*multipart.FileHeader{{}}, + }} + _, err := getUploadFile(ctx) + So(err, ShouldNotBeNil) + }) + Convey("Test getUploadFile", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&http.Request{}), "ParseMultipartForm", + func(r *http.Request, maxMemory int64) error { + return nil + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, errors.New("mock err") + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, nil + }) + defer patch.Reset() + ctx := fakecontext.NewMockContext() + ctx.Gin().Request = &http.Request{} + ctx.Gin().Request.Header = make(map[string][]string) + ctx.Gin().Request.Header.Set(common.HeaderDataContentType, "mock-data") + ctx.Gin().Request.MultipartForm = &multipart.Form{File: map[string][]*multipart.FileHeader{ + "file": []*multipart.FileHeader{{}}, + }} + _, err := getUploadFile(ctx) + So(err, ShouldBeNil) + }) +} + +func TestBuildEnvironment(t *testing.T) { + Convey("Test buildEnvironment", t, func() { + v := storage.FunctionVersionValue{ + Function: storage.Function{}, + FunctionVersion: storage.FunctionVersion{ + Environment: "mock env", + }, + FunctionLayer: nil, + } + gomonkey.ApplyFunc(utils.DecryptETCDValue, func(text string) (string, error) { + return text, errors.New("mock err") + }) + buildEnvironment(nil, v) + }) +} + +func TestUpdateFunctionLayer(t *testing.T) { + Convey("Test updateFunctionLayer 1", t, func() { + request := model.FunctionUpdateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + Layers: []string{"sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:1"}, + }, + } + fv := &storage.FunctionVersionValue{FunctionLayer: make([]storage.FunctionLayer, 0)} + err := updateFunctionLayer(request, fv) + So(err, ShouldBeNil) + }) + Convey("Test updateFunctionLayer 2", t, func() { + request := model.FunctionUpdateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + Layers: []string{"sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:a"}, + }, + } + fv := &storage.FunctionVersionValue{FunctionLayer: make([]storage.FunctionLayer, 0)} + err := updateFunctionLayer(request, fv) + So(err, ShouldNotBeNil) + }) +} + +func TestBuildFunctionVersion(t *testing.T) { + Convey("Test buildFunctionVersion with err", t, func() { + request := model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + Layers: []string{ + "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:1", + "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:x", + }, + }, + } + _, err := buildFunctionVersion(request) + So(err, ShouldNotBeNil) + }) + Convey("Test buildFunctionVersion with EncodeEnv err", t, func() { + patch := gomonkey.ApplyFunc(utils.EncodeEnv, func(m map[string]string) (string, error) { + return "", errors.New("mock err") + }) + defer patch.Reset() + request := model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + Layers: []string{ + "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:1", + }, + Environment: map[string]string{}, + }, + } + _, err := buildFunctionVersion(request) + So(err, ShouldNotBeNil) + }) + Convey("Test buildFunctionVersion with Marshal err", t, func() { + patch := gomonkey.ApplyFunc(utils.EncodeEnv, func(m map[string]string) (string, error) { + return "", nil + }).ApplyFunc(json.Marshal, func(v interface{}) ([]byte, error) { + return nil, errors.New("mock err") + }) + defer patch.Reset() + request := model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + Layers: []string{ + "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:1", + }, + Environment: map[string]string{}, + CustomResources: map[string]float64{}, + }, + } + _, err := buildFunctionVersion(request) + So(err, ShouldNotBeNil) + }) + Convey("Test buildFunctionVersion with functionVersionResult err", t, func() { + patch := gomonkey.ApplyFunc(utils.EncodeEnv, func(m map[string]string) (string, error) { + return "", nil + }) + defer patch.Reset() + request := model.FunctionCreateRequest{ + FunctionBasicInfo: model.FunctionBasicInfo{ + Layers: []string{ + "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:1", + }, + Environment: map[string]string{}, + CustomResources: map[string]float64{}, + }, + } + _, err := buildFunctionVersion(request) + So(err, ShouldNotBeNil) + }) + Convey("Test buildFunctionVersion success with hook handler", t, func() { + request := model.FunctionCreateRequest{ + Name: "0@base@hello", + Runtime: "java1.8", + Kind: "faas", + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 500, + Timeout: 30, + ConcurrentNum: "100", + Handler: "test.handler", + MinInstance: "0", + MaxInstance: "100", + StorageType: "local", + CodePath: "/home", + }, + } + fv, err := buildFunctionVersion(request) + So(err, ShouldBeNil) + So(len(fv.FunctionVersion.HookHandler) > 0, ShouldBeTrue) + So(fv.FunctionVersion.Runtime, ShouldEqual, "java8") + request = model.FunctionCreateRequest{ + Name: "0-base-hello", + Runtime: "java8", + Kind: "yrlib", + FunctionBasicInfo: model.FunctionBasicInfo{ + CPU: 500, + Memory: 500, + Timeout: 30, + ConcurrentNum: "100", + Handler: "test.handler", + MinInstance: "0", + MaxInstance: "100", + StorageType: "local", + CodePath: "/home", + }, + } + fv, err = buildFunctionVersion(request) + So(err, ShouldBeNil) + So(fv.FunctionVersion.Runtime, ShouldEqual, "java1.8") + So(len(fv.FunctionVersion.HookHandler) > 0, ShouldBeTrue) + }) +} + +func TestGetUploadFunctionCodeHelper(t *testing.T) { + Convey("Test getUploadFunctionCodeHelper", t, func() { + txn := &storage.Txn{} + fv := storage.FunctionVersionValue{} + patches := gomonkey.ApplyFunc(storage.UpdateFunctionVersion, + func(storage.Transaction, storage.FunctionVersionValue) error { + return errors.New("mock err") + }) + _, err := getUploadFunctionCodeHelper(txn, fv, "mock-bucket-id", "mock=obj-id") + So(err, ShouldNotBeNil) + patches.Reset() + + patches = gomonkey.ApplyFunc(storage.UpdateFunctionVersion, + func(storage.Transaction, storage.FunctionVersionValue) error { + return nil + }).ApplyFunc(publish.SavePublishFuncVersion, func(txn storage.Transaction, + funcVersionValue storage.FunctionVersionValue) error { + return errors.New("mock err") + }) + _, err = getUploadFunctionCodeHelper(txn, fv, "mock-bucket-id", "mock=obj-id") + So(err, ShouldNotBeNil) + patches.Reset() + + patches = gomonkey.ApplyFunc(storage.UpdateFunctionVersion, + func(storage.Transaction, storage.FunctionVersionValue) error { + return nil + }).ApplyFunc(publish.SavePublishFuncVersion, func(txn storage.Transaction, + funcVersionValue storage.FunctionVersionValue) error { + return nil + }) + defer patches.Reset() + + Convey("when IsObjectReferred err and Commit err", func() { + patch2 := gomonkey.ApplyFunc(storage.IsObjectReferred, func(storage.Transaction, string, string) (bool, error) { + return false, errors.New("mock err") + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return errors.New("mock commit err") + }) + defer patch2.Reset() + _, err = getUploadFunctionCodeHelper(txn, fv, "mock-bucket-id", "mock=obj-id") + So(err, ShouldNotBeNil) + }) + + Convey("when pkgstore.Delete err and Commit err", func() { + patch2 := gomonkey.ApplyFunc(storage.IsObjectReferred, func(storage.Transaction, string, string) (bool, error) { + return false, nil + }).ApplyFunc(pkgstore.Delete, func(bucketID, objectID string) error { + return errors.New("mock err") + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return errors.New("mock commit err") + }) + defer patch2.Reset() + _, err = getUploadFunctionCodeHelper(txn, fv, "mock-bucket-id", "mock=obj-id") + So(err, ShouldNotBeNil) + }) + + Convey("when TenantInfo err", func() { + patch2 := gomonkey.ApplyFunc(storage.IsObjectReferred, func(storage.Transaction, string, string) (bool, error) { + return false, nil + }).ApplyFunc(pkgstore.Delete, func(bucketID, objectID string) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn), "GetCtx", func(tx *storage.Txn) server.Context { + return fakecontext.NewContext() + }) + defer patch2.Reset() + _, err = getUploadFunctionCodeHelper(txn, fv, "mock-bucket-id", "mock=obj-id") + So(err, ShouldNotBeNil) + }) + + }) +} + +func TestDeleteFunctionByVersion(t *testing.T) { + Convey("Test deleteFunctionByVersion", t, func() { + txn := &storage.Txn{} + Convey("with GetAliasNumByFunctionName err", func() { + patches := gomonkey.ApplyFunc(storage.GetAliasNumByFunctionName, func(storage.Transaction, string) (int, error) { + return 0, errors.New("mock err") + }) + defer patches.Reset() + err := deleteFunctionByVersion(txn, "mock-func", "mock-ver") + So(err, ShouldNotBeNil) + }) + Convey("with AliasRoutingExist err", func() { + patches := gomonkey.ApplyFunc(storage.GetAliasNumByFunctionName, func(storage.Transaction, string) (int, error) { + return 0, nil + }).ApplyFunc(storage.AliasRoutingExist, func(txn storage.Transaction, funcName, funcVer string) (bool, error) { + return true, errors.New("mock err") + }) + defer patches.Reset() + err := deleteFunctionByVersion(txn, "mock-func", "mock-ver") + So(err, ShouldNotBeNil) + }) + Convey("with AliasRoutingExist", func() { + patches := gomonkey.ApplyFunc(storage.GetAliasNumByFunctionName, func(storage.Transaction, string) (int, error) { + return 0, nil + }).ApplyFunc(storage.AliasRoutingExist, func(txn storage.Transaction, funcName, funcVer string) (bool, error) { + return true, nil + }) + defer patches.Reset() + err := deleteFunctionByVersion(txn, "mock-func", "mock-ver") + So(err, ShouldNotBeNil) + }) + patches := gomonkey.ApplyFunc(storage.GetAliasNumByFunctionName, func(storage.Transaction, string) (int, error) { + return 0, nil + }).ApplyFunc(storage.AliasRoutingExist, func(txn storage.Transaction, funcName, funcVer string) (bool, error) { + return false, nil + }).ApplyFunc(storage.DeleteFunctionVersion, func(txn storage.Transaction, funcName string, funcVer string) error { + return nil + }) + defer patches.Reset() + Convey("with DeleteFunctionStatus err", func() { + patch2 := gomonkey.ApplyFunc(storage.DeleteFunctionStatus, func(storage.Transaction, string, string) error { + return errors.New("mock err") + }) + defer patch2.Reset() + err := deleteFunctionByVersion(txn, "mock-func", "mock-ver") + So(err, ShouldNotBeNil) + }) + Convey("with DeleteTriggerByFuncNameVersion err", func() { + patch2 := gomonkey.ApplyFunc(storage.DeleteFunctionStatus, func(storage.Transaction, string, string) error { + return nil + }).ApplyFunc(DeleteTriggerByFuncNameVersion, func(storage.Transaction, string, string) error { + return errors.New("mock err") + }) + defer patch2.Reset() + err := deleteFunctionByVersion(txn, "mock-func", "mock-ver") + So(err, ShouldNotBeNil) + }) + Convey("with ok", func() { + patch2 := gomonkey.ApplyFunc(storage.DeleteFunctionStatus, func(storage.Transaction, string, string) error { + return nil + }).ApplyFunc(DeleteTriggerByFuncNameVersion, func(storage.Transaction, string, string) error { + return nil + }) + defer patch2.Reset() + err := deleteFunctionByVersion(txn, "mock-func", "mock-ver") + So(err, ShouldBeNil) + }) + }) +} + +func TestDeleteResponseForMeta(t *testing.T) { + Convey("Test deleteResponseForMeta etcd err", t, func() { + ctx := fakecontext.NewMockContext() + txn := storage.GetTxnByKind(ctx, "") + patches := gomonkey.ApplyFunc(storage.GetFunctionVersions, func(txn storage.Transaction, + funcName string) ([]storage.FunctionVersionValue, error) { + return []storage.FunctionVersionValue{{ + FunctionVersion: storage.FunctionVersion{Version: "v1", Environment: ""}, + Function: storage.Function{Name: "0-test-hello"}}}, nil + }).ApplyFunc(storage.GetAliasNumByFunctionName, + func(txn storage.Transaction, funcName string) (int, error) { + return 0, nil + }).ApplyFunc(storage.AliasRoutingExist, func(txn storage.Transaction, funcName, funcVer string) (bool, error) { + return false, nil + }).ApplyFunc(storage.DeleteFunctionVersion, func(txn storage.Transaction, funcName string, + funcVer string) error { + return nil + }).ApplyFunc(storage.DeleteFunctionStatus, func(txn storage.Transaction, funcName string, funcVer string) error { + return nil + }).ApplyFunc(publish.DeletePublishFunction, func(txn storage.Transaction, name string, + tenantInfo server.TenantInfo, funcVersion string) { + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return snerror.New(10000, "etcd transation failed") + }) + defer patches.Reset() + _, err := DeleteResponseForMeta(ctx, server.TenantInfo{TenantID: "123456"}, "0-test-hello", "v1", "") + So(err, ShouldNotBeNil) + }) +} + +func TestCheckFunctionLayerList(t *testing.T) { + Convey("Test checkFunctionLayerList", t, func() { + txn := &storage.Txn{} + urns := []string{"a"} + Convey("when ParseLayerInfo err", func() { + patches := gomonkey.ApplyFunc(ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, errors.New("mock err") + }) + defer patches.Reset() + err := checkFunctionLayerList(txn, urns, "go") + So(err, ShouldNotBeNil) + }) + Convey("when found", func() { + patches := gomonkey.ApplyFunc(ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, nil + }).ApplyFunc(storage.GetLayerVersionTx, func( + txn storage.Transaction, layerName string, layerVersion int) (storage.LayerValue, error) { + return storage.LayerValue{CompatibleRuntimes: []string{"go"}}, nil + }) + defer patches.Reset() + err := checkFunctionLayerList(txn, urns, "go") + So(err, ShouldBeNil) + }) + Convey("when not found", func() { + patches := gomonkey.ApplyFunc(ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, nil + }).ApplyFunc(storage.GetLayerVersionTx, func( + txn storage.Transaction, layerName string, layerVersion int) (storage.LayerValue, error) { + return storage.LayerValue{CompatibleRuntimes: []string{"gg"}}, nil + }) + defer patches.Reset() + err := checkFunctionLayerList(txn, urns, "go") + So(err, ShouldNotBeNil) + }) + Convey("when repetitive", func() { + patches := gomonkey.ApplyFunc(ParseLayerInfo, func(ctx server.Context, s string) (model.LayerQueryInfo, error) { + return model.LayerQueryInfo{}, nil + }).ApplyFunc(storage.GetLayerVersionTx, func( + txn storage.Transaction, layerName string, layerVersion int) (storage.LayerValue, error) { + return storage.LayerValue{CompatibleRuntimes: []string{"go"}}, nil + }) + defer patches.Reset() + err := checkFunctionLayerList(txn, []string{"a", "a"}, "go") + So(err, ShouldNotBeNil) + }) + }) +} + +func TestCanDelFunctionPackage(t *testing.T) { + Convey("Test canDelFunctionPackage", t, func() { + txn := &storage.Txn{} + Convey("when IsObjectReferred err", func() { + fvo := storage.FunctionVersionValue{} + fvo.FunctionVersion.Version = "a" + fvs := []storage.FunctionVersionValue{fvo} + patches := gomonkey.ApplyFunc(storage.IsObjectReferred, + func(txn storage.Transaction, bucketID string, objectID string) (bool, error) { + return false, errors.New("mock err") + }) + defer patches.Reset() + err := canDelFunctionPackage(txn, "a", fvs) + So(err, ShouldNotBeNil) + }) + Convey("when referred", func() { + fvo := storage.FunctionVersionValue{} + fvo.FunctionVersion.Version = "a" + fvs := []storage.FunctionVersionValue{fvo} + patches := gomonkey.ApplyFunc(storage.IsObjectReferred, + func(txn storage.Transaction, bucketID string, objectID string) (bool, error) { + return true, nil + }) + defer patches.Reset() + err := canDelFunctionPackage(txn, "a", fvs) + So(err, ShouldBeNil) + }) + Convey("when AddUncontrolledTx err", func() { + fvo := storage.FunctionVersionValue{} + fvo.FunctionVersion.Version = "a" + fvs := []storage.FunctionVersionValue{fvo} + patches := gomonkey.ApplyFunc(storage.IsObjectReferred, + func(txn storage.Transaction, bucketID string, objectID string) (bool, error) { + return false, nil + }).ApplyFunc(storage.AddUncontrolledTx, func(txn storage.Transaction, bucketID, objectID string) error { + return errors.New("mock err") + }) + defer patches.Reset() + err := canDelFunctionPackage(txn, "a", fvs) + So(err, ShouldNotBeNil) + }) + }) +} + +func Test_getFunctionByAlias(t *testing.T) { + Convey("Test getFunctionByAlias", t, func() { + patches := gomonkey.ApplyFunc(storage.GetFunctionByFunctionNameAndAlias, func( + c server.Context, funcName string, aliasName string) string { + return "1" + }).ApplyFunc(getFunctionByVersion, func(c server.Context, f string, v string, _ string) (model.FunctionGetResponse, error) { + return model.FunctionGetResponse{}, nil + }) + defer patches.Reset() + _, err := getFunctionByAlias(fakecontext.NewMockContext(), "f", "a", "") + So(err, ShouldBeNil) + }) +} + +func TestDeleteTriggerByFuncName(t *testing.T) { + Convey("Test DeleteTriggerByFuncName", t, func() { + patches := gomonkey.ApplyFunc(publish.DeleteTriggerByFuncName, func(txn storage.Transaction, funcName string) error { + return errors.New("mock err") + }) + defer patches.Reset() + err := DeleteTriggerByFuncName(&storage.Txn{}, "mock-fun") + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteTriggerByFuncName 2", t, func() { + patches := gomonkey.ApplyFunc(publish.DeleteTriggerByFuncName, func(txn storage.Transaction, funcName string) error { + return nil + }).ApplyFunc(storage.DeleteTriggerByFunctionNameVersion, func(txn storage.Transaction, funcName, verOrAlias string) error { + return errors.New("mock err") + }) + defer patches.Reset() + err := DeleteTriggerByFuncName(&storage.Txn{}, "mock-fun") + So(err, ShouldNotBeNil) + }) +} + +func Test_buildUpdateFunctionVersion(t *testing.T) { + Convey("Test buildUpdateFunctionVersion", t, func() { + Convey("with json Marshal err", func() { + patch := gomonkey.ApplyFunc(json.Marshal, func(v interface{}) ([]byte, error) { + return nil, errors.New("mock err") + }) + defer patch.Reset() + req := model.FunctionUpdateRequest{} + fv := &storage.FunctionVersionValue{} + req.CustomResources = map[string]float64{"a": 0.1} + err := buildUpdateFunctionVersion(req, fv) + So(err, ShouldNotBeNil) + }) + Convey("with utils.EncodeEnv err", func() { + patch := gomonkey.ApplyFunc(utils.EncodeEnv, func(m map[string]string) (string, error) { + return "", errors.New("mock err") + }) + defer patch.Reset() + req := model.FunctionUpdateRequest{} + fv := &storage.FunctionVersionValue{} + req.CustomResources = map[string]float64{"a": 0.1} + req.Environment = map[string]string{"env1": "a"} + err := buildUpdateFunctionVersion(req, fv) + So(err, ShouldNotBeNil) + }) + Convey("with Atoi ConcurrentNum err", func() { + patch := gomonkey.ApplyFunc(utils.EncodeEnv, func(m map[string]string) (string, error) { + return "", nil + }) + defer patch.Reset() + req := model.FunctionUpdateRequest{} + fv := &storage.FunctionVersionValue{} + req.ConcurrentNum = "abc" + req.Environment = map[string]string{"env1": "a"} + err := buildUpdateFunctionVersion(req, fv) + So(err, ShouldNotBeNil) + }) + Convey("with strconv.ParseInt err", func() { + req := model.FunctionUpdateRequest{} + fv := &storage.FunctionVersionValue{} + req.MinInstance = "abc" + err := buildUpdateFunctionVersion(req, fv) + So(err, ShouldNotBeNil) + }) + Convey("with strconv.ParseInt err 2", func() { + req := model.FunctionUpdateRequest{} + fv := &storage.FunctionVersionValue{} + req.MaxInstance = "abc" + err := buildUpdateFunctionVersion(req, fv) + So(err, ShouldNotBeNil) + }) + Convey("with updateFunctionLayer err", func() { + patch := gomonkey.ApplyFunc(updateFunctionLayer, func(request model.FunctionUpdateRequest, fv *storage.FunctionVersionValue) error { + return errors.New("mock err") + }) + defer patch.Reset() + req := model.FunctionUpdateRequest{} + fv := &storage.FunctionVersionValue{} + err := buildUpdateFunctionVersion(req, fv) + So(err, ShouldNotBeNil) + }) + }) +} + +func Test_deleteAllFunctions(t *testing.T) { + Convey("Test deleteAllFunctions", t, func() { + mode := 0 + patches := gomonkey.ApplyFunc(storage.DeleteFunctionVersions, func(txn storage.Transaction, funcName string) error { + if mode == 0 { + return errors.New("mock err") + } + return nil + }).ApplyFunc(storage.DeleteFunctionStatuses, func(txn storage.Transaction, funcName string) error { + if mode == 1 { + return errors.New("mock err") + } + return nil + }).ApplyFunc(storage.DeleteAliasByFunctionName, func(txn storage.Transaction, funcName string) error { + if mode == 2 { + return errors.New("mock err") + } + return nil + }).ApplyFunc(DeleteTriggerByFuncName, func(txn storage.Transaction, funcName string) error { + if mode == 3 { + return errors.New("mock err") + } + return nil + }) + defer patches.Reset() + for mode = 0; mode < 4; mode++ { + err := deleteAllFunctions(&storage.Txn{}, "mock-func-name") + So(err, ShouldNotBeNil) + } + }) +} + +func Test_deletePackage(t *testing.T) { + Convey("Test deletePackage", t, func() { + txn := &storage.Txn{} + patches := gomonkey.NewPatches() + defer patches.Reset() + Convey("when txn.Commit err", func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := deletePackage(txn, fakecontext.NewMockContext(), nil) + So(err, ShouldNotBeNil) + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return nil + }) + Convey("when deleteFunctionPackage err", func() { + patch := gomonkey.ApplyFunc(deleteFunctionPackage, func(ctx server.Context, fvs []storage.FunctionVersionValue) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := deletePackage(txn, fakecontext.NewMockContext(), nil) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/layer.go b/functionsystem/apps/meta_service/function_repo/service/layer.go new file mode 100644 index 0000000000000000000000000000000000000000..2a40f8ae9c1ad0ab2eb4aff1b11258b829eba532 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/layer.go @@ -0,0 +1,539 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "strconv" + "strings" + "time" + + common "meta_service/common/constants" + "meta_service/common/logger/log" + "meta_service/common/snerror" + "meta_service/common/timeutil" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/utils" +) + +const ( + maxLayerSize = 1000 + maxLayerVersionSize = 10000 + maxLayerVersionNum = 1000000 + compatibleRuntimeMinSize = 1 + compatibleRuntimeMaxSize = 5 + defaultBase = 10 +) + +func buildLayer( + ctx server.Context, mlayer *model.Layer, layerName string, layerVersion int, layer storage.LayerValue, +) error { + mlayer.Name = layerName + mlayer.Version = layerVersion + + info, err := ctx.TenantInfo() + if err != nil { + return err + } + mlayer.LayerURN = utils.BuildLayerURN(info.BusinessID, info.TenantID, info.ProductID, layerName) + mlayer.LayerVersionURN = utils.BuildLayerVersionURN( + info.BusinessID, info.TenantID, info.ProductID, layerName, strconv.Itoa(layerVersion)) + + mlayer.LayerSize = layer.Package.Size + mlayer.LayerSHA256 = layer.Package.Signature + mlayer.CreateTime = layer.CreateTime + mlayer.UpdateTime = layer.UpdateTime + mlayer.CompatibleRuntimes = layer.CompatibleRuntimes + mlayer.Description = layer.Description + mlayer.LicenseInfo = layer.LicenseInfo + return nil +} + +func isCompatibleRuntimeValid(cps []string) error { + seen := make(map[string]interface{}, len(cps)) + + for _, cp := range cps { + var found bool + for _, vcp := range config.RepoCfg.CompatibleRuntimeType { + if cp == vcp { + found = true + break + } + } + if !found { + log.GetLogger().Errorf("compatible runtime %v is invalid", cps) + return errmsg.New(errmsg.CompatibleRuntimeError) + } + + if _, ok := seen[cp]; ok { + log.GetLogger().Errorf("duplicate compatible runtimes %v", cps) + return errmsg.New(errmsg.DuplicateCompatibleRuntimes) + } + seen[cp] = struct{}{} + } + return nil +} + +// ParseLayerInfo parses input layer URN or layer name. +func ParseLayerInfo(ctx server.Context, s string) (model.LayerQueryInfo, error) { + info, valid := utils.ParseLayerVerInfoFromURN(s) + if !valid { + // s is just the layerName + return model.LayerQueryInfo{LayerName: s}, nil + } + tenant, err := ctx.TenantInfo() + if err != nil { + return model.LayerQueryInfo{}, err + } + if info.BusinessID != tenant.BusinessID || info.TenantInfo != tenant.TenantID { + log.GetLogger().Errorf( + "query tenant %s, %s is not the same as header tenant %s, %s", + info.BusinessID, info.TenantInfo, tenant.BusinessID, tenant.TenantID) + return model.LayerQueryInfo{}, snerror.New(errmsg.InvalidFunctionLayer, "urn "+s+" is invalid") + } + return model.LayerQueryInfo{LayerName: info.LayerName, LayerVersion: info.LayerVersion}, nil +} + +func getLayerNextVersion(txn *storage.Txn, layerName string) (int, error) { + size, version, err := storage.GetLayerSizeAndLatestVersion(txn, layerName) + if err != nil { + if err != errmsg.KeyNotFoundError { + log.GetLogger().Errorf("failed to get layer latest version: %s", err.Error()) + return 0, err + } + n, err := storage.CountLayerTx(txn) + if err != nil { + log.GetLogger().Errorf("failed to count layer: %s", err.Error()) + return 0, err + } + if n >= maxLayerSize { + log.GetLogger().Errorf("max layer size has reached") + return 0, errmsg.New(errmsg.TenantLayerSizeOutOfLimit, n, maxLayerSize) + } + return storage.NewLayerVersion, nil + } + if size > maxLayerVersionSize { + log.GetLogger().Errorf("max layer version size has reached") + return 0, errmsg.New(errmsg.LayerVersionSizeOutOfLimit, size, maxLayerVersionSize) + } + if version >= maxLayerVersionNum { + log.GetLogger().Errorf("max layer version num has reached") + return 0, errmsg.New(errmsg.LayerVersionNumOutOfLimit, maxLayerVersionNum) + } + version++ + return version, nil +} + +func createPackage(ctx server.Context, layerName, base64Content string) (pkgstore.Package, error) { + name := layerName + "-" + strconv.FormatInt(timeutil.NowUnixMillisecond(), defaultBase) + b, err := base64.StdEncoding.DecodeString(base64Content) + if err != nil { + log.GetLogger().Errorf("failed to decode req.ZipFile, %s", err.Error()) + return nil, snerror.New(errmsg.SaveFileError, "failed to decode req.ZipFile") + } + pkg, err := pkgstore.NewPackage(ctx, name, bytes.NewReader(b), int64(len(b))) + if err != nil { + return nil, err + } + return pkg, nil +} + +// CreateLayer handles the request of creating a layer +func CreateLayer(ctx server.Context, layerName string, req model.CreateLayerRequest) (model.Layer, error) { + compatibleRuntimes, err := getCompatibleRuntimes(req.CompatibleRuntimes) + if err != nil { + return model.Layer{}, err + } + if err := isCompatibleRuntimeValid(compatibleRuntimes); err != nil { + return model.Layer{}, err + } + + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + version, err := getLayerNextVersion(txn, layerName) + if err != nil { + return model.Layer{}, err + } + + tempFile, err := getUploadFile(ctx) + if err != nil { + log.GetLogger().Errorf("failed to get upload file, err: %s", err.Error()) + return model.Layer{}, err + } + name := layerName + "-" + strconv.FormatInt(timeutil.NowUnixMillisecond(), 10) + pkg, err := pkgstore.NewPackage(ctx, name, tempFile, req.ZipFileSize) + if err != nil { + return model.Layer{}, err + } + defer pkg.Close() + + uploader, err := pkgstore.NewUploader(ctx, pkg) + if err != nil { + return model.Layer{}, err + } + + if err := storage.AddUncontrolled(ctx, uploader.BucketID(), uploader.ObjectID()); err != nil { + log.GetLogger().Errorf("failed to add uncontrolled: %s", err.Error()) + return model.Layer{}, err + } + + if err := uploader.Upload(); err != nil { + storage.RemoveUncontrolled(ctx, uploader.BucketID(), uploader.ObjectID()) + return model.Layer{}, err + } + + layer := createLayerValue(pkg, uploader, req, compatibleRuntimes) + res, err := createLayerHelper(txn, layerName, version, layer) + if err != nil { + // remove uncontrolled iff pkg deletion is successful + if e := uploader.Rollback(); e == nil { + storage.RemoveUncontrolled(ctx, uploader.BucketID(), uploader.ObjectID()) + } + return model.Layer{}, err + } + storage.RemoveUncontrolled(ctx, uploader.BucketID(), uploader.ObjectID()) + return res, err +} + +func getCompatibleRuntimes(compatibleRuntimes string) ([]string, error) { + var result []string + err := json.Unmarshal([]byte(compatibleRuntimes), &result) + if err != nil { + return nil, err + } + if len(result) > compatibleRuntimeMaxSize || len(result) < compatibleRuntimeMinSize { + return nil, errmsg.NewParamError( + "compatibleRuntimes size must be between %d and %d", compatibleRuntimeMinSize, compatibleRuntimeMaxSize) + } + return result, nil +} + +func createLayerValue(pkg pkgstore.Package, uploader pkgstore.Uploader, + req model.CreateLayerRequest, compatibleRuntimes []string) storage.LayerValue { + return storage.LayerValue{ + Package: storage.Package{ + StorageType: common.S3StorageType, + S3Package: storage.S3Package{ + Signature: pkg.Signature(), + Size: pkg.Size(), + BucketID: uploader.BucketID(), + ObjectID: uploader.ObjectID(), + }, + }, + CompatibleRuntimes: compatibleRuntimes, + Description: req.Description, + LicenseInfo: req.LicenseInfo, + CreateTime: time.Now(), + UpdateTime: time.Now(), + } +} + +func createLayerHelper( + txn *storage.Txn, layerName string, layerVersion int, layer storage.LayerValue, +) (model.Layer, error) { + if err := storage.CreateLayer(txn, layerName, layerVersion, layer); err != nil { + log.GetLogger().Errorf("failed to create layer: %s", err.Error()) + return model.Layer{}, err + } + + if err := txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit transaction: %s", err.Error()) + return model.Layer{}, err + } + + var res model.Layer + if err := buildLayer(txn.GetCtx(), &res, layerName, layerVersion, layer); err != nil { + log.GetLogger().Errorf("failed to build layer") + return model.Layer{}, err + } + return res, nil +} + +// DeleteLayer handles the request of deleting all layer versions +func DeleteLayer(ctx server.Context, layerName string) error { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + layers, err := storage.GetLayerTx(txn, layerName) + if err != nil { + log.GetLogger().Errorf("failed to get layer: %s", err.Error()) + return err + } + usedVersion, err := storage.IsLayerUsed(txn, layerName) + if err != nil { + log.GetLogger().Errorf("failed to check is layer used: %s", err.Error()) + return err + } + if usedVersion != 0 { + log.GetLogger().Errorf("layer is still in used") + return errmsg.New(errmsg.LayerIsUsed, usedVersion) + } + + return deleteLayerHelper(txn, layers, layerName) +} + +func deleteLayerHelper(txn *storage.Txn, layers []storage.LayerTuple, layerName string) error { + for _, v := range layers { + if err := storage.AddUncontrolledTx(txn, v.Value.Package.BucketID, v.Value.Package.ObjectID); err != nil { + log.GetLogger().Errorf("failed to add uncontrolled: %s", err.Error()) + return err + } + } + if err := storage.DeleteLayer(txn, layerName); err != nil { + log.GetLogger().Errorf("failed to delete layer: %s", err.Error()) + return err + } + if err := txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit: %s", err.Error()) + return err + } + for _, v := range layers { + if err := pkgstore.Delete( + v.Value.Package.BucketID, v.Value.Package.ObjectID); err == nil { + storage.RemoveUncontrolled(txn.GetCtx(), v.Value.Package.BucketID, v.Value.Package.ObjectID) + } + } + return nil +} + +// DeleteLayerVersion handles the request of deleting a specific layer version +func DeleteLayerVersion(ctx server.Context, layerName string, layerVersion int) error { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + layer, err := storage.GetLayerVersionTx(txn, layerName, layerVersion) + if err != nil { + if err == errmsg.KeyNotFoundError { + return snerror.NewWithFmtMsg( + errmsg.LayerVersionNotFound, + "this version [%d] of layer [%s] does not exist", layerVersion, layerName) + } + log.GetLogger().Errorf("failed to get layer version: %s", err.Error()) + return err + } + used, err := storage.IsLayerVersionUsed(txn, layerName, layerVersion) + if err != nil { + log.GetLogger().Errorf("failed to check is layer used failed: %s", err.Error()) + return err + } + if used { + log.GetLogger().Errorf("layer is still in used") + return errmsg.New(errmsg.LayerIsUsed, layerVersion) + } + + if err := storage.AddUncontrolledTx(txn, layer.Package.BucketID, layer.Package.ObjectID); err != nil { + log.GetLogger().Errorf("failed to add uncontrolled: %s", err.Error()) + return err + } + if err := storage.DeleteLayerVersion(txn, layerName, layerVersion); err != nil { + log.GetLogger().Errorf("failed to delete layer: %s", err.Error()) + return err + } + if err := txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit: %s", err.Error()) + return err + } + if err := pkgstore.Delete(layer.Package.BucketID, layer.Package.ObjectID); err == nil { + storage.RemoveUncontrolled(ctx, layer.Package.BucketID, layer.Package.ObjectID) + } + return nil +} + +func filterStream(stream *storage.LayerPrepareStmt, compatibleRuntime string) *storage.LayerPrepareStmt { + stream = stream.Filter(func(key storage.LayerKey, val storage.LayerValue) bool { + var compatible bool + for _, cr := range val.CompatibleRuntimes { + if cr == compatibleRuntime { + compatible = true + break + } + } + return compatible + }) + return stream +} + +func getLayerListHelper( + ctx server.Context, stream *storage.LayerPrepareStmt, compatibleRuntime string, pageIndex, pageSize int, +) (model.LayerList, error) { + if compatibleRuntime != "" { + stream = filterStream(stream, compatibleRuntime) + } + tuples, total, err := stream.ExecuteWithPage(pageIndex, pageSize) + if err != nil { + log.GetLogger().Errorf("failed to execute with page: %s", err.Error()) + return model.LayerList{}, err + } + layers := make([]model.Layer, len(tuples), len(tuples)) + for i, tuple := range tuples { + if err := buildLayer(ctx, &layers[i], tuple.Key.LayerName, tuple.Key.LayerVersion, tuple.Value); err != nil { + log.GetLogger().Errorf("failed to build layer: %s", err.Error()) + return model.LayerList{}, err + } + } + return model.LayerList{ + Total: total, + LayerVersions: layers, + }, nil +} + +// GetLayerList handles the request of querying multiple layers +func GetLayerList(ctx server.Context, req model.GetLayerListRequest) (model.LayerList, error) { + if req.CompatibleRuntime != "" { + if err := isCompatibleRuntimeValid([]string{req.CompatibleRuntime}); err != nil { + return model.LayerList{}, err + } + } + if req.OwnerFlag != 1 { + return model.LayerList{}, snerror.NewWithFmtMsg(0, "ownerflag %v is not implemented", req.OwnerFlag) + } + + stream, err := storage.GetLayerStream(ctx, "") + if err != nil { + log.GetLogger().Errorf("failed to get stream: %s", err.Error()) + return model.LayerList{}, err + } + if req.LayerName != "" { + stream = stream.Filter(func(key storage.LayerKey, val storage.LayerValue) bool { + return strings.Contains(key.LayerName, req.LayerName) + }) + } + if req.LayerVersion != 0 { + stream = stream.Filter(func(key storage.LayerKey, val storage.LayerValue) bool { + return key.LayerVersion == req.LayerVersion + }) + } + res, err := getLayerListHelper(ctx, stream, req.CompatibleRuntime, req.PageIndex, req.PageSize) + if err != nil { + if err == errmsg.KeyNotFoundError { + return model.LayerList{}, snerror.New(errmsg.LayerVersionNotFound, "the layer does not exist") + } + return model.LayerList{}, err + } + return res, nil +} + +// GetLayerVersion handles the request of querying a specific layer version +func GetLayerVersion(ctx server.Context, layerName string, layerVersion int) (model.Layer, error) { + var res model.Layer + + if layerVersion == 0 { + layer, version, err := storage.GetLayerLatestVersion(ctx, layerName) + if err != nil { + if err == errmsg.KeyNotFoundError { + return model.Layer{}, snerror.New(errmsg.LayerVersionNotFound, "can not find this layer") + } + log.GetLogger().Errorf("failed to get layer latest version: %s", err.Error()) + return model.Layer{}, err + } + if err := buildLayer(ctx, &res, layerName, version, layer); err != nil { + log.GetLogger().Errorf("failed to build layer: %s", err) + return model.Layer{}, err + } + + return res, nil + } + + layer, err := storage.GetLayerVersion(ctx, layerName, layerVersion) + if err != nil { + if err == errmsg.KeyNotFoundError { + return model.Layer{}, snerror.NewWithFmtMsg( + errmsg.LayerVersionNotFound, "this version [%d] of layer [%s] does not exist", + layerVersion, layerName) + } + log.GetLogger().Errorf("failed to get layer version: %s", err.Error()) + return model.Layer{}, err + } + if err := buildLayer(ctx, &res, layerName, layerVersion, layer); err != nil { + log.GetLogger().Errorf("failed to build layer: %s", err) + return model.Layer{}, err + } + + return res, nil +} + +// GetLayerVersionList is similar to "GetLayerList". The main difference is that "GetLayerList" fuzzy searches +// layerName. +func GetLayerVersionList(ctx server.Context, req model.GetLayerVersionListRequest) (model.LayerList, error) { + if req.CompatibleRuntime != "" { + if err := isCompatibleRuntimeValid([]string{req.CompatibleRuntime}); err != nil { + return model.LayerList{}, err + } + } + + stream, err := storage.GetLayerStream(ctx, req.LayerName) + if err != nil { + log.GetLogger().Errorf("failed to get stream: %s", err.Error()) + return model.LayerList{}, err + } + res, err := getLayerListHelper(ctx, stream, req.CompatibleRuntime, req.PageIndex, req.PageSize) + if err != nil { + if err == errmsg.KeyNotFoundError { + return model.LayerList{}, snerror.New(errmsg.LayerVersionNotFound, "this layer version does not exist") + } + return model.LayerList{}, err + } + return res, nil +} + +// UpdateLayer handles the request of updating a specific layer version +func UpdateLayer(ctx server.Context, layerName string, req model.UpdateLayerRequest) (model.Layer, error) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + layer, err := storage.GetLayerVersionTx(txn, layerName, req.Version) + if err != nil { + log.GetLogger().Errorf("failed to get layer latest version: %s", err.Error()) + return model.Layer{}, err + } + if !layer.UpdateTime.Equal(req.LastUpdateTime) { + log.GetLogger().Errorf("check updateTime error, a concurrent update may happen") + return model.Layer{}, snerror.New(errmsg.RevisionIDError, "last update time is not the same") + } + + layer.LicenseInfo = req.LicenseInfo + layer.Description = req.Description + layer.UpdateTime = time.Now() + + if err := storage.UpdateLayer(txn, layerName, req.Version, layer); err != nil { + log.GetLogger().Errorf("failed to update layer: %s", err.Error()) + return model.Layer{}, err + } + if err := txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit: %s", err.Error()) + return model.Layer{}, err + } + + var res model.Layer + if err := buildLayer(ctx, &res, layerName, req.Version, layer); err != nil { + log.GetLogger().Errorf("failed to build layer: %s", err) + return model.Layer{}, err + } + return res, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/service/layer_test.go b/functionsystem/apps/meta_service/function_repo/service/layer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1f7b281697df65931f35cb09b4aa711b4b2141f3 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/layer_test.go @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "archive/zip" + "bytes" + "errors" + "io" + "mime/multipart" + "net/http" + "os" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "meta_service/common/constants" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/test/fakecontext" +) + +type mockPackage struct{} + +func (m *mockPackage) Name() string { return "" } + +func (m *mockPackage) FileName() string { return "" } + +func (m *mockPackage) Move() string { return "" } + +func (m *mockPackage) Signature() string { return "" } + +func (m *mockPackage) Size() int64 { return 64 } + +func (m *mockPackage) Close() error { return nil } + +type zipFile struct { + Name, Body string +} + +func newZipFile(files []zipFile) ([]byte, error) { + buf := new(bytes.Buffer) + + w := zip.NewWriter(buf) + for _, file := range files { + f, err := w.Create(file.Name) + if err != nil { + return nil, err + } + + _, err = f.Write([]byte(file.Body)) + if err != nil { + return nil, err + } + } + + err := w.Close() + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func TestLayer(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyMethod(reflect.TypeOf(&http.Request{}), "ParseMultipartForm", + func(r *http.Request, maxMemory int64) error { + return nil + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, errors.New("mock err") + }).ApplyMethod(reflect.TypeOf(&multipart.FileHeader{}), "Open", + func(*multipart.FileHeader) (multipart.File, error) { + return &os.File{}, nil + }).ApplyFunc(pkgstore.NewPackage, func(c server.Context, name string, reader io.Reader, + n int64) (pkgstore.Package, error) { + return &mockPackage{}, nil + }) + defer patches.Reset() + ctx := fakecontext.NewContext() + ctx.Gin().Request = &http.Request{Header: make(map[string][]string)} + ctx.Gin().Request.Header.Set(constants.HeaderCompatibleRuntimes, "[\"cpp11\"]") + ctx.Gin().Request.Header.Set(constants.HeaderDescription, "description") + ctx.Gin().Request.Header.Set(constants.HeaderLicenseInfo, "license") + ctx.Gin().Request.MultipartForm = &multipart.Form{File: map[string][]*multipart.FileHeader{ + "file": []*multipart.FileHeader{{}}, + }} + ctx.InitTenantInfo(server.TenantInfo{ + BusinessID: "yrk", + TenantID: "qqq", + ProductID: "", + }) + + req := model.CreateLayerRequest{ + ZipFileSize: 100, + CompatibleRuntimes: "[\"cpp11\"]", + Description: "this is my layer", + LicenseInfo: "MIT", + } + _, err := CreateLayer(ctx, "my-layer", req) + require.NoError(t, err, "create layer") + + _, err = CreateLayer(ctx, "my-layer", req) + require.NoError(t, err, "create layer") + + _, err = CreateLayer(ctx, "my-layer-2", req) + require.NoError(t, err, "create layer") + + err = DeleteLayerVersion(ctx, "my-layer", 1) + require.NoError(t, err, "delete layer") + + err = DeleteLayerVersion(ctx, "my-layer-2", 1) + require.NoError(t, err, "delete layer") + + _, err = CreateLayer(ctx, "my-layer", req) + require.NoError(t, err, "create layer") + + layer, err := GetLayerVersion(ctx, "my-layer", 0) + require.NoError(t, err, "get layer") + assert.Equal(t, "my-layer", layer.Name) + assert.Equal(t, 3, layer.Version) + assert.Equal(t, "this is my layer", layer.Description) + assert.Equal(t, "MIT", layer.LicenseInfo) + + layer, err = GetLayerVersion(ctx, "my-layer", 2) + require.NoError(t, err, "get layer") + assert.Equal(t, "my-layer", layer.Name) + assert.Equal(t, 2, layer.Version) + assert.Equal(t, "this is my layer", layer.Description) + assert.Equal(t, "MIT", layer.LicenseInfo) + + _, err = UpdateLayer(ctx, "my-layer", model.UpdateLayerRequest{ + Description: "new description", + LicenseInfo: "Apache", + LastUpdateTime: layer.UpdateTime, + Version: 2, + }) + require.NoError(t, err, "update layer") + + list, err := GetLayerVersionList(ctx, model.GetLayerVersionListRequest{ + LayerName: "my-layer", + CompatibleRuntime: "", + PageIndex: 0, + PageSize: 0, + }) + require.NoError(t, err, "get layer list") + assert.Equal(t, 2, list.Total) + assert.Equal(t, 2, len(list.LayerVersions)) + + listreq := model.GetLayerListRequest{ + LayerName: "layer", + LayerVersion: 0, + CompatibleRuntime: "", + OwnerFlag: 1, + PageIndex: 0, + PageSize: 0, + } + list, err = GetLayerList(ctx, listreq) + require.NoError(t, err, "get layer list") + assert.Equal(t, 2, list.Total) + assert.Equal(t, 2, len(list.LayerVersions)) +} + +func TestParseLayerInfo(t *testing.T) { + Convey("Test ParseLayerInfo", t, func() { + ctx := fakecontext.NewMockContext() + // sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:1 + _, err := ParseLayerInfo(ctx, "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:layer:aa:1") + So(err, ShouldBeNil) + _, err = ParseLayerInfo(ctx, "sn:cn:yrk123:i1fe539427b24702acc11fbb4e134e17:layer:aa:1") + So(err, ShouldNotBeNil) + }) +} + +func TestCreateLayerHelper(t *testing.T) { + Convey("Test createLayerHelper", t, func() { + txn := &storage.Txn{} + Convey("when CreateLayer err", func() { + patches := gomonkey.ApplyFunc(storage.CreateLayer, + func(*storage.Txn, string, int, storage.LayerValue) error { + return errors.New("mock err") + }) + defer patches.Reset() + _, err := createLayerHelper(txn, "a", 1, storage.LayerValue{}) + So(err, ShouldNotBeNil) + }) + Convey("when Commit err", func() { + patches := gomonkey.ApplyFunc(storage.CreateLayer, + func(*storage.Txn, string, int, storage.LayerValue) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return errors.New("mock err") + }) + defer patches.Reset() + _, err := createLayerHelper(txn, "a", 1, storage.LayerValue{}) + So(err, ShouldNotBeNil) + }) + Convey("when buildLayer err", func() { + patches := gomonkey.ApplyFunc(storage.CreateLayer, func( + *storage.Txn, string, int, storage.LayerValue) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return nil + }).ApplyFunc(buildLayer, func( + server.Context, *model.Layer, string, int, storage.LayerValue) error { + return errors.New("mock err") + }) + defer patches.Reset() + _, err := createLayerHelper(txn, "a", 1, storage.LayerValue{}) + So(err, ShouldNotBeNil) + }) + + }) +} + +func TestGetLayerVersion(t *testing.T) { + ctx := fakecontext.NewMockContext() + Convey("Test GetLayerVersion when GetLayerLatestVersion err", t, func() { + patches := gomonkey.ApplyFunc(storage.GetLayerLatestVersion, func( + ctx server.Context, layerName string) (storage.LayerValue, int, error) { + return storage.LayerValue{}, 0, errmsg.KeyNotFoundError + }) + _, err := GetLayerVersion(ctx, "mock-layer", 0) + So(err, ShouldNotBeNil) + patches.Reset() + + patches.ApplyFunc(storage.GetLayerLatestVersion, func( + ctx server.Context, layerName string) (storage.LayerValue, int, error) { + return storage.LayerValue{}, 0, errors.New("mock err") + }) + _, err = GetLayerVersion(ctx, "mock-layer", 0) + So(err, ShouldNotBeNil) + patches.Reset() + }) + Convey("Test GetLayerVersion when buildLayer err", t, func() { + patches := gomonkey.ApplyFunc(storage.GetLayerLatestVersion, func( + ctx server.Context, layerName string) (storage.LayerValue, int, error) { + return storage.LayerValue{}, 0, nil + }).ApplyFunc(buildLayer, func( + ctx server.Context, mlayer *model.Layer, layerName string, layerVersion int, layer storage.LayerValue, + ) error { + return errors.New("mock err") + }) + _, err := GetLayerVersion(ctx, "mock-layer", 0) + So(err, ShouldNotBeNil) + patches.Reset() + }) + Convey("Test GetLayerVersion when buildLayer err", t, func() { + patches := gomonkey.ApplyFunc(storage.GetLayerVersion, func( + server.Context, string, int) (storage.LayerValue, error) { + return storage.LayerValue{}, errmsg.KeyNotFoundError + }) + _, err := GetLayerVersion(ctx, "mock-layer", 1) + So(err, ShouldNotBeNil) + patches.Reset() + + patches.ApplyFunc(storage.GetLayerVersion, func( + server.Context, string, int) (storage.LayerValue, error) { + return storage.LayerValue{}, errors.New("mock err") + }) + _, err = GetLayerVersion(ctx, "mock-layer", 1) + So(err, ShouldNotBeNil) + patches.Reset() + }) + Convey("Test GetLayerVersion when buildLayer err 2", t, func() { + patches := gomonkey.ApplyFunc(storage.GetLayerVersion, func( + server.Context, string, int) (storage.LayerValue, error) { + return storage.LayerValue{}, nil + }).ApplyFunc(buildLayer, func( + ctx server.Context, mlayer *model.Layer, layerName string, layerVersion int, layer storage.LayerValue, + ) error { + return errors.New("mock err") + }) + _, err := GetLayerVersion(ctx, "mock-layer", 1) + So(err, ShouldNotBeNil) + patches.Reset() + }) +} + +func TestGetLayerVersionList(t *testing.T) { + Convey("Test GetLayerVersionList", t, func() { + mode := 0 + patches := gomonkey.ApplyFunc(isCompatibleRuntimeValid, func(cps []string) error { + if mode == 1 { + return errors.New("mock err") + } + return nil + }).ApplyFunc(storage.GetLayerStream, func(ctx server.Context, layerName string) (*storage.LayerPrepareStmt, error) { + if mode == 2 { + return nil, errors.New("mock err") + } + return nil, nil + }).ApplyFunc(getLayerListHelper, func( + ctx server.Context, stream *storage.LayerPrepareStmt, compatibleRuntime string, pageIndex, pageSize int, + ) (model.LayerList, error) { + if mode == 3 { + return model.LayerList{}, errmsg.KeyNotFoundError + } + return model.LayerList{}, errors.New("mock err") + }) + defer patches.Reset() + for mode = 0; mode < 4; mode++ { + _, err := GetLayerVersionList(fakecontext.NewMockContext(), model.GetLayerVersionListRequest{ + CompatibleRuntime: "custom-mock", + }) + So(err, ShouldNotBeNil) + } + }) +} + +func TestUpdateLayer(t *testing.T) { + Convey("Test UpdateLayer", t, func() { + mockLastUpdateTime := time.Now() + time2 := mockLastUpdateTime.Add(time.Second * 10) + txn := &storage.Txn{} + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(storage.NewTxn, func(c server.Context) *storage.Txn { + return txn + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Cancel", func(_ *storage.Txn) {}) + mode := 0 + patches.ApplyFunc(storage.GetLayerVersionTx, func(txn storage.Transaction, layerName string, layerVersion int) (storage.LayerValue, error) { + if mode == 0 { + return storage.LayerValue{}, errors.New("mock err") + } + return storage.LayerValue{UpdateTime: mockLastUpdateTime}, nil + }) + patches.ApplyFunc(storage.UpdateLayer, func(txn *storage.Txn, layerName string, layerVersion int, layer storage.LayerValue) error { + if mode == 2 { + return errors.New("mock err") + } + return nil + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + if mode == 3 { + return errors.New("mock err") + } + return nil + }) + patches.ApplyFunc(buildLayer, func( + ctx server.Context, mlayer *model.Layer, layerName string, layerVersion int, layer storage.LayerValue, + ) error { + if mode == 4 { + return errors.New("mock err") + } + return nil + }) + for mode = 0; mode < 5; mode++ { + req := model.UpdateLayerRequest{} + if mode == 1 { + req.LastUpdateTime = time2 + } else { + req.LastUpdateTime = mockLastUpdateTime + } + _, err := UpdateLayer(fakecontext.NewMockContext(), "mock-layer", req) + So(err, ShouldNotBeNil) + } + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/podpool.go b/functionsystem/apps/meta_service/function_repo/service/podpool.go new file mode 100644 index 0000000000000000000000000000000000000000..2928d18f116c854b1e13de5e71faeeee0efd7b40 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/podpool.go @@ -0,0 +1,410 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service is processing service codes +package service + +import ( + "errors" + "fmt" + "regexp" + + "meta_service/common/logger/log" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/utils/constants" +) + +const ( + idRegex = "^[a-z0-9]([-a-z0-9]{0,38}[a-z0-9])?$" + groupRegex = "^[a-zA-Z0-9]([-a-zA-Z0-9]{0,38}[a-zA-Z0-9])?$" + minSize = 0 + maxImageLen = 200 + maxInitImageLen = 200 + maxRuntimeClassLen = 64 + defaultPageSize = 10 + defaultPageIndex = 1 + minIdleTime = -1 +) + +var ( + errIDFormat = errors.New("pod pool id can contain only lowercase letters, digits and '-'. " + + "it cannot start or end with '-' and cannot exceed 40 characters or less than 1 characters") + errGroupFormat = errors.New("pod pool group can contain only letters, digits and '-'. " + + "it cannot start or end with '-' and cannot exceed 40 characters") +) + +// CreatePodPool create pod pool +func CreatePodPool(ctx server.Context, req model.PodPoolCreateRequest) (model.PodPoolCreateResponse, error) { + if len(req.Pools) == 0 { + return model.PodPoolCreateResponse{}, errors.New("1failed to create pod pool, err: input pod pool list is empty") + } + var res model.PodPoolCreateResponse + var returnErrMsg string + for _, pool := range req.Pools { + if err := createPodPool(ctx, pool); err != nil { + res.FailedPools = append(res.FailedPools, pool.Id) + returnErrMsg = joinErrMsg(returnErrMsg, err) + } + } + if returnErrMsg != "" { + return res, errors.New(returnErrMsg) + } + return res, nil +} + +func createPodPool(ctx server.Context, pool model.Pool) error { + if err := validatePodPoolCreateParams(&pool); err != nil { + return err + } + txn := storage.GetTxnByKind(ctx, "") + defer txn.Cancel() + info, err := storage.GetPodPool(txn, pool.Id) + if err == nil || info.Status == constants.Deleted { + errMsg := fmt.Sprintf("failed to create pod pool, id: %s,"+ + "err: pool id already exists or is deleting", pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + poolVal := convertPoolModelToPoolValue(pool) + err = storage.CreateUpdatePodPool(txn, poolVal) + if err != nil { + return err + } + if err = txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit transaction: %s", err.Error()) + return err + } + return nil +} + +func joinErrMsg(errMsg string, err error) string { + if errMsg == "" { + return err.Error() + } + return errMsg + ";" + err.Error() +} + +func validatePodPoolCreateParams(pool *model.Pool) error { + var errMsg string + if pool == nil { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: pod pool is empty", pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + isMatch, err := regexp.MatchString(idRegex, pool.Id) + if err != nil { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: %s", pool.Id, err.Error()) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if !isMatch { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: %s", pool.Id, errIDFormat.Error()) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if pool.Group == "" { + pool.Group = "default" + } + isMatch, err = regexp.MatchString(groupRegex, pool.Group) + if err != nil { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, group: %s, err: %s", + pool.Id, pool.Group, err.Error()) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if !isMatch { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, group: %s, err: %s", + pool.Id, pool.Group, errGroupFormat.Error()) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + return validatePodPoolCreateParamsLength(pool) +} + +func validatePodPoolCreateParamsLength(pool *model.Pool) error { + var errMsg string + if pool.Size < minSize { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: size is less than 0", pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if pool.Size != 0 && pool.MaxSize == 0 { + pool.MaxSize = pool.Size + } + if pool.MaxSize < pool.Size { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: max_size is less than size", pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if pool.MaxSize > pool.Size && len(pool.HorizontalPodAutoscalerSpec) > 0 { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, "+ + "err: max_size greater than size and horizontal_pod_autoscaler_spec cannot be set at the same time", + pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if pool.IdleRecycleTime.Reserved < minIdleTime { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: idle time of reserved should not less than %d", + pool.Id, minIdleTime) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if pool.IdleRecycleTime.Scaled < minIdleTime { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: idle time of scaled should not less than %d", + pool.Id, minIdleTime) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if len(pool.Image) > maxImageLen { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: image len is not in [1, 200]", pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if len(pool.InitImage) > maxInitImageLen { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: init image len is not in [1, 200]", pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if len(pool.RuntimeClassName) > maxRuntimeClassLen { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: runtime_class_name should not greater than %d", + pool.Id, maxRuntimeClassLen) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if pool.PodPendingDurationThreshold < 0 { + errMsg = fmt.Sprintf("failed to create pod pool, id: %s, err: podPendingDurationThreshold should not be less"+ + " than 0, which is %d", pool.Id, pool.PodPendingDurationThreshold) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + return nil +} + +// DeletePodPool delete pod pool +func DeletePodPool(ctx server.Context, id, group string) error { + if id == "" && group == "" { + return errors.New("id and group cannot be empty at the same time") + } + // start transaction + txn := storage.GetTxnByKind(ctx, "") + defer txn.Cancel() + // query all without page info + pools, _, err := storage.GetPodPoolList(ctx, id, group, -1, -1) + if err != nil { + log.GetLogger().Errorf("failed to get pod pool to delete, id: %s, group: %s, err: %s", id, group, err.Error()) + return err + } + if len(pools) == 0 { + return errmsg.PoolNotFoundError + } + for _, pool := range pools { + err = deletePodPool(txn, pool.Value) + if err != nil { + log.GetLogger().Errorf("failed to delete pod pool, id: %s, err: %s", pool.Value.Id, err.Error()) + return err + } + } + if err = txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit transaction: %s", err.Error()) + return err + } + return nil +} + +func deletePodPool(txn storage.Transaction, pool storage.PoolValue) error { + pool.Status = constants.Deleted + if err := storage.CreateUpdatePodPool(txn, pool); err != nil { + return err + } + return nil +} + +// UpdatePodPool update pod pool info +func UpdatePodPool(ctx server.Context, req model.PodPoolUpdateRequest) error { + if req.Size < minSize { + errMsg := fmt.Sprintf("failed to update pod pool, err: size is less than 0") + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if req.MaxSize < minSize { + errMsg := fmt.Sprintf("failed to update pod pool, err: max_size is less than 0") + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + // start transaction + txn := storage.GetTxnByKind(ctx, "") + defer txn.Cancel() + pool, err := storage.GetPodPool(txn, req.ID) + if err != nil { + if err == errmsg.KeyNotFoundError { + return errmsg.PoolNotFoundError + } + return err + } + if pool.Status == constants.Deleted { + errMsg := fmt.Sprintf("the pod pool has already deleted, id: %s", pool.Id) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } else if pool.Status == constants.New || pool.Status == constants.Update { + errMsg := fmt.Sprintf("pool status is %d , not allowed to update", pool.Status) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if !pool.Scalable { + // not scalable, ignore max_size + req.MaxSize = req.Size + } else { + // if max size is 0, but size is not default, set max_size to old + if req.MaxSize == 0 && req.Size != 0 { + req.MaxSize = pool.MaxSize + } + } + pool.Size = req.Size + pool.MaxSize = req.MaxSize + pool.Status = constants.Update + if len(req.HorizontalPodAutoscalerSpec) != 0 { + pool.HorizontalPodAutoscalerSpec = req.HorizontalPodAutoscalerSpec + } + if req.MaxSize < req.Size { + errMsg := fmt.Sprintf("failed to update pod pool, err: max_size is less than size") + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if pool.MaxSize > pool.Size && len(pool.HorizontalPodAutoscalerSpec) > 0 { + errMsg := fmt.Sprintf("failed to update pod pool, " + + "err: max_size greater than size and horizontal_pod_autoscaler_spec cannot be set at the same time") + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if err = storage.CreateUpdatePodPool(txn, pool); err != nil { + return err + } + if err = txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit transaction: %s", err.Error()) + return err + } + return nil +} + +// GetPodPool get pod pool info +func GetPodPool(ctx server.Context, req model.PodPoolGetRequest) (model.PodPoolGetResponse, error) { + err := validatePodPoolGetParams(req) + if err != nil { + return model.PodPoolGetResponse{}, nil + } + resp := model.PodPoolGetResponse{} + txn := storage.GetTxnByKind(ctx, "") + defer txn.Cancel() + pageIndex := req.Offset + pageSize := req.Limit + if pageIndex <= 0 { + pageIndex = defaultPageIndex + } + if pageSize <= 0 { + pageSize = defaultPageSize + } + tuples, total, err := storage.GetPodPoolList(ctx, req.ID, req.Group, pageIndex, pageSize) + if err != nil { + log.GetLogger().Errorf("failed to get pool list :%s", err.Error()) + return resp, nil + } + pools := make([]model.Pool, len(tuples), len(tuples)) + for i, tuple := range tuples { + pools[i] = convertPoolValueToPoolModel(tuple.Value) + } + resp.PodPools = pools + resp.Count = total + return resp, nil +} + +func convertPoolValueToPoolModel(val storage.PoolValue) model.Pool { + return model.Pool{ + Id: val.Id, + Group: val.Group, + Size: val.Size, + MaxSize: val.MaxSize, + ReadyCount: val.ReadyCount, + Status: val.Status, + Msg: val.Msg, + Image: val.Image, + InitImage: val.InitImage, + Reuse: val.Reuse, + Labels: val.Labels, + Environment: val.Environment, + Resources: val.Resources, + Volumes: val.Volumes, + VolumeMounts: val.VolumeMounts, + RuntimeClassName: val.RuntimeClassName, + NodeSelector: val.NodeSelector, + Tolerations: val.Tolerations, + Affinities: val.Affinities, + HorizontalPodAutoscalerSpec: val.HorizontalPodAutoscalerSpec, + TopologySpreadConstraints: val.TopologySpreadConstraints, + PodPendingDurationThreshold: val.PodPendingDurationThreshold, + IdleRecycleTime: val.IdleRecycleTime, + Scalable: val.Scalable, + } +} + +func convertPoolModelToPoolValue(val model.Pool) storage.PoolValue { + poolVal := storage.PoolValue{} + poolVal.Id = val.Id + poolVal.Group = val.Group + poolVal.Size = val.Size + poolVal.MaxSize = val.MaxSize + poolVal.ReadyCount = val.ReadyCount + poolVal.Status = val.Status + poolVal.Msg = val.Msg + poolVal.Image = val.Image + poolVal.InitImage = val.InitImage + poolVal.Reuse = val.Reuse + poolVal.Labels = val.Labels + poolVal.Environment = val.Environment + poolVal.Resources = val.Resources + poolVal.VolumeMounts = val.VolumeMounts + poolVal.Volumes = val.Volumes + poolVal.Affinities = val.Affinities + poolVal.Tolerations = val.Tolerations + poolVal.NodeSelector = val.NodeSelector + poolVal.RuntimeClassName = val.RuntimeClassName + poolVal.HorizontalPodAutoscalerSpec = val.HorizontalPodAutoscalerSpec + poolVal.TopologySpreadConstraints = val.TopologySpreadConstraints + poolVal.PodPendingDurationThreshold = val.PodPendingDurationThreshold + poolVal.IdleRecycleTime = val.IdleRecycleTime + if val.MaxSize > val.Size { + poolVal.Scalable = true + } else { + poolVal.Scalable = false + } + return poolVal +} + +func validatePodPoolGetParams(req model.PodPoolGetRequest) error { + if req.Limit <= 0 { + errMsg := fmt.Sprintf("pod pool page: %d should greater than 0", req.Limit) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + if req.Offset < 0 { + errMsg := fmt.Sprintf("pod pool offset: %d should greater equal than 0", req.Offset) + log.GetLogger().Errorf(errMsg) + return errors.New(errMsg) + } + return nil +} diff --git a/functionsystem/apps/meta_service/function_repo/service/podpool_test.go b/functionsystem/apps/meta_service/function_repo/service/podpool_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0a6544485c751b68bfe978be05a11ebc681cbfb1 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/podpool_test.go @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service is processing service codes +package service + +import ( + "fmt" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/assertions/should" + . "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/model" + "meta_service/function_repo/storage" + "meta_service/function_repo/test/fakecontext" + "meta_service/function_repo/utils/constants" +) + +func Test_updatePodPool(t *testing.T) { + Convey("Test update pool with deleted status", t, func() { + patches := gomonkey.ApplyFunc(storage.GetPodPool, func(storage.Transaction, string) (storage.PoolValue, error) { + pool := storage.PoolValue{} + pool.Status = constants.Deleted + return pool, nil + }) + defer patches.Reset() + ctx := fakecontext.NewMockContext() + err := UpdatePodPool(ctx, model.PodPoolUpdateRequest{ID: "pool1"}) + So(err, ShouldNotBeNil) + }) + Convey("Test update pool with new status", t, func() { + patches := gomonkey.ApplyFunc(storage.GetPodPool, func(storage.Transaction, string) (storage.PoolValue, error) { + pool := storage.PoolValue{} + pool.Status = constants.New + return pool, nil + }) + defer patches.Reset() + ctx := fakecontext.NewMockContext() + err := UpdatePodPool(ctx, model.PodPoolUpdateRequest{ID: "pool1"}) + So(err, ShouldNotBeNil) + }) + Convey("Test update pool with update status", t, func() { + patches := gomonkey.ApplyFunc(storage.GetPodPool, func(storage.Transaction, string) (storage.PoolValue, error) { + pool := storage.PoolValue{} + pool.Status = constants.Update + return pool, nil + }) + defer patches.Reset() + ctx := fakecontext.NewMockContext() + err := UpdatePodPool(ctx, model.PodPoolUpdateRequest{ID: "pool1"}) + So(err, ShouldNotBeNil) + }) + Convey("Test update pool with failed status", t, func() { + patches := gomonkey.ApplyFunc(storage.GetPodPool, func(storage.Transaction, string) (storage.PoolValue, error) { + pool := storage.PoolValue{} + pool.Status = constants.Failed + return pool, nil + }) + defer patches.Reset() + + patche1s := gomonkey.ApplyFunc(storage.CreateUpdatePodPool, func(storage.Transaction, storage.PoolValue) error { + return fmt.Errorf("update pod pool failed") + }) + defer patche1s.Reset() + ctx := fakecontext.NewMockContext() + err := UpdatePodPool(ctx, model.PodPoolUpdateRequest{ID: "pool1"}) + So(err.Error(), ShouldEqual, "update pod pool failed") + }) + Convey("Test update pool max size less than 0", t, func() { + patches := gomonkey.ApplyFunc(storage.GetPodPool, func(storage.Transaction, string) (storage.PoolValue, error) { + pool := storage.PoolValue{} + pool.Status = constants.Running + pool.Scalable = true + pool.Size = 1 + return pool, nil + }) + defer patches.Reset() + + ctx := fakecontext.NewMockContext() + err := UpdatePodPool(ctx, model.PodPoolUpdateRequest{ID: "pool1", MaxSize: -2}) + So(err.Error(), ShouldEqual, "failed to update pod pool, err: max_size is less than 0") + }) + Convey("Test update pool max size less than size", t, func() { + patches := gomonkey.ApplyFunc(storage.GetPodPool, func(storage.Transaction, string) (storage.PoolValue, error) { + pool := storage.PoolValue{} + pool.Status = constants.Running + pool.Scalable = true + pool.Size = 1 + return pool, nil + }) + defer patches.Reset() + + ctx := fakecontext.NewMockContext() + err := UpdatePodPool(ctx, model.PodPoolUpdateRequest{ID: "pool1", MaxSize: 1, Size: 2, + HorizontalPodAutoscalerSpec: "TEST"}) + So(err.Error(), ShouldEqual, "failed to update pod pool, err: max_size is less than size") + }) + Convey("Test update pool max size and horizontal_pod_autoscaler_spec", t, func() { + patches := gomonkey.ApplyFunc(storage.GetPodPool, func(storage.Transaction, string) (storage.PoolValue, error) { + pool := storage.PoolValue{} + pool.Status = constants.Running + pool.Scalable = true + pool.Size = 1 + return pool, nil + }) + defer patches.Reset() + + ctx := fakecontext.NewMockContext() + err := UpdatePodPool(ctx, model.PodPoolUpdateRequest{ID: "pool1", MaxSize: 2, Size: 1, + HorizontalPodAutoscalerSpec: "TEST"}) + So(err.Error(), ShouldEqual, "failed to update pod pool, err: max_size greater than size and horizontal_pod_autoscaler_spec cannot be set at the same time") + }) + +} + +func Test_createPodPool(t *testing.T) { + Convey("Test create pool error", t, func() { + ctx := fakecontext.NewMockContext() + _, err := CreatePodPool(ctx, model.PodPoolCreateRequest{Pools: []model.Pool{{Id: "pool1", Size: 2, + MaxSize: 1}}}) + So(err.Error(), should.ContainSubstring, "err: max_size is less than size") + _, err = CreatePodPool(ctx, model.PodPoolCreateRequest{Pools: []model.Pool{{Id: "pool1", Size: 2, + MaxSize: 3, HorizontalPodAutoscalerSpec: "test"}}}) + So(err.Error(), should.ContainSubstring, "max_size greater than size and horizontal_pod_autoscaler_spec cannot be set at the same time") + _, err = CreatePodPool(ctx, model.PodPoolCreateRequest{Pools: []model.Pool{{Id: "pool1", Size: 2, + MaxSize: 3, IdleRecycleTime: model.IdleRecyclePolicy{Reserved: -2}}}}) + So(err.Error(), should.ContainSubstring, "err: idle time of reserved should not less than") + _, err = CreatePodPool(ctx, model.PodPoolCreateRequest{Pools: []model.Pool{{Id: "pool1", Size: 2, + MaxSize: 3, IdleRecycleTime: model.IdleRecyclePolicy{Scaled: -2}}}}) + So(err.Error(), should.ContainSubstring, "err: idle time of scaled should not less than") + + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/publish.go b/functionsystem/apps/meta_service/function_repo/service/publish.go new file mode 100644 index 0000000000000000000000000000000000000000..1c02f80f9df966d025a7ea03e00eb67b0e4128e7 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/publish.go @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "errors" + "fmt" + "regexp" + "time" + + "meta_service/common/constants" + "meta_service/common/logger/log" + "meta_service/common/snerror" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/storage/publish" + "meta_service/function_repo/utils" +) + +// CheckFunctionVersion check whether the number of function versions exceeds the upper limit +func CheckFunctionVersion(ctx server.Context, functionName string) error { + total, err := storage.GetFunctionVersionSizeByName(ctx, functionName) + if err != nil { + return snerror.NewWithFmtMsg(errmsg.FunctionVersionNotFound, "failed to get function versions size: %s", + err.Error()) + } + versionMax := config.RepoCfg.FunctionCfg.VersionMax + if uint(total) >= versionMax+1 { + return snerror.NewWithFmtMsg(errmsg.FunctionVersionOutOfLimit, "the number of existing function "+ + "versions is greater than the set value: %d", total) + } + return nil +} + +func checkRepublish(txn *storage.Txn, funcVersion storage.FunctionVersionValue, req model.PublishRequest) error { + if funcVersion.Function.Version != "" { + _, err := storage.GetFunctionVersion(txn, funcVersion.Function.Name, + funcVersion.Function.Version) + if err != nil { + if err == errmsg.KeyNotFoundError { + return nil + } + return snerror.NewWithFmtMsg(errmsg.FunctionNotFound, "failed to get function version: %s", err.Error()) + } + } + return nil +} + +func checkVersionNumberDuplicate(txn *storage.Txn, funcVersion storage.FunctionVersionValue, + req model.PublishRequest, +) error { + _, err := storage.GetFunctionVersion(txn, funcVersion.Function.Name, funcVersion.Function.Version) + if err == errmsg.KeyNotFoundError { + return nil + } + return snerror.NewWithFmtMsg(errmsg.RepeatedPublishmentError, "duplicate version number") +} + +func validateVersionNumber(versionNum string) error { + matched, err := regexp.MatchString(`^[A-Za-z0-9][A-Za-z0-9._-]{0,40}[A-Za-z0-9]$|^[A-Za-z0-9]$`, versionNum) + if err != nil { + return err + } + if !matched { + return errors.New("versionNumber can only contain digits, letters, dots(.), hyphens(-), and underscores(_), " + + "and must start and end with a digit or letter, with less than 42 characters") + } + return nil +} + +func validateAndSetDefaultVersionNumber(versionNumber string) (string, error) { + if versionNumber == "" { + return fmt.Sprintf("v%s", time.Now().Format("20060102-150405")), nil + } else if err := validateVersionNumber(versionNumber); err != nil { + return "", err + } else { + return versionNumber, nil + } +} + +func checkPublishFunctionParams(ctx server.Context, funcName string, req model.PublishRequest, txn *storage.Txn) ( + storage.FunctionVersionValue, error, +) { + defaultVersion := utils.GetDefaultVersion() + if req.Kind == constants.Faas { + defaultVersion = utils.GetDefaultFaaSVersion() + } + funcVersion, err := storage.GetFunctionVersion(txn, funcName, defaultVersion) + if err != nil { + return storage.FunctionVersionValue{}, snerror.NewWithFmtMsg(errmsg.FunctionNotFound, + "failed to get function version") + } + if req.RevisionID != funcVersion.FunctionVersion.RevisionID { + return storage.FunctionVersionValue{}, snerror.NewWithFmtMsg(errmsg.RevisionIDError, + "revisionId is non latest version") + } + if err = checkRepublish(txn, funcVersion, req); err != nil { + return storage.FunctionVersionValue{}, err + } + // update latest function version + if funcVersion.Function.Version, err = validateAndSetDefaultVersionNumber(req.VersionNumber); err != nil { + return storage.FunctionVersionValue{}, err + } + if err = checkVersionNumberDuplicate(txn, funcVersion, req); err != nil { + return storage.FunctionVersionValue{}, err + } + if funcVersion.FunctionVersion.Package.StorageType == constants.S3StorageType && + funcVersion.FunctionVersion.Package.BucketID == "" && funcVersion.FunctionVersion.Package.ObjectID == "" { + return storage.FunctionVersionValue{}, errors.New("empty function version's bucket info") + } + + return funcVersion, nil +} + +// PublishFunction publish function version meta data and register function version value to etcd +func PublishFunction(ctx server.Context, funcName string, req model.PublishRequest) (model.PublishResponse, error) { + txn := storage.NewTxn(ctx) + defer txn.Cancel() + funcVersion, err := checkPublishFunctionParams(ctx, funcName, req, txn) + if err != nil { + return model.PublishResponse{}, err + } + if err = storage.UpdateFunctionVersion(txn, funcVersion); err != nil { + return model.PublishResponse{}, errors.New("failed to update function version") + } + + // publish function version meta data + publishFuncVersion := buildPublishFunctionVersion(req.VersionDesc, funcVersion) + if err = storage.CreateFunctionVersion(txn, publishFuncVersion); err != nil { + return model.PublishResponse{}, err + } + + if err = storage.CreateFunctionStatus(txn, funcName, publishFuncVersion.FunctionVersion.Version); err != nil { + return model.PublishResponse{}, err + } + + // publish function version register data + if err = publish.SavePublishFuncVersion(txn, publishFuncVersion); err != nil { + return model.PublishResponse{}, err + } + if err = txn.Commit(); err != nil { + return model.PublishResponse{}, err + } + + // recollect the func version + rsp, err := buildGetFunctionResponse(ctx, publishFuncVersion, "") + if err != nil { + log.GetLogger().Errorf("failed to build function response, %s", err) + return model.PublishResponse{}, err + } + if rsp.LastModified == "" { + rsp.LastModified = rsp.Created + } + return rsp, nil +} + +func buildPublishFunctionVersion(desc string, funcVersion storage.FunctionVersionValue) storage.FunctionVersionValue { + funcVersion.FunctionVersion.PublishTime = utils.NowTimeF() + funcVersion.FunctionVersion.Description = desc + funcVersion.FunctionVersion.Version = funcVersion.Function.Version + return funcVersion +} diff --git a/functionsystem/apps/meta_service/function_repo/service/publish_test.go b/functionsystem/apps/meta_service/function_repo/service/publish_test.go new file mode 100644 index 0000000000000000000000000000000000000000..85b89d335192c965382cb52672f86a0d2af1c97f --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/publish_test.go @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package service is processing service codes +package service + +import ( + "errors" + "regexp" + "strings" + "testing" + "time" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/test/fakecontext" + "meta_service/function_repo/utils" +) + +func TestCheckFunctionVersion(t *testing.T) { + ctx := fakecontext.NewContext() + ctx.InitTenantInfo(server.TenantInfo{ + BusinessID: "yrk", + TenantID: "qqq", + ProductID: "", + }) + + err := CheckFunctionVersion(ctx, "functionName") + if err != nil { + t.Errorf("err:%s", err.Error()) + } +} + +func TestPublishFunction(t *testing.T) { + ctx := fakecontext.NewContext() + ctx.InitTenantInfo(server.TenantInfo{ + BusinessID: "yrk", + TenantID: "qqq", + ProductID: "", + }) + req := model.PublishRequest{ + RevisionID: "123456", + VersionDesc: "version 123456", + } + + patches := gomonkey.NewPatches() + patches.ApplyFunc(storage.GetFunctionVersion, func(txn storage.Transaction, funcName, funcVer string) (storage.FunctionVersionValue, error) { + if funcVer != "" && funcVer != utils.GetDefaultVersion() && funcVer != utils.GetDefaultFaaSVersion() { + return storage.FunctionVersionValue{}, errmsg.KeyNotFoundError + } + return storage.FunctionVersionValue{ + Function: storage.Function{ + Version: "", + }, + FunctionVersion: storage.FunctionVersion{ + RevisionID: "123456", + Version: "", + Package: storage.Package{ + StorageType: "s3", + S3Package: storage.S3Package{ + BucketID: "abc", + ObjectID: "def", + }, + }, + }, + }, nil + }) + + patches.ApplyFunc(storage.UpdateFunctionVersion, func(txn storage.Transaction, fv storage.FunctionVersionValue) error { + return nil + }) + + defer patches.Reset() + + _, err := PublishFunction(ctx, "functionName", req) + if err != nil { + t.Errorf("err :%s", err.Error()) + } +} + +func TestVersionNumberValidation(t *testing.T) { + Convey("Test version number validation rule", t, func() { + // k: version number + // v: if ok, true if so, false is not + cases := map[string]bool{ + // valid cases + "B001": true, // upper case letters, and digits + "B.2.3": true, // upper case letters, and . + "2.3.4": true, // digits, and . + "a3-b02": true, // letters, digits, and - + "a": true, // single letter + "1": true, // single digits + strings.Repeat("a", 42): true, // max length + + // invalid cases + "a0@": false, // end invalid + "???": false, // invalid char + "-x": false, // start invalid + "x-": false, // invalid end + "000x??a": false, // invalid char + "-": false, // invalid start and end + ".": false, // invalid start and end + "----------": false, // invalid start and end + strings.Repeat("a", 43): false, // exceed max length + } + + for v, e := range cases { + So((validateVersionNumber(v) == nil), ShouldEqual, e) + } + }) + + Convey("Test version number default as expected", t, func() { + dftV, err := validateAndSetDefaultVersionNumber("") + So(err, ShouldBeNil) + match, err := regexp.MatchString("^v[0-9]+-[0-9]+$", dftV) + So(err, ShouldBeNil) + So(match, ShouldBeTrue) + t, err := time.Parse("v20060102-150405", dftV) + So(err, ShouldBeNil) + // should be now. In case ut runs slow, less than 1 min would be fine + So(time.Now().Sub(t).Minutes(), ShouldBeLessThanOrEqualTo, 1) + }) + + Convey("Test invalid version number returns empty v and and error", t, func() { + version, err := validateAndSetDefaultVersionNumber("-3") + So(err, ShouldNotBeNil) + So(version, ShouldEqual, "") + }) +} + +func TestCheckRepublish(t *testing.T) { + Convey("Test checkRepublish 1", t, func() { + patch := gomonkey.ApplyFunc(storage.GetFunctionVersion, + func(txn storage.Transaction, funcName, funcVer string) (storage.FunctionVersionValue, error) { + return storage.FunctionVersionValue{}, errmsg.KeyNotFoundError + }) + defer patch.Reset() + txn := &storage.Txn{} + funcVersion := storage.FunctionVersionValue{} + funcVersion.Function.Version = "1" + req := model.PublishRequest{} + err := checkRepublish(txn, funcVersion, req) + So(err, ShouldBeNil) + }) + Convey("Test checkRepublish 2", t, func() { + patch := gomonkey.ApplyFunc(storage.GetFunctionVersion, + func(txn storage.Transaction, funcName, funcVer string) (storage.FunctionVersionValue, error) { + return storage.FunctionVersionValue{}, errors.New("mock err") + }) + defer patch.Reset() + txn := &storage.Txn{} + funcVersion := storage.FunctionVersionValue{} + funcVersion.Function.Version = "1" + req := model.PublishRequest{} + err := checkRepublish(txn, funcVersion, req) + So(err, ShouldNotBeNil) + }) + Convey("Test checkRepublish 3", t, func() { + patch := gomonkey.ApplyFunc(storage.GetFunctionVersion, + func(txn storage.Transaction, funcName, funcVer string) (storage.FunctionVersionValue, error) { + funcVersion := storage.FunctionVersionValue{} + funcVersion.FunctionVersion.RevisionID = "1" + return funcVersion, nil + }) + defer patch.Reset() + txn := &storage.Txn{} + funcVersion := storage.FunctionVersionValue{} + funcVersion.Function.Version = "1" + req := model.PublishRequest{RevisionID: "1"} + err := checkRepublish(txn, funcVersion, req) + So(err, ShouldBeNil) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/query.go b/functionsystem/apps/meta_service/function_repo/service/query.go new file mode 100644 index 0000000000000000000000000000000000000000..471022d7fee07dae314259d9a95ef6ee7e2f6b4e --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/query.go @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "fmt" + "regexp" + "strings" + + "meta_service/common/constants" + "meta_service/common/logger/log" + "meta_service/common/snerror" + "meta_service/common/urnutils" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/utils" +) + +const ( + // the alias name can contain a maximum of 16 characters and a minimum of 2 characters, + // including lowercase letters, digits, and hyphens (-). + // it must start with a lowercase letter and end with a lowercase letter and digit + aliasNameRegex = "^[a-z][0-9a-z-]{0,14}[0-9a-z]$" + // version can contain a maximum of 16 characters and a minimum of 1 characters, + // including "$latest", "latest" or only digits. + // UPDATE: now version can contain digits(0-9), letters(a-z, A-Z), dots(.), hyphens(-), and underscores(_), + // and must start and end with a digit(0-9) or letter(a-z, A-Z), with less than 42 characters + versionRegex = "^[\\$]?latest$|^[0-9]{1,16}$|^[A-Za-z0-9][A-Za-z0-9._-]{0,40}[A-Za-z0-9]$|^[A-Za-z0-9]$" + // The function name can contain the function name with or without the service ID. + // Function name without serviceid must start with a lowercase letter and end with a lowercase letter or digit. + // Only lowercase letters, digits, and hyphens (-) or (@) are supported. The function name contains 1 to 128 characters + // Function name with serviceid add a serviceid based on the function name without serviceid. + // serviceid must start with 0- and end with a hyphen (-) or (@), and contain 1 to 16 digits or letters. + funcNameRegex = "(^0-[0-9a-zA-Z]{1,16}-([a-z0-9][a-z0-9-]{0,126}[a-z0-9]|[a-z])$)|" + + "(^[a-z][a-z0-9-]{0,126}[a-z0-9]$|^[a-z]$)|(^0@[0-9a-zA-Z]{1,16}@([a-z0-9][a-z0-9-]{0,126}[a-z0-9]|[a-z])$)" +) + +func parseTenantInfo(tenantInfo string) (string, string) { + var productID, tenantID string + index := strings.Index(tenantInfo, urnutils.TenantProductSplitStr) + if index > 0 { + // tenantInfo format is tenantID@productID + productID = tenantInfo[index+1:] + tenantID = tenantInfo[0:index] + return productID, tenantID + } + return "", tenantInfo +} + +func checkContextInfo(ctx server.Context, funcID string) error { + tenantInfo, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenantinfo, error: %s", err.Error()) + return err + } + + info, ok := utils.ParseTriggerInfoFromURN(funcID) + if !ok { + log.GetLogger().Errorf("invalid URN: %s", funcID) + return errmsg.NewParamError(fmt.Sprintf("invalid URN: %s", funcID)) + } + + productID, tenantID := parseTenantInfo(info.FunctionInfo.TenantInfo) + + if tenantInfo.BusinessID != info.FunctionInfo.BusinessID || + tenantInfo.ProductID != productID || tenantInfo.TenantID != tenantID { + log.GetLogger().Errorf("urnTenantInfo is not the same as tenantInfo from request header") + return snerror.NewWithFmtMsg(errmsg.InvalidQueryURN, "invalid URN: %s", funcID) + } + return nil +} + +func checkFunctionName(name string) error { + match, err := regexp.MatchString(funcNameRegex, name) + if err != nil { + return err + } + if !match { + return snerror.NewWithFmtMsg(errmsg.FunctionNameFormatErr, "invalid function name: %s", name) + } + return nil +} + +func checkVersion(version, qualifier string) (string, error) { + match, err := regexp.MatchString(versionRegex, version) + if err != nil { + log.GetLogger().Errorf("incorrect version format, error: %s", err.Error()) + return "", err + } + if match { + return version, nil + } + if qualifier == "" { + err = snerror.NewWithFmtMsg(errmsg.InvalidQueryURN, "invalid version: %s", version) + log.GetLogger().Errorf("invalid version: %s", version) + } else { + err = snerror.NewWithFmtMsg(errmsg.InvalidQualifier, "invalid qualifier: %s", qualifier) + log.GetLogger().Errorf("invalid qualifier: %s", qualifier) + } + return "", err +} + +func checkVerOrAlias(verOrAlias, qualifier string) (string, string, error) { + if verOrAlias == "" && qualifier == "" { + return "", "", nil + } + if verOrAlias == "" && qualifier != "" { + verOrAlias = qualifier + } + // check aliasName + match, err := regexp.MatchString(aliasNameRegex, verOrAlias) + if err == nil && match && verOrAlias != constants.DefaultLatestFaaSVersion { + return "", verOrAlias, nil + } + + // check version + version, err := checkVersion(verOrAlias, qualifier) + if err != nil { + log.GetLogger().Errorf("failed to check version, error: %s", err.Error()) + return "", "", err + } + return version, "", nil +} + +// CheckAndGetVerOrAlias check function version and alias name +func CheckAndGetVerOrAlias(funcID, qualifier string) (model.FunctionQueryInfo, error) { + var tmpAlias string + funcInfo := model.FunctionQueryInfo{} + + info, ok := utils.ParseTriggerInfoFromURN(funcID) + if !ok { + log.GetLogger().Errorf("failed to Parse urn, URN: %s", funcID) + return model.FunctionQueryInfo{}, errmsg.NewParamError(fmt.Sprintf("invalid URN: %s", funcID)) + } + + err := checkFunctionName(info.FunctionInfo.FunctionName) + if err != nil { + return model.FunctionQueryInfo{}, err + } + funcInfo.FunctionName = info.FunctionInfo.FunctionName + tmpAlias = info.VerOrAlias + + if tmpAlias != "" && qualifier != "" && qualifier != tmpAlias { + log.GetLogger().Errorf( + "queryInfo is not the same as qualifier, urn is %s, qualifier is %s", funcID, qualifier) + return model.FunctionQueryInfo{}, snerror.NewWithFmtMsg(errmsg.InvalidQueryURN, "invalid URN: %s", funcID) + } + + funcInfo.FunctionVersion, funcInfo.AliasName, err = checkVerOrAlias(tmpAlias, qualifier) + if err != nil { + log.GetLogger().Errorf("failed to get version or alias, error: %s", err.Error()) + return model.FunctionQueryInfo{}, err + } + return funcInfo, nil +} + +func parseFunctionInfoByFuncID( + ctx server.Context, funcID string, qualifier string, +) (model.FunctionQueryInfo, error) { + if err := checkContextInfo(ctx, funcID); err != nil { + log.GetLogger().Errorf("failed to check functionID, error: %s", err.Error()) + return model.FunctionQueryInfo{}, err + } + funcInfo, err := CheckAndGetVerOrAlias(funcID, qualifier) + if err != nil { + log.GetLogger().Errorf("failed to check version or aliasname, error: %s", err.Error()) + return model.FunctionQueryInfo{}, err + } + return funcInfo, nil +} + +func parseFunctionInfoByName(funcName string, qualifier string) (model.FunctionQueryInfo, error) { + info := model.FunctionQueryInfo{} + info.FunctionName = funcName + + if qualifier == "" { + return info, nil + } + + match, err := regexp.MatchString(aliasNameRegex, qualifier) + if err == nil && match && qualifier != utils.GetDefaultFaaSVersion() { + info.AliasName = qualifier + return info, nil + } + + if match, err = regexp.MatchString(versionRegex, qualifier); err == nil && match { + info.FunctionVersion = qualifier + return info, nil + } + err = snerror.NewWithFmtMsg(errmsg.InvalidQualifier, "invalid qualifier: %s", qualifier) + log.GetLogger().Errorf("invalid qualifier: %s", qualifier) + return model.FunctionQueryInfo{}, err +} + +// ParseFunctionInfo gets function info +func ParseFunctionInfo(ctx server.Context, queryInfo, qualifier string) (model.FunctionQueryInfo, error) { + var info model.FunctionQueryInfo + if queryInfo == "" { + log.GetLogger().Errorf("empty functionid") + return model.FunctionQueryInfo{}, snerror.NewWithFmtMsg(errmsg.InvalidQueryURN, "empty functionid") + } + + err := checkFunctionName(queryInfo) + if err != nil { + info, err = parseFunctionInfoByFuncID(ctx, queryInfo, qualifier) + if err != nil { + log.GetLogger().Errorf("failed to get function info by functionID, err: %s", err.Error()) + return model.FunctionQueryInfo{}, err + } + return info, nil + } + info, err = parseFunctionInfoByName(queryInfo, qualifier) + if err != nil { + log.GetLogger().Errorf("failed to get function info by name, err: %s", err.Error()) + return model.FunctionQueryInfo{}, err + } + return info, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/service/query_test.go b/functionsystem/apps/meta_service/function_repo/service/query_test.go new file mode 100644 index 0000000000000000000000000000000000000000..8412cd80ce15045bfed7d83fc53672aaa76e87a7 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/query_test.go @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "testing" + + "meta_service/function_repo/server" + "meta_service/function_repo/test/fakecontext" +) + +func TestCheckContextInfo(t *testing.T) { + ctx := fakecontext.NewContext() + info := server.TenantInfo{ + BusinessID: "businessid", + TenantID: "tenantid", + ProductID: "productid", + } + ctx.InitTenantInfo(info) + + funcID := "sn:cn:businessid:tenantid@productid:function:test:$latest" + err := checkContextInfo(ctx, funcID) + if err != nil { + t.Errorf("tenantInfo is empty") + } + + funcID = "sn:cn:businessid:tenantid@productid111:function:test:$latest" + err = checkContextInfo(ctx, funcID) + if err == nil { + t.Errorf("tenantInfo is empty") + } + + funcID = "sn:cn:businessid:tenantid@:function:test:$latest" + err = checkContextInfo(ctx, funcID) + if err == nil { + t.Errorf("tenantInfo is empty") + } + + funcID = "sn:cn:businessid:tenantid:function:test:$latest" + err = checkContextInfo(ctx, funcID) + if err == nil { + t.Errorf("tenantInfo is empty") + } +} + +func TestCheckAndGetVerOrAlias(t *testing.T) { + funcID := "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test:$latest" + qualifier := "123" + info, err := CheckAndGetVerOrAlias(funcID, qualifier) + if err == nil { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test:$latest" + qualifier = "" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + if !(err == nil && info.FunctionName == "test" && info.FunctionVersion == "$latest" && info.AliasName == "") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test:12354515" + qualifier = "" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + if !(err == nil && info.FunctionName == "test" && info.FunctionVersion == "12354515" && info.AliasName == "") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test:12354515ZSDSSas" + qualifier = "" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + // yes, now 12354515ZSDSSas can be an version (start with digits, not meet alias requirement) + if err != nil { + t.Errorf("funcID is error, funcID is %s", funcID) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test:myalias" + qualifier = "" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + if !(err == nil && info.FunctionName == "test" && info.FunctionVersion == "" && info.AliasName == "myalias") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test" + qualifier = "myalias12312" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + if !(err == nil && info.FunctionName == "test" && info.FunctionVersion == "" && info.AliasName == "myalias12312") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test" + qualifier = "myalias1233ZZZADS" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + // yes, now myalias1233ZZZADS can be an version (len 17, longer than alias requirement) + if err != nil { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test" + qualifier = "myalias1233ZZZADS000000000000000000000aaa00" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + if err == nil { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test" + qualifier = "123545" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + if !(err == nil && info.FunctionName == "test" && info.FunctionVersion == "123545" && info.AliasName == "") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcID = "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:test" + qualifier = "" + info, err = CheckAndGetVerOrAlias(funcID, qualifier) + if !(err == nil && info.FunctionName == "test" && info.FunctionVersion == "" && info.AliasName == "") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } +} + +func TestGetFunctionQueryInfoByFuncID(t *testing.T) { + ctx := fakecontext.NewContext() + info := server.TenantInfo{ + BusinessID: "businessid", + TenantID: "tenantid", + ProductID: "productid", + } + ctx.InitTenantInfo(info) + funcID := "sn:cn:businessid:tenantid@productid:function:test:$latest" + qualifier := "" + funcInfo, err := parseFunctionInfoByFuncID(ctx, funcID, qualifier) + if !(err == nil && funcInfo.FunctionName == "test" && funcInfo.FunctionVersion == "$latest" && funcInfo.AliasName == "") { + t.Errorf("tenantInfo is err") + } + + funcID = "sn:cn:businessid:tenantid@productid111:function:test:$latest" + qualifier = "" + funcInfo, err = parseFunctionInfoByFuncID(ctx, funcID, qualifier) + if err == nil { + t.Errorf("tenantInfo is empty") + } + + funcID = "sn:cn:businessid:tenantid@productid:function:test:12354515" + qualifier = "" + funcInfo, err = parseFunctionInfoByFuncID(ctx, funcID, qualifier) + if !(err == nil && funcInfo.FunctionName == "test" && funcInfo.FunctionVersion == "12354515" && funcInfo.AliasName == "") { + t.Errorf("funcID is err") + } + + funcID = "sn:cn:businessid:tenantid@productid:function:test:serses21323" + qualifier = "" + funcInfo, err = parseFunctionInfoByFuncID(ctx, funcID, qualifier) + if !(err == nil && funcInfo.FunctionName == "test" && funcInfo.FunctionVersion == "" && funcInfo.AliasName == "serses21323") { + t.Errorf("funcID is err") + } + + funcID = "sn:cn:businessid:tenantid@productid:function:test:serses21323" + qualifier = "123" + funcInfo, err = parseFunctionInfoByFuncID(ctx, funcID, qualifier) + if err == nil { + t.Errorf("qualifier is empty") + } +} + +func TestGetFunctionQueryInfoByName(t *testing.T) { + funcName := "0-cpptest-helloworld" + qualifier := "$latest" + info, err := parseFunctionInfoByName(funcName, qualifier) + if !(err == nil && info.FunctionName == "0-cpptest-helloworld" && info.FunctionVersion == "$latest" && info.AliasName == "") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcName = "0-cpptest-helloworld" + qualifier = "1223323" + info, err = parseFunctionInfoByName(funcName, qualifier) + if !(err == nil && info.FunctionName == "0-cpptest-helloworld" && info.FunctionVersion == "1223323" && info.AliasName == "") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcName = "0-cpptest-helloworld" + qualifier = "sdsddwae112" + info, err = parseFunctionInfoByName(funcName, qualifier) + if !(err == nil && info.FunctionName == "0-cpptest-helloworld" && info.FunctionVersion == "" && info.AliasName == "sdsddwae112") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcName = "0-cpptest-helloworld" + qualifier = "" + info, err = parseFunctionInfoByName(funcName, qualifier) + if !(err == nil && info.FunctionName == "0-cpptest-helloworld" && info.FunctionVersion == "" && info.AliasName == "") { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } + + funcName = "0-cpptest-helloworld" + qualifier = "ZSDW123xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + info, err = parseFunctionInfoByName(funcName, qualifier) + if err == nil { + t.Errorf("qualifier is error, qualifier is %s", qualifier) + } +} + +func TestGetFunctionQueryInfo(t *testing.T) { + ctx := fakecontext.NewContext() + tenInfo := server.TenantInfo{ + BusinessID: "businessid", + TenantID: "tenantid", + ProductID: "productid", + } + ctx.InitTenantInfo(tenInfo) + + funcID := "sn:cn:businessid:tenantid@productid:function:test:$latest" + qualifier := "" + info, err := ParseFunctionInfo(ctx, funcID, qualifier) + if !(err == nil && info.FunctionName == "test" && info.FunctionVersion == "$latest" && info.AliasName == "") { + t.Errorf("tenantInfo is err") + } + + funcID = "sn:cn:businessid:tenantid@productid11:function:test:$latest" + qualifier = "" + info, err = ParseFunctionInfo(ctx, funcID, qualifier) + if err == nil { + t.Errorf("tenantInfo is err") + } + + funcID = "0-cpptest-helloworld" + qualifier = "123" + info, err = ParseFunctionInfo(ctx, funcID, qualifier) + if !(err == nil && info.FunctionName == "0-cpptest-helloworld" && info.FunctionVersion == "123" && info.AliasName == "") { + t.Errorf("tenantInfo is err") + } + + funcID = "0-cpptestXX-hew001-001" + qualifier = "123" + info, err = ParseFunctionInfo(ctx, funcID, qualifier) + if !(err == nil && info.FunctionName == "0-cpptestXX-hew001-001" && info.FunctionVersion == "123" && info.AliasName == "") { + t.Errorf("tenantInfo is err") + } + + funcID = "cpptest-helloworld" + qualifier = "" + info, err = ParseFunctionInfo(ctx, funcID, qualifier) + if !(err == nil && info.FunctionName == "cpptest-helloworld" && info.FunctionVersion == "" && info.AliasName == "") { + t.Errorf("tenantInfo is err") + } +} diff --git a/functionsystem/apps/meta_service/function_repo/service/reserved_instance.go b/functionsystem/apps/meta_service/function_repo/service/reserved_instance.go new file mode 100644 index 0000000000000000000000000000000000000000..ff9927d8da76c46b18d3545e7c0188bcef9b958b --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/reserved_instance.go @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "meta_service/common/logger/log" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/utils/constants" +) + +func validateInstanceConfig(ctx server.Context, insConfig model.InstanceConfig, info model.ReserveInsBaseInfo, + isUpdate bool, +) error { + maxReservedInstanceLabels := int64(config.RepoCfg.FunctionCfg.InstanceLabelMax) + if insConfig.MaxInstance < insConfig.MinInstance { + return errmsg.NewParamError("maxInstance %d is less than minInstance %d") + } + if isUpdate { + _, getErr := storage.GetReserveInstanceConfig(ctx, insConfig.ClusterID, info) + if getErr != nil { + return errmsg.NewParamError("failed to find reserve instance config for update") + } + } else { + cnt, err := storage.CountReserveInstanceLabels(ctx, insConfig.ClusterID, info) + if err != nil { + return err + } + if cnt >= maxReservedInstanceLabels { + log.GetLogger().Errorf("label %s for function %s version %s, already reach %d", info.InstanceLabel, + info.FuncName, info.Version, maxReservedInstanceLabels) + return errmsg.NewParamError("label count reached max(%d)", maxReservedInstanceLabels) + } + } + return nil +} + +// CreateReserveInstance create reserve instance +func CreateReserveInstance(ctx server.Context, req model.CreateReserveInsRequest, + isUpdate bool) (model.CreateReserveInsResponse, + error, +) { + txn := storage.GetTxnByKind(ctx, "faas") + defer txn.Cancel() + createResp := model.CreateReserveInsResponse{} + createResp.Code = 0 + _, err := storage.GetFunctionByFunctionNameAndVersion(ctx, req.FuncName, req.Version, constants.Faas) + if err != nil { + log.GetLogger().Errorf("%s|failed to get function %s by version %s", req.TraceID, req.FuncName, req.Version) + return createResp, err + } + info := model.ReserveInsBaseInfo{ + FuncName: req.FuncName, + Version: req.Version, TenantID: req.TenantID, InstanceLabel: req.InstanceLabel, + } + clusterSet := make(map[string]bool) + for _, insConfig := range req.InstanceConfigInfos { + if insConfig.ClusterID == "" { + insConfig.ClusterID = constants.DefaultClusterID + } + if _, exists := config.RepoCfg.ClusterID[insConfig.ClusterID]; !exists { + return createResp, errmsg.NewParamError("clusterID %s is not found", insConfig.ClusterID) + } + if _, ok := clusterSet[insConfig.ClusterID]; ok { + return createResp, errmsg.NewParamError("clusterID %s is repeated", insConfig.ClusterID) + } else { + clusterSet[insConfig.ClusterID] = true + } + if err := validateInstanceConfig(ctx, insConfig, info, isUpdate); err != nil { + return createResp, err + } + err = storage.CreateOrUpdateReserveInstanceConfig(txn, info, insConfig) + if err != nil { + log.GetLogger().Errorf("%s|failed to create reserve instance,cluster:%s,functionName:%s,version:%s, "+ + "err:%s", req.TraceID, insConfig.ClusterID, info.FuncName, info.Version, err.Error()) + return createResp, err + } + } + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to create reserve instance when committing: %s", err.Error()) + return createResp, err + } + var instanceConfigs []model.InstanceConfig + for cluster := range config.RepoCfg.ClusterID { + instanceConfig, err := storage.GetReserveInstanceConfig(ctx, cluster, info) + if err != nil { + log.GetLogger().Warnf("%s|failed to get instance config cluster:%s", req.TraceID, cluster) + } else { + instanceConfigs = append(instanceConfigs, instanceConfig) + } + } + createResp.ReserveInsBaseInfo = info + createResp.InstanceConfigInfos = instanceConfigs + return createResp, nil +} + +// DeleteReserveInstance delete reserve instance +func DeleteReserveInstance(ctx server.Context, req model.DeleteReserveInsRequest) (model.DeleteReserveInsResponse, + error, +) { + txn := storage.GetTxnByKind(ctx, constants.Faas) + defer txn.Cancel() + info := model.ReserveInsBaseInfo{ + FuncName: req.FuncName, + Version: req.Version, TenantID: req.TenantID, InstanceLabel: req.InstanceLabel, + } + _, err := storage.GetFunctionByFunctionNameAndVersion(ctx, req.FuncName, req.Version, constants.Faas) + if err != nil { + log.GetLogger().Errorf("%s|failed to get function %s by version %s", req.TraceID, req.FuncName, req.Version) + return model.DeleteReserveInsResponse{}, err + } + for cluster := range config.RepoCfg.ClusterID { + storage.DeleteReserveInstanceConfig(txn, cluster, info) + } + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to delete reserve instance when committing: %s", err.Error()) + return model.DeleteReserveInsResponse{}, err + } + return model.DeleteReserveInsResponse{}, nil +} + +// GetReserveInstanceConfigs get reserved instance config +func GetReserveInstanceConfigs(ctx server.Context, req model.GetReserveInsRequest) (model.GetReserveInsResponse, + error, +) { + txn := storage.GetTxnByKind(ctx, constants.Faas) + defer txn.Cancel() + info := model.ReserveInsBaseInfo{ + FuncName: req.FuncName, Version: req.Version, TenantID: req.TenantID, + InstanceLabel: "", + } + _, err := storage.GetFunctionByFunctionNameAndVersion(ctx, req.FuncName, req.Version, constants.Faas) + if err != nil { + log.GetLogger().Errorf("failed to get function %s by version %s", req.FuncName, req.Version) + return model.GetReserveInsResponse{}, err + } + resp := model.GetReserveInsResponse{} + resp.GetReserveInsResults = make([]model.GetReserveInsResult, 0) + labelIndex := make(map[string]int) + cnt := 0 + for cluster := range config.RepoCfg.ClusterID { + results, err := storage.GetReserveInstanceConfigList(ctx, cluster, info) + if err != nil { + return model.GetReserveInsResponse{}, err + } + for _, val := range results { + index, ok := labelIndex[val.InstanceLabel] + if ok { + resp.GetReserveInsResults[index].InstanceConfigInfos = append(resp.GetReserveInsResults[index]. + InstanceConfigInfos, val.InstanceConfig) + } else { + insRes := model.GetReserveInsResult{ + InstanceLabel: val.InstanceLabel, + InstanceConfigInfos: make([]model.InstanceConfig, 0), + } + insRes.InstanceConfigInfos = append(insRes.InstanceConfigInfos, val.InstanceConfig) + resp.GetReserveInsResults = append(resp.GetReserveInsResults, insRes) + labelIndex[val.InstanceLabel] = cnt + cnt++ + } + } + } + resp.Total = len(resp.GetReserveInsResults) + return resp, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/service/reserved_instance_test.go b/functionsystem/apps/meta_service/function_repo/service/reserved_instance_test.go new file mode 100644 index 0000000000000000000000000000000000000000..904132e75b360dc27f6a4f6a98f1cb5237ea415e --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/reserved_instance_test.go @@ -0,0 +1,42 @@ +package service + +import ( + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/assertions/should" + . "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/model" + "meta_service/function_repo/storage" + "functionsystem/pkg/meta_service/server" + "functionsystem/pkg/meta_service/test/fakecontext" +) + +func Test_createReserveInstance(t *testing.T) { + + Convey("Test Create Reserve Instance ClusterID not exist", t, func() { + patches := gomonkey.ApplyFunc(storage.GetFunctionByFunctionNameAndVersion, func(ctx server.Context, name string, + version string, kind string) (storage.FunctionVersionValue, error) { + funcVer := storage.FunctionVersionValue{} + return funcVer, nil + }) + defer patches.Reset() + ctx := fakecontext.NewMockContext() + req := model.CreateReserveInsRequest{} + req.FuncName = "" + req.Version = "latest" + req.TenantID = "0" + req.InstanceLabel = "label001" + req.InstanceConfigInfos = []model.InstanceConfig{ + { + ClusterID: "cluster-001", + MinInstance: 1, + MaxInstance: 100, + }, + } + _, err := CreateReserveInstance(ctx, req, false) + ShouldNotBeNil(err) + So(err.Error(), should.ContainSubstring, "clusterID cluster-001 is not found") + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/service.go b/functionsystem/apps/meta_service/function_repo/service/service.go new file mode 100644 index 0000000000000000000000000000000000000000..1ebabe7abe837dad1fa53a4158f386bfac5ec3e6 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/service.go @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "meta_service/common/constants" + "meta_service/common/urnutils" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" +) + +// GetServiceID gets function names by service id +func GetServiceID(ctx server.Context, id, kind string) (model.ServiceGetResponse, error) { + resp := model.ServiceGetResponse{} + newID := buildNewServicePrefix(id, kind) + names, err := storage.GetFunctionNamesByServiceID(ctx, newID, kind) + if err != nil { + return model.ServiceGetResponse{}, err + } + resp.Total = len(names) + resp.FunctionNames = names + return resp, nil +} + +func buildNewServicePrefix(id, kind string) (perfix string) { + if kind == constants.Faas { + if id == "" { + return urnutils.FaaSServicePrefix + } + return urnutils.FaaSServicePrefix + id + urnutils.DefaultFaaSSeparator + } + if id == "" { + return urnutils.ServicePrefix + } + return urnutils.ServicePrefix + id + urnutils.DefaultSeparator +} diff --git a/functionsystem/apps/meta_service/function_repo/service/service_test.go b/functionsystem/apps/meta_service/function_repo/service/service_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1dec4d0458dfbc51fa3cfdd03cf377e1e05d28f9 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/service_test.go @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "meta_service/common/urnutils" + "meta_service/function_repo/initialize" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/test" + "meta_service/function_repo/test/fakecontext" +) + +func TestBuildNewServicePreix(t *testing.T) { + id := "hello" + res := buildNewServicePrefix(id, "") + str := urnutils.ServicePrefix + id + urnutils.DefaultSeparator + if !(res == str) { + t.Errorf("get newServicePreix failed, res is %s", res) + } + id = "" + res = buildNewServicePrefix(id, "") + if !(res == urnutils.ServicePrefix) { + t.Errorf("get newServicePreix failed, res is %s", res) + } +} + +// TestMain init router test +func TestMain(m *testing.M) { + if !initialize.Initialize(test.ConfigPath, "/home/sn/config/log.json") { + fmt.Println("failed to initialize") + os.Exit(1) + } + test.ResetETCD() + result := m.Run() + test.ResetETCD() + os.Exit(result) +} + +func TestGetServiceID(t *testing.T) { + type args struct { + ctx server.Context + } + + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test get service id", + args{fakecontext.NewMockContext()}, model.FunctionVersion{}, false}, + } + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + _, err := GetServiceID(tt.args.ctx, "func-id", "") + assert.Equal(t, err, nil) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/trigger.go b/functionsystem/apps/meta_service/function_repo/service/trigger.go new file mode 100644 index 0000000000000000000000000000000000000000..aed1adce75fdb3f2e3d81adc5ec89442d15a2350 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/trigger.go @@ -0,0 +1,518 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "time" + + "meta_service/common/logger/log" + "meta_service/common/snerror" + "meta_service/common/uuid" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/storage/publish" + "meta_service/function_repo/utils" + "meta_service/function_repo/utils/constants" +) + +const ( + // triggerMaxCount is the maximum of trigger + triggerMaxCount = 32 + appSecretMaxLen = 256 +) + +var triggerSpecTable = map[string]struct { + Create func(txn *storage.Txn, raw interface{}, funcInfo model.FunctionQueryInfo, funcID, triggerID string, + ) (storage.TriggerSpec, error) + + Update func(raw interface{}) (storage.TriggerSpec, string, error) +}{ + model.HTTPType: { + Create: createHTTPTriggerSpec, + }, +} + +func checkFunctionNameAndVersion(txn *storage.Txn, funcName string, version string) error { + _, err := storage.GetFunctionVersion(txn, funcName, version) + if err != nil { + if err == errmsg.KeyNotFoundError { + log.GetLogger().Errorf("this version [%s] of function [%s] does not exist", funcName, version) + return errmsg.New(errmsg.FunctionVersionNotFound, funcName, version) + } + log.GetLogger().Errorf("failed to get function version, error: %s", err.Error()) + return err + } + return nil +} + +func checkFuncInfo(txn *storage.Txn, info model.FunctionQueryInfo) error { + if err := checkFunctionNameAndVersion(txn, info.FunctionName, constants.DefaultVersion); err != nil { + log.GetLogger().Errorf("failed to check function name and version, error: %s", err.Error()) + return err + } + if info.AliasName != "" { + exist, err := storage.AliasNameExist(txn, info.FunctionName, info.AliasName) + if err != nil { + log.GetLogger().Errorf("failed to get aliasname, error: %s", err.Error()) + return err + } + if !exist { + log.GetLogger().Errorf( + "this alias name [%s] of function [%s] does not exist", info.FunctionName, info.AliasName) + return errmsg.New(errmsg.AliasNameNotFound, utils.RemoveServiceID(info.FunctionName), info.AliasName) + } + return nil + } + if err := checkFunctionNameAndVersion(txn, info.FunctionName, info.FunctionVersion); err != nil { + log.GetLogger().Errorf("failed to check function name and version, error: %s", err.Error()) + return err + } + return nil +} + +func getVerOrAlias(i model.FunctionQueryInfo) string { + var verOrAlias string + if i.AliasName != "" { + verOrAlias = i.AliasName + } else { + verOrAlias = i.FunctionVersion + } + return verOrAlias +} + +func createTrigger(txn *storage.Txn, req model.TriggerCreateRequest, funcInfo model.FunctionQueryInfo, +) (storage.TriggerValue, error) { + info := storage.TriggerValue{} + info.TriggerID = uuid.New().String() + info.FuncName = funcInfo.FunctionName + info.TriggerType = req.TriggerType + info.RevisionID = utils.GetUTCRevisionID() + info.CreateTime = time.Now() + info.UpdateTime = time.Now() + + var err error + if t, ok := triggerSpecTable[req.TriggerType]; ok { + info.EtcdSpec, err = t.Create(txn, req.Spec, funcInfo, req.FuncID, info.TriggerID) + } else { + err = errmsg.NewParamError("invalid triggerType: %s", info.TriggerType) + } + if err != nil { + log.GetLogger().Errorf("failed to convert %s trigger spec, error: %s", info.TriggerType, err.Error()) + return storage.TriggerValue{}, err + } + return info, nil +} + +func buildModelTriggerSpec(info storage.TriggerValue, funcID string) (spec interface{}, err error) { + triggerSpec := model.TriggerSpec{ + FuncID: funcID, + TriggerID: info.TriggerID, + TriggerType: info.TriggerType, + } + spec, err = info.EtcdSpec.BuildModelSpec(triggerSpec) + if err != nil { + log.GetLogger().Errorf("failed to convert spec, error: %s", err.Error()) + } + return +} + +func buildTriggerResponse(ctx server.Context, triggerInfo storage.TriggerValue, funcName, verOrAlias string, +) (model.TriggerInfo, error) { + resp := model.TriggerInfo{} + tenantInfo, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("invalid tenantInfo, error: %s", err.Error()) + return model.TriggerInfo{}, err + } + funcID := utils.BuildTriggerURN(tenantInfo, funcName, verOrAlias) + resp.FuncID = funcID + resp.TriggerType = triggerInfo.TriggerType + resp.TriggerID = triggerInfo.TriggerID + resp.RevisionID = triggerInfo.RevisionID + resp.CreateTime = triggerInfo.CreateTime + resp.UpdateTime = triggerInfo.UpdateTime + resp.Spec, err = buildModelTriggerSpec(triggerInfo, funcID) + if err != nil { + log.GetLogger().Errorf("failed to convert spec, error: %s", err.Error()) + return model.TriggerInfo{}, err + } + return resp, nil +} + +func updateTrigger(req model.TriggerUpdateRequest) (storage.TriggerFunctionIndexValue, storage.TriggerValue, error) { + info := storage.TriggerValue{} + info.TriggerID = req.TriggerID + info.TriggerType = req.TriggerType + info.RevisionID = req.RevisionID + + var ( + err error + funcID string + ) + if t, ok := triggerSpecTable[req.TriggerType]; ok && t.Update != nil { + info.EtcdSpec, funcID, err = t.Update(req.Spec) + } else if t.Update == nil { + err = errmsg.NewParamError("update function not found, triggerType: %s", info.TriggerType) + } else { + err = errmsg.NewParamError("invalid triggerType: %s", info.TriggerType) + } + if err != nil { + log.GetLogger().Errorf("failed to convert %s trigger spec, error: %s", info.TriggerType, err.Error()) + return storage.TriggerFunctionIndexValue{}, storage.TriggerValue{}, err + } + funcInfo, err := CheckAndGetVerOrAlias(funcID, "") + if err != nil { + log.GetLogger().Errorf("failed to check version or alias, error: %s", err.Error()) + return storage.TriggerFunctionIndexValue{}, storage.TriggerValue{}, err + } + info.FuncName = funcInfo.FunctionName + tfInfo := storage.TriggerFunctionIndexValue{} + tfInfo.FunctionName = funcInfo.FunctionName + tfInfo.VersionOrAlias = getVerOrAlias(funcInfo) + return tfInfo, info, nil +} + +func storeCreateTrigger( + txn *storage.Txn, triggerInfo storage.TriggerValue, funcInfo model.FunctionQueryInfo, funcID string, +) error { + spec, err := buildModelTriggerSpec(triggerInfo, funcID) + if err != nil { + log.GetLogger().Errorf("failed to convert spec, error: %s", err.Error()) + return err + } + + verOrAlias := getVerOrAlias(funcInfo) + if verOrAlias == "" { + log.GetLogger().Errorf("empty version and alias") + return snerror.NewWithFmtMsg(errmsg.EmptyAliasAndVersion, "empty version and alias") + } + + err = storage.SaveTriggerInfo(txn, funcInfo.FunctionName, verOrAlias, triggerInfo.TriggerID, triggerInfo) + if err != nil { + log.GetLogger().Errorf("failed to save trigger, error: %s", err.Error()) + return err + } + if err := publish.AddTrigger(txn, funcInfo.FunctionName, verOrAlias, triggerInfo, spec); err != nil { + log.GetLogger().Errorf("failed to publish trigger, error: %s", err.Error()) + return err + } + + return nil +} + +func storeUpdateTrigger( + txn *storage.Txn, info storage.TriggerValue, funcInfo storage.TriggerFunctionIndexValue, funcID string, +) error { + triggerInfo, err := storage.GetTriggerInfo(txn, funcInfo.FunctionName, funcInfo.VersionOrAlias, info.TriggerID) + if err != nil { + if err == errmsg.KeyNotFoundError { + log.GetLogger().Errorf("this triggerid [%s] does not exist", info.TriggerID) + return errmsg.New(errmsg.TriggerIDNotFound, info.TriggerID) + } + log.GetLogger().Errorf("failed to get trigger, error: %s", err.Error()) + return err + } + + if triggerInfo.RevisionID != info.RevisionID { + log.GetLogger().Errorf("revisionId is not the same as latest versions") + return snerror.NewWithFmtMsg(errmsg.RevisionIDError, "different revisionId") + } + info.RevisionID = utils.GetUTCRevisionID() + info.FuncName = triggerInfo.FuncName + info.CreateTime = triggerInfo.CreateTime + info.UpdateTime = time.Now() + spec, err := buildModelTriggerSpec(info, funcID) + if err != nil { + log.GetLogger().Errorf("failed to convert request, error: %s", err.Error()) + return err + } + + if funcInfo.VersionOrAlias == "" { + log.GetLogger().Errorf("empty version and alias") + return snerror.NewWithFmtMsg(errmsg.EmptyAliasAndVersion, "empty version and alias") + } + + err = storage.SaveTriggerInfo(txn, funcInfo.FunctionName, funcInfo.VersionOrAlias, info.TriggerID, info) + if err != nil { + log.GetLogger().Errorf("failed to save trigger, error: %s", err.Error()) + return err + } + if err := publish.AddTrigger(txn, funcInfo.FunctionName, funcInfo.VersionOrAlias, info, spec); err != nil { + log.GetLogger().Errorf("failed to publish trigger, error: %s", err.Error()) + return err + } + return nil +} + +// CreateTrigger creates trigger info +func CreateTrigger(ctx server.Context, req model.TriggerCreateRequest) (model.TriggerResponse, error) { + resp := model.TriggerResponse{} + funcID := req.FuncID + funcInfo, err := ParseFunctionInfo(ctx, funcID, "") + if err != nil { + log.GetLogger().Errorf("failed to get function info, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + if err = checkFuncInfo(txn, funcInfo); err != nil { + log.GetLogger().Errorf("failed to check function info, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + verOrAlias := getVerOrAlias(funcInfo) + triggerList, err := storage.GetTriggerByFunctionNameVersion(txn, funcInfo.FunctionName, "") + if err != nil { + log.GetLogger().Errorf("failed to get trigger, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + if len(triggerList) >= triggerMaxCount { + log.GetLogger().Errorf("trigger count reaches the maximum value") + return model.TriggerResponse{}, errmsg.New(errmsg.TriggerNumOutOfLimit, triggerMaxCount) + } + + triggerInfo, err := createTrigger(txn, req, funcInfo) + if err != nil { + log.GetLogger().Errorf("failed to change request to trigger info, error: %s", err.Error()) + e, ok := err.(snerror.SNError) + if !ok { + e = errmsg.NewParamError("invalid spec in request") + } + return model.TriggerResponse{}, e + } + + if err := storeCreateTrigger(txn, triggerInfo, funcInfo, funcID); err != nil { + return model.TriggerResponse{}, err + } + + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + resp.TriggerInfo, err = buildTriggerResponse(ctx, triggerInfo, funcInfo.FunctionName, verOrAlias) + if err != nil { + log.GetLogger().Errorf("failed to convert trigger info to response, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + return resp, nil +} + +// GetTrigger gets trigger info by trigger id +func GetTrigger(ctx server.Context, tid string) (model.TriggerResponse, error) { + var triggerInfo storage.TriggerValue + resp := model.TriggerResponse{} + + funcInfo, err := storage.GetFunctionInfoByTriggerID(ctx, tid) + if err != nil { + log.GetLogger().Errorf("failed to get function info by triggerid from storage, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + triggerInfo, err = storage.GetTriggerInfoByTriggerID(ctx, funcInfo.FunctionName, funcInfo.VersionOrAlias, tid) + if err != nil { + log.GetLogger().Errorf("failed to get trigger info by triggerid from storage, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + resp.TriggerInfo, err = buildTriggerResponse(ctx, triggerInfo, funcInfo.FunctionName, funcInfo.VersionOrAlias) + if err != nil { + log.GetLogger().Errorf("failed to convert trigger info to response, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + return resp, nil +} + +// GetTriggerList get trigger infos by function id +func GetTriggerList(ctx server.Context, pageIndex, pageSize int, fid string) (model.TriggerListGetResponse, error) { + resp := model.TriggerListGetResponse{} + funcInfo, err := ParseFunctionInfo(ctx, fid, "") + if err != nil { + log.GetLogger().Errorf("failed to get function info, error: %s", err.Error()) + return model.TriggerListGetResponse{}, err + } + + verOrAlias := getVerOrAlias(funcInfo) + triggerList, err := storage.GetTriggerInfoList(ctx, funcInfo.FunctionName, verOrAlias, pageIndex, pageSize) + if err != nil { + if err == errmsg.KeyNotFoundError { + resp.Count = 0 + return resp, nil + } + log.GetLogger().Errorf("failed to get trigger list from storage, error: %s", err.Error()) + return model.TriggerListGetResponse{}, err + } + + resp.Count = len(triggerList) + resp.TriggerList = make([]model.TriggerInfo, resp.Count, resp.Count) + for index, triggerInfo := range triggerList { + respTrigger, err := buildTriggerResponse(ctx, triggerInfo, funcInfo.FunctionName, verOrAlias) + if err != nil { + log.GetLogger().Errorf("failed to convert triggerInfo to response, error: %s", err.Error()) + return model.TriggerListGetResponse{}, err + } + resp.TriggerList[index] = respTrigger + } + return resp, nil +} + +// DeleteTriggerByID deletes trigger info by trigger id +func DeleteTriggerByID(ctx server.Context, tid string) error { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + funcInfo, err := storage.GetFunctionInfo(txn, tid) + if err != nil { + if err == errmsg.KeyNotFoundError { + log.GetLogger().Errorf("this triggerid [%s] does not exist", tid) + return errmsg.New(errmsg.FunctionNotFound, tid) + } + log.GetLogger().Errorf("failed to get function, error: %s", err.Error()) + return err + } + + triggerInfo, err := storage.GetTriggerInfo(txn, funcInfo.FunctionName, funcInfo.VersionOrAlias, tid) + if err != nil { + log.GetLogger().Errorf("failed to get trigger, error: %s", err.Error()) + return err + } + if err := storage.DeleteTrigger(txn, funcInfo.FunctionName, funcInfo.VersionOrAlias, tid); err != nil { + log.GetLogger().Errorf("failed to delete trigger, error: %s", err.Error()) + return err + } + if err := publish.DeleteTrigger(txn, funcInfo.FunctionName, funcInfo.VersionOrAlias, triggerInfo); err != nil { + log.GetLogger().Errorf("failed to delete publishings, error: %s", err.Error()) + return err + } + + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteTriggerByFuncID deletes trigger info by function id +func DeleteTriggerByFuncID(ctx server.Context, fid string) error { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + funcInfo, err := ParseFunctionInfo(ctx, fid, "") + if err != nil { + log.GetLogger().Errorf("failed to get function info, error: %s", err.Error()) + return err + } + + verOrAlias := getVerOrAlias(funcInfo) + if err := DeleteTriggerByFuncNameVersion(txn, funcInfo.FunctionName, verOrAlias); err != nil { + log.GetLogger().Errorf("failed to delete publishings, error: %s", err.Error()) + return err + } + if err := txn.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteTriggerByFuncName deletes trigger all info by function name +func DeleteTriggerByFuncName(txn storage.Transaction, funcName string) error { + if err := publish.DeleteTriggerByFuncName(txn, funcName); err != nil { + log.GetLogger().Errorf("failed to delete publishings, error: %s", err.Error()) + return err + } + + if err := storage.DeleteTriggerByFunctionNameVersion(txn, funcName, ""); err != nil { + log.GetLogger().Errorf("failed to delete trigger, error: %s", err) + return err + } + + return nil +} + +// DeleteTriggerByFuncNameVersion deletes trigger publish info by function name and version +func DeleteTriggerByFuncNameVersion(txn storage.Transaction, funcName string, versionOrAlias string) error { + infos, err := storage.GetTriggerByFunctionNameVersion(txn, funcName, versionOrAlias) + if err != nil { + log.GetLogger().Errorf("failed to get trigger, error %s", err.Error()) + return err + } + + err = storage.DeleteTriggerByFunctionNameVersion(txn, funcName, versionOrAlias) + if err != nil { + log.GetLogger().Errorf("failed to delete trigger by function %s version %s, error: %s", + funcName, versionOrAlias, err.Error()) + return err + } + + for _, triggerInfo := range infos { + if err := publish.DeleteTrigger(txn, funcName, versionOrAlias, triggerInfo); err != nil { + log.GetLogger().Errorf("failed to delete publishings, error: %s", err.Error()) + return err + } + } + return nil +} + +// UpdateTrigger update trigger info +func UpdateTrigger(ctx server.Context, req model.TriggerUpdateRequest) (model.TriggerResponse, error) { + // start transaction + txn := storage.NewTxn(ctx) + defer txn.Cancel() + + resp := model.TriggerResponse{} + funcInfo, info, err := updateTrigger(req) + if err != nil { + log.GetLogger().Errorf("failed to convert request to trigger info, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + tenantInfo, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + funcID := utils.BuildTriggerURN(tenantInfo, funcInfo.FunctionName, funcInfo.VersionOrAlias) + if err := storeUpdateTrigger(txn, info, funcInfo, funcID); err != nil { + log.GetLogger().Errorf("failed to convert request, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + err = txn.Commit() + if err != nil { + log.GetLogger().Errorf("failed to commit, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + + resp.TriggerInfo, err = buildTriggerResponse(ctx, info, funcInfo.FunctionName, funcInfo.VersionOrAlias) + if err != nil { + log.GetLogger().Errorf("failed to convert triggerInfo to response, error: %s", err.Error()) + return model.TriggerResponse{}, err + } + return resp, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/service/trigger_http.go b/functionsystem/apps/meta_service/function_repo/service/trigger_http.go new file mode 100644 index 0000000000000000000000000000000000000000..42ca40ee3e464b5fa1dbfd13d48682738559c34c --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/trigger_http.go @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "regexp" + "sort" + "strings" + + "meta_service/common/logger/log" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/storage" + "meta_service/function_repo/utils" +) + +const ( + defaultAlgorithm = "HMAC-SHA256" + maxHashLen = 12 + appidMaxLen = 32 + resourceIDMaxLen = 256 + splitStr = "/" + + // resourceIDRegex contains uppercase letters, lowercase letters, digits, hyphens (-) and Multi-level routing(/). + resourceIDRegex = "^[a-z0-9A-Z-/.]+$" +) + +// httpMethods must be sorted in ascending order +var httpMethods = [...]string{"DELETE", "GET", "POST", "PUT"} + +func createHTTPTriggerSpec( + txn *storage.Txn, raw interface{}, funcInfo model.FunctionQueryInfo, funcID, triggerID string, +) (storage.TriggerSpec, error) { + httpSpec, ok := raw.(*model.HTTPTriggerSpec) + if !ok { + log.GetLogger().Errorf("failed to assert HTTPTriggerSpec") + return nil, errors.New("assert HTTPTriggerSpec") + } + exist, err := storage.CheckResourceIDExist( + txn, funcInfo.FunctionName, getVerOrAlias(funcInfo), httpSpec.ResourceID) + if err != nil { + log.GetLogger().Errorf("failed to get resourceID, error: %s", err.Error()) + return nil, err + } + if exist { + log.GetLogger().Errorf("the functionID and resourceID exists") + return nil, errmsg.New(errmsg.TriggerPathRepeated) + } + + httpEtcdSpec := storage.HTTPTriggerEtcdSpec{} + if err := parseHTTPTriggerSpec(httpSpec, funcID, triggerID); err != nil { + log.GetLogger().Errorf("failed to parse httpspec, error: %s", err.Error()) + return nil, errmsg.NewParamError("invalid httpspec in request") + } + httpEtcdSpec.HTTPMethod = httpSpec.HTTPMethod + httpEtcdSpec.ResourceID = httpSpec.ResourceID + httpEtcdSpec.AuthFlag = httpSpec.AuthFlag + httpEtcdSpec.AuthAlgorithm = httpSpec.AuthAlgorithm + httpEtcdSpec.TriggerURL = httpSpec.TriggerURL + httpEtcdSpec.AppID = httpSpec.AppID + httpEtcdSpec.AppSecret = httpSpec.AppSecret + return &httpEtcdSpec, nil +} + +func parseHTTPTriggerSpec(spec *model.HTTPTriggerSpec, fid, tid string) error { + urlPrefix := config.RepoCfg.TriggerCfg.URLPrefix + spec.TriggerID = tid + spec.FuncID = fid + + err := checkResourceID(spec.ResourceID) + if err != nil { + log.GetLogger().Errorf("failed to check resouceID, error: %s", err.Error()) + return err + } + if spec.AuthFlag { + if err := checkHTTPTriggerAppInfo(spec); err != nil { + log.GetLogger().Errorf("failed to check app info, error: %s", err.Error()) + return err + } + } + if err := checkHTTPMethod(spec.HTTPMethod); err != nil { + log.GetLogger().Errorf("failed to check http method, error: %s", err.Error()) + return err + } + appID, err := buildAppID(fid) + if err != nil { + log.GetLogger().Errorf("failed to build appID, error: %s", err.Error()) + return err + } + spec.TriggerURL = urlPrefix + appID + "/" + spec.ResourceID + return nil +} + +func checkResourceID(rid string) error { + if rid == "" || len(rid) > resourceIDMaxLen { + log.GetLogger().Errorf("invalid resourceID length: %d", len(rid)) + return errmsg.NewParamError("invalid resourceID length: %d", len(rid)) + } + if match, err := regexp.MatchString(resourceIDRegex, rid); err != nil || !match { + log.GetLogger().Errorf("incorrect resourceID format: %s", rid) + return errmsg.NewParamError("incorrect resourceID format: %s", rid) + } + return nil +} + +func checkHTTPTriggerAppInfo(spec *model.HTTPTriggerSpec) error { + if spec.AuthAlgorithm == "" || spec.AuthAlgorithm == defaultAlgorithm { + spec.AuthAlgorithm = defaultAlgorithm + if spec.AppID == "" || len(spec.AppID) > appidMaxLen { + log.GetLogger().Errorf("invalid appID len: %d", len(spec.AppID)) + return errmsg.NewParamError("invalid appID len: %d", len(spec.AppID)) + } + if err := encryptHTTPTriggerAppSecret(spec); err != nil { + log.GetLogger().Errorf("failed to encrypt appSecret, error: %s", err.Error()) + return err + } + } + return nil +} + +func encryptHTTPTriggerAppSecret(spec *model.HTTPTriggerSpec) error { + var err error + if spec.AppSecret == "" || len(spec.AppSecret) > appSecretMaxLen { + log.GetLogger().Errorf("invalid appSecret: %s", spec.AppSecret) + return errmsg.NewParamError("invalid appSecret: %s", spec.AppSecret) + } + + if err != nil { + log.GetLogger().Errorf("failed to encrpt appSecret, error: %s", err.Error()) + return err + } + return nil +} + +func checkHTTPMethod(method string) error { + methods := strings.Split(method, splitStr) + for _, v := range methods { + if idx := sort.SearchStrings(httpMethods[:], v); idx == len(httpMethods) || httpMethods[idx] != v { + log.GetLogger().Errorf("invalid httpMethod: %s", v) + return errmsg.NewParamError("invalid httpMethod: %s", v) + } + } + return nil +} + +func buildAppID(fid string) (string, error) { + info, ok := utils.ParseTriggerInfoFromURN(fid) + if !ok { + return "", errmsg.NewParamError("invalid URN: %s", fid) + } + + sum := sha256.Sum256([]byte(fid)) + hashCode := hex.EncodeToString(sum[:]) + if len(hashCode) > maxHashLen { + hashCode = hashCode[:maxHashLen] + } + appID := config.RepoCfg.URNCfg.Prefix + ":" + config.RepoCfg.URNCfg.Zone + ":" + info.FunctionInfo.BusinessID + + ":" + "function:" + hashCode + ":" + info.FunctionInfo.FunctionName + ":" + info.VerOrAlias + return appID, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/service/trigger_http_test.go b/functionsystem/apps/meta_service/function_repo/service/trigger_http_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0af71c897797a9a429abe0c97b208b0f6235d8f8 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/trigger_http_test.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "errors" + "testing" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + + "meta_service/function_repo/model" + "meta_service/function_repo/storage" +) + +func TestCheckResourceID(t *testing.T) { + Convey("Test checkResourceID", t, func() { + err := checkResourceID("") + So(err, ShouldNotBeNil) + err2 := checkResourceID("^&*%") + So(err2, ShouldNotBeNil) + err3 := checkResourceID("abc123") + So(err3, ShouldBeNil) + }) +} + +func Test_createHTTPTriggerSpec(t *testing.T) { + Convey("Test createHTTPTriggerSpec", t, func() { + Convey("with assert HTTPTriggerSpec err", func() { + _, err := createHTTPTriggerSpec(&storage.Txn{}, "abc", model.FunctionQueryInfo{}, "mock-funcID", "mock-triggerID") + So(err, ShouldNotBeNil) + }) + Convey("with storage.CheckResourceIDExist err", func() { + patch := gomonkey.ApplyFunc(storage.CheckResourceIDExist, func( + txn *storage.Txn, funcName string, verOrAlias string, resourceID string) (bool, error) { + return false, errors.New("mock err") + }) + defer patch.Reset() + _, err := createHTTPTriggerSpec(&storage.Txn{}, &model.HTTPTriggerSpec{}, model.FunctionQueryInfo{}, "mock-funcID", "mock-triggerID") + So(err, ShouldNotBeNil) + }) + Convey("with storage.CheckResourceID Exist", func() { + patch := gomonkey.ApplyFunc(storage.CheckResourceIDExist, func( + txn *storage.Txn, funcName string, verOrAlias string, resourceID string) (bool, error) { + return true, nil + }) + defer patch.Reset() + _, err := createHTTPTriggerSpec(&storage.Txn{}, &model.HTTPTriggerSpec{}, model.FunctionQueryInfo{}, "mock-funcID", "mock-triggerID") + So(err, ShouldNotBeNil) + }) + Convey("with parseHTTPTriggerSpec", func() { + patch := gomonkey.ApplyFunc(storage.CheckResourceIDExist, func( + txn *storage.Txn, funcName string, verOrAlias string, resourceID string) (bool, error) { + return false, nil + }).ApplyFunc(parseHTTPTriggerSpec, func(spec *model.HTTPTriggerSpec, fid, tid string) error { + return errors.New("mock err") + }) + defer patch.Reset() + _, err := createHTTPTriggerSpec(&storage.Txn{}, &model.HTTPTriggerSpec{}, model.FunctionQueryInfo{}, "mock-funcID", "mock-triggerID") + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/service/trigger_test.go b/functionsystem/apps/meta_service/function_repo/service/trigger_test.go new file mode 100644 index 0000000000000000000000000000000000000000..67cabfd3de801cad5436038e8756326331b89804 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/service/trigger_test.go @@ -0,0 +1,903 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package service + +import ( + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + + "meta_service/common/crypto" + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/storage/publish" + "meta_service/function_repo/test/fakecontext" + "meta_service/function_repo/utils" +) + +func TestMethodIsValid(t *testing.T) { + method := "PUT" + if checkHTTPMethod(method) != nil { + t.Errorf("method is invalid, method is %s", method) + } + method = "GET" + if checkHTTPMethod(method) != nil { + t.Errorf("method is invalid, method is %s", method) + } + method = "POST" + if checkHTTPMethod(method) != nil { + t.Errorf("method is invalid, method is %s", method) + } + method = "DELETE" + if checkHTTPMethod(method) != nil { + t.Errorf("method is invalid, method is %s", method) + } + method = "get" + if checkHTTPMethod(method) == nil { + t.Errorf("method is invalid, method is %s", method) + } +} + +func TestHttpMethodIsValid(t *testing.T) { + method := "PUT" + if checkHTTPMethod(method) != nil { + t.Errorf("method is invalid, method is %s", method) + } + method = "PUT" + if checkHTTPMethod(method) != nil { + t.Errorf("method is invalid, method is %s", method) + } + method = "PUT/GET/DELETE" + if checkHTTPMethod(method) != nil { + t.Errorf("method is invalid, method is %s", method) + } +} + +func TestHTTPTrigger(t *testing.T) { + Convey("Test trigger", t, func() { + patches := gomonkey.NewPatches() + txn := &storage.Txn{} + patches.ApplyFunc(storage.NewTxn, func(c server.Context) *storage.Txn { + return txn + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { return nil }) + patches.ApplyMethod(reflect.TypeOf(txn), "Cancel", func(_ *storage.Txn) {}) + patches.ApplyFunc(storage.GetTriggerInfo, + func(txn *storage.Txn, funcName string, verOrAlias string, triggerID string) (storage.TriggerValue, error) { + return storage.TriggerValue{}, nil + }) + patches.ApplyFunc(storage.SaveTriggerInfo, + func(txn *storage.Txn, funcName string, verOrAlias string, triggerID string, val storage.TriggerValue) error { + return nil + }) + patches.ApplyFunc(publish.AddTrigger, + func(txn *storage.Txn, funcName, verOrAlias string, info storage.TriggerValue, trigger interface{}) error { + return nil + }) + defer patches.Reset() + + Convey("Test CreateTrigger", func() { + patches.ApplyFunc(storage.GetFunctionVersion, + func(txn storage.Transaction, funcName, funcVer string) (storage.FunctionVersionValue, error) { + return storage.FunctionVersionValue{}, nil + }) + patches.ApplyFunc(storage.GetTriggerByFunctionNameVersion, + func(txn storage.Transaction, funcName string, funcVer string) ([]storage.TriggerValue, error) { + return nil, nil + }) + patches.ApplyFunc(storage.CheckResourceIDExist, + func(txn *storage.Txn, funcName string, verOrAlias string, resourceID string) (bool, error) { + return false, nil + }) + defer patches.Reset() + + req := model.TriggerCreateRequest{ + FuncID: "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:0-test-aa:$latest", + TriggerType: "HTTP", + Spec: &model.HTTPTriggerSpec{ + TriggerSpec: model.TriggerSpec{ + FuncID: "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:0-test-aa:$latest", + TriggerID: "", + TriggerType: "HTTP", + }, + HTTPMethod: "PUT", + ResourceID: "iamsff", + AuthFlag: false, + AuthAlgorithm: "", + TriggerURL: "", + AppID: "", + AppSecret: "", + }, + } + _, err := CreateTrigger(fakecontext.NewMockContext(), req) + So(err, ShouldBeNil) + }) + + Convey("Test Delete", func() { + patches.ApplyFunc(publish.DeleteTrigger, + func(txn storage.Transaction, funcName, verOrAlias string, info storage.TriggerValue) error { + return nil + }) + defer patches.Reset() + + Convey("Test DeleteTriggerByID", func() { + patches.ApplyFunc(storage.GetFunctionInfo, + func(txn *storage.Txn, triggerID string) (storage.TriggerFunctionIndexValue, error) { + return storage.TriggerFunctionIndexValue{}, nil + }) + patches.ApplyFunc(storage.DeleteTrigger, + func(txn *storage.Txn, funcName string, verOrAlias string, triggerID string) error { + return nil + }) + defer patches.Reset() + + err := DeleteTriggerByID(fakecontext.NewMockContext(), "1f05e8e5-ce0d-4c4a-903a-eae27b8f9d04") + So(err, ShouldBeNil) + }) + + Convey("Test DeleteTriggerByFuncID", func() { + patches.ApplyFunc(storage.GetTriggerByFunctionNameVersion, + func(txn storage.Transaction, funcName string, funcVer string) ([]storage.TriggerValue, error) { + return []storage.TriggerValue{ + { + TriggerType: model.HTTPType, + EtcdSpec: &storage.HTTPTriggerEtcdSpec{}, + }, + }, nil + }) + patches.ApplyFunc(storage.DeleteTriggerByFunctionNameVersion, + func(txn storage.Transaction, funcName string, funcVer string) error { + return nil + }) + defer patches.Reset() + + err := DeleteTriggerByFuncID(fakecontext.NewMockContext(), "0-test-aa") + So(err, ShouldBeNil) + }) + }) + }) +} + +func TestGetHTTPTrigger(t *testing.T) { + patches := gomonkey.NewPatches() + patches.ApplyFunc(storage.GetFunctionInfoByTriggerID, + func(ctx server.Context, triggerID string) (storage.TriggerFunctionIndexValue, error) { + return storage.TriggerFunctionIndexValue{}, nil + }) + patches.ApplyFunc(storage.GetTriggerInfoByTriggerID, + func(ctx server.Context, funcName string, verOrAlias string, triggerID string) (storage.TriggerValue, error) { + return storage.TriggerValue{ + TriggerType: model.HTTPType, + EtcdSpec: &storage.HTTPTriggerEtcdSpec{}, + }, nil + }) + defer patches.Reset() + + _, err := GetTrigger(fakecontext.NewMockContext(), "") + assert.NoError(t, err) +} + +func TestTriggerGetList(t *testing.T) { + patches := gomonkey.NewPatches() + outputs := []gomonkey.OutputCell{ + { + Values: gomonkey.Params{nil, errmsg.KeyNotFoundError}, + }, + } + patches.ApplyFuncSeq(storage.GetTriggerInfoList, outputs) + defer patches.Reset() + + _, err := GetTriggerList(fakecontext.NewMockContext(), 0, 0, + "sn:cn:yrk:i1fe539427b24702acc11fbb4e134e17:function:0-test-aa:$latest") + assert.NoError(t, err) +} + +func TestUpdateTrigger(t *testing.T) { + type args struct { + ctx server.Context + req model.TriggerUpdateRequest + } + tests := []struct { + name string + args args + want model.FunctionVersion + wantErr bool + }{ + {"test update trigger success", args{fakecontext.NewMockContext(), model.TriggerUpdateRequest{ + TriggerID: "deleteFunc", + RevisionID: "java1.8", + TriggerMode: "mode", + TriggerType: "type", + }}, model.FunctionVersion{}, false}, + {"test update trigger success", args{fakecontext.NewMockContext(), model.TriggerUpdateRequest{ + TriggerID: "deleteFunc", + RevisionID: "java1.8", + TriggerMode: "mode", + TriggerType: "HTTP", + Spec: model.HTTPTriggerSpec{ + TriggerSpec: model.TriggerSpec{ + FuncID: "sn:cn:businessid:tenantid@productid:function:test:$latest", + TriggerID: "", + TriggerType: "HTTP", + }, + HTTPMethod: "PUT", + ResourceID: "iamsff", + AuthFlag: false, + AuthAlgorithm: "", + TriggerURL: "", + AppID: "", + AppSecret: "", + }, + }}, model.FunctionVersion{}, false}, + } + + tt := tests[0] + t.Run(tt.name, func(t *testing.T) { + _, err := UpdateTrigger(tt.args.ctx, tt.args.req) + assert.NotNil(t, err) + assert.Equal(t, errmsg.InvalidUserParam, err.(snerror.SNError).Code()) + }) + tt = tests[1] + t.Run(tt.name, func(t *testing.T) { + _, err := UpdateTrigger(tt.args.ctx, tt.args.req) + assert.NotNil(t, err) + assert.Equal(t, errmsg.InvalidUserParam, err.(snerror.SNError).Code()) + }) +} + +func TestCheckHTTPTriggerAppInfo(t *testing.T) { + Convey("Test checkHTTPTriggerAppInfo", t, func() { + spec := &model.HTTPTriggerSpec{AuthAlgorithm: "mockAlgo"} + err := checkHTTPTriggerAppInfo(spec) + So(err, ShouldBeNil) + }) + Convey("Test checkHTTPTriggerAppInfo invalid appID len err", t, func() { + spec := &model.HTTPTriggerSpec{AuthAlgorithm: defaultAlgorithm} + err := checkHTTPTriggerAppInfo(spec) + So(err, ShouldNotBeNil) + }) + Convey("Test checkHTTPTriggerAppInfo encrypt invalid appSecret err", t, func() { + spec := &model.HTTPTriggerSpec{AuthAlgorithm: defaultAlgorithm, AppID: "123"} + err := checkHTTPTriggerAppInfo(spec) + So(err, ShouldNotBeNil) + }) + Convey("Test checkHTTPTriggerAppInfo failed to encrypt appSecret err", t, func() { + spec := &model.HTTPTriggerSpec{AuthAlgorithm: defaultAlgorithm, AppID: "123", AppSecret: "123"} + patch := gomonkey.ApplyFunc(crypto.Encrypt, func(content string, secret []byte) ([]byte, error) { + return nil, errors.New("mock err") + }) + defer patch.Reset() + err := checkHTTPTriggerAppInfo(spec) + So(err, ShouldNotBeNil) + }) + Convey("Test checkHTTPTriggerAppInfo", t, func() { + spec := &model.HTTPTriggerSpec{AuthAlgorithm: defaultAlgorithm, AppID: "123", AppSecret: "123"} + patch := gomonkey.ApplyFunc(crypto.Encrypt, func(content string, secret []byte) ([]byte, error) { + return []byte{0}, nil + }) + defer patch.Reset() + err := checkHTTPTriggerAppInfo(spec) + So(err, ShouldBeNil) + }) + +} + +func TestStoreUpdateTrigger(t *testing.T) { + ctx := fakecontext.NewMockContext() + txn := storage.NewTxn(ctx) + info := storage.TriggerValue{ + TriggerID: "abc123", + FuncName: "mock-func", + TriggerType: "", + RevisionID: "1", + RawEtcdSpec: nil, + EtcdSpec: nil, + CreateTime: time.Time{}, + UpdateTime: time.Time{}, + } + funcInfo := storage.TriggerFunctionIndexValue{ + FunctionName: "mock-func", + VersionOrAlias: "1", + } + Convey("Test storeUpdateTrigger with GetTriggerInfo err", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{}, errmsg.KeyNotFoundError + }) + defer patch.Reset() + err := storeUpdateTrigger(txn, info, funcInfo, "mock-func-id") + So(err, ShouldNotBeNil) + }) + Convey("Test storeUpdateTrigger with KeyNotFoundError", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{}, errors.New("mock err") + }) + defer patch.Reset() + err := storeUpdateTrigger(txn, info, funcInfo, "mock-func-id") + So(err, ShouldNotBeNil) + }) + Convey("Test storeUpdateTrigger with revisionId not same", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{RevisionID: "2"}, nil + }) + defer patch.Reset() + err := storeUpdateTrigger(txn, info, funcInfo, "mock-func-id") + So(err, ShouldNotBeNil) + }) + Convey("Test storeUpdateTrigger with convert request err", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{RevisionID: "1"}, nil + }).ApplyFunc(buildModelTriggerSpec, + func(info storage.TriggerValue, funcID string) (spec interface{}, err error) { + return nil, errors.New("mock err") + }) + defer patch.Reset() + err := storeUpdateTrigger(txn, info, funcInfo, "mock-func-id") + So(err, ShouldNotBeNil) + }) + Convey("Test storeUpdateTrigger with empty version and alias err", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{RevisionID: "1"}, nil + }).ApplyFunc(buildModelTriggerSpec, + func(info storage.TriggerValue, funcID string) (spec interface{}, err error) { + return nil, nil + }) + defer patch.Reset() + funcInfo2 := storage.TriggerFunctionIndexValue{ + FunctionName: "mock-func", + VersionOrAlias: "", + } + err := storeUpdateTrigger(txn, info, funcInfo2, "mock-func-id") + So(err, ShouldNotBeNil) + }) + Convey("Test storeUpdateTrigger with SaveTriggerInfo err", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{RevisionID: "1"}, nil + }).ApplyFunc(buildModelTriggerSpec, + func(info storage.TriggerValue, funcID string) (spec interface{}, err error) { + return nil, nil + }).ApplyFunc(storage.SaveTriggerInfo, + func(*storage.Txn, string, string, string, storage.TriggerValue) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := storeUpdateTrigger(txn, info, funcInfo, "mock-func-id") + So(err, ShouldNotBeNil) + }) + Convey("Test storeUpdateTrigger with publish trigger err", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{RevisionID: "1"}, nil + }).ApplyFunc(buildModelTriggerSpec, + func(info storage.TriggerValue, funcID string) (spec interface{}, err error) { + return nil, nil + }).ApplyFunc(storage.SaveTriggerInfo, + func(*storage.Txn, string, string, string, storage.TriggerValue) error { + return nil + }).ApplyFunc(publish.AddTrigger, + func(*storage.Txn, string, string, storage.TriggerValue, interface{}) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := storeUpdateTrigger(txn, info, funcInfo, "mock-func-id") + So(err, ShouldNotBeNil) + }) + Convey("Test storeUpdateTrigger", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, + func(*storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{RevisionID: "1"}, nil + }).ApplyFunc(buildModelTriggerSpec, + func(info storage.TriggerValue, funcID string) (spec interface{}, err error) { + return nil, nil + }).ApplyFunc(storage.SaveTriggerInfo, + func(*storage.Txn, string, string, string, storage.TriggerValue) error { + return nil + }).ApplyFunc(publish.AddTrigger, + func(*storage.Txn, string, string, storage.TriggerValue, interface{}) error { + return nil + }) + defer patch.Reset() + err := storeUpdateTrigger(txn, info, funcInfo, "mock-func-id") + So(err, ShouldBeNil) + }) +} + +func TestCheckFuncInfo(t *testing.T) { + txn := &storage.Txn{} + info := model.FunctionQueryInfo{AliasName: "abc", FunctionVersion: "abcd"} + Convey("Test checkFuncInfo", t, func() { + patch := gomonkey.ApplyFunc(checkFunctionNameAndVersion, func(*storage.Txn, string, string) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := checkFuncInfo(txn, info) + So(err, ShouldNotBeNil) + }) + Convey("Test checkFuncInfo 2", t, func() { + patch := gomonkey.ApplyFunc(checkFunctionNameAndVersion, func(*storage.Txn, string, string) error { + return nil + }).ApplyFunc(storage.AliasNameExist, func(txn *storage.Txn, funcName, aliasName string) (bool, error) { + return false, errors.New("mock err") + }) + defer patch.Reset() + err := checkFuncInfo(txn, info) + So(err, ShouldNotBeNil) + }) + Convey("Test checkFuncInfo 3", t, func() { + patch := gomonkey.ApplyFunc(checkFunctionNameAndVersion, func(*storage.Txn, string, string) error { + return nil + }).ApplyFunc(storage.AliasNameExist, func(txn *storage.Txn, funcName, aliasName string) (bool, error) { + return false, nil + }) + defer patch.Reset() + err := checkFuncInfo(txn, info) + So(err, ShouldNotBeNil) + }) + Convey("Test checkFuncInfo 4", t, func() { + patch := gomonkey.ApplyFunc(checkFunctionNameAndVersion, func(*storage.Txn, string, string) error { + return nil + }).ApplyFunc(storage.AliasNameExist, func(txn *storage.Txn, funcName, aliasName string) (bool, error) { + return true, nil + }) + defer patch.Reset() + err := checkFuncInfo(txn, info) + So(err, ShouldBeNil) + }) + Convey("Test checkFuncInfo 5", t, func() { + info.AliasName = "" + patch := gomonkey.ApplyFunc(checkFunctionNameAndVersion, func(tx *storage.Txn, name string, ver string) error { + if ver == "abcd" { + return errors.New("mock err") + } + return nil + }) + defer patch.Reset() + err := checkFuncInfo(txn, info) + So(err, ShouldNotBeNil) + }) +} + +func UpdateTriggerPatches(setErr int) [6]*gomonkey.Patches { + patches := [...]*gomonkey.Patches{ + gomonkey.ApplyFunc(updateTrigger, + func(req model.TriggerUpdateRequest) (storage.TriggerFunctionIndexValue, storage.TriggerValue, error) { + return storage.TriggerFunctionIndexValue{}, storage.TriggerValue{}, nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&fakecontext.Context{}), "TenantInfo", + func(c *fakecontext.Context) (server.TenantInfo, error) { + if setErr == 1 { + return server.TenantInfo{}, errors.New("mock err") + } + return server.TenantInfo{}, nil + }), + gomonkey.ApplyFunc(storeUpdateTrigger, func( + txn *storage.Txn, info storage.TriggerValue, funcInfo storage.TriggerFunctionIndexValue, funcID string, + ) error { + if setErr == 2 { + return errors.New("mock err") + } + return nil + }), + gomonkey.ApplyMethod(reflect.TypeOf(&storage.Txn{}), "Commit", func(tx *storage.Txn) error { + if setErr == 3 { + return errors.New("mock err") + } + return nil + }), + gomonkey.ApplyFunc(buildTriggerResponse, func( + ctx server.Context, triggerInfo storage.TriggerValue, funcName, verOrAlias string, + ) (model.TriggerInfo, error) { + if setErr == 4 { + return model.TriggerInfo{}, errors.New("mock err") + } + return model.TriggerInfo{}, nil + }), + gomonkey.ApplyFunc(utils.BuildTriggerURN, func(info server.TenantInfo, funcName, verOrAlias string) string { + return "abc123" + }), + } + return patches +} + +func TestUpdateTrigger2(t *testing.T) { + ctx := fakecontext.NewMockContext() + txnPatch := gomonkey.ApplyMethod(reflect.TypeOf(&storage.Txn{}), "Cancel", func(tx *storage.Txn) { + return + }) + defer txnPatch.Reset() + Convey("Test UpdateTrigger", t, func() { + for i := 0; i <= 4; i++ { + fmt.Println("case ", i) + patches := UpdateTriggerPatches(i) + req := model.TriggerUpdateRequest{} + _, err := UpdateTrigger(ctx, req) + if i == 0 { + So(err, ShouldBeNil) + } else { + So(err, ShouldNotBeNil) + } + for _, patch := range patches { + patch.Reset() + } + } + }) +} + +func TestCheckFunctionNameAndVersion(t *testing.T) { + Convey("Test checkFunctionNameAndVersion", t, func() { + txn := &storage.Txn{} + Convey("when KeyNotFoundError", func() { + patch := gomonkey.ApplyFunc(storage.GetFunctionVersion, + func(storage.Transaction, string, string) (storage.FunctionVersionValue, error) { + return storage.FunctionVersionValue{}, errmsg.KeyNotFoundError + }) + defer patch.Reset() + err := checkFunctionNameAndVersion(txn, "abc", "1") + So(err, ShouldNotBeNil) + }) + Convey("when other err", func() { + patch := gomonkey.ApplyFunc(storage.GetFunctionVersion, + func(storage.Transaction, string, string) (storage.FunctionVersionValue, error) { + return storage.FunctionVersionValue{}, errors.New("mock err") + }) + defer patch.Reset() + err := checkFunctionNameAndVersion(txn, "abc", "1") + So(err, ShouldNotBeNil) + }) + + }) + +} + +func TestGetTriggerList(t *testing.T) { + ctx := fakecontext.NewMockContext() + Convey("Test GetTriggerList", t, func() { + Convey("when ParseFunctionInfo err", func() { + patches := gomonkey.ApplyFunc(ParseFunctionInfo, + func(ctx server.Context, queryInfo, qualifier string) (model.FunctionQueryInfo, error) { + return model.FunctionQueryInfo{}, errors.New("mock err") + }) + defer patches.Reset() + _, err := GetTriggerList(ctx, 0, 0, "mock-fid") + So(err, ShouldNotBeNil) + }) + patch := gomonkey.ApplyFunc(ParseFunctionInfo, + func(ctx server.Context, queryInfo, qualifier string) (model.FunctionQueryInfo, error) { + return model.FunctionQueryInfo{}, nil + }) + defer patch.Reset() + Convey("when GetTriggerInfoList err", func() { + patch2 := gomonkey.ApplyFunc(storage.GetTriggerInfoList, + func(server.Context, string, string, int, int) ([]storage.TriggerValue, error) { + return nil, errors.New("mock err") + }) + defer patch2.Reset() + _, err := GetTriggerList(ctx, 0, 0, "mock-fid") + So(err, ShouldNotBeNil) + }) + Convey("when buildTriggerResponse err", func() { + patch2 := gomonkey.ApplyFunc(storage.GetTriggerInfoList, + func(server.Context, string, string, int, int) ([]storage.TriggerValue, error) { + triggerValues := []storage.TriggerValue{{}} + return triggerValues, nil + }).ApplyFunc(buildTriggerResponse, + func(server.Context, storage.TriggerValue, string, string) (model.TriggerInfo, error) { + return model.TriggerInfo{}, errors.New("mock err") + }) + defer patch2.Reset() + _, err := GetTriggerList(ctx, 0, 0, "mock-fid") + So(err, ShouldNotBeNil) + }) + Convey("when ok", func() { + patch2 := gomonkey.ApplyFunc(storage.GetTriggerInfoList, + func(server.Context, string, string, int, int) ([]storage.TriggerValue, error) { + triggerValues := []storage.TriggerValue{{}} + return triggerValues, nil + }).ApplyFunc(buildTriggerResponse, + func(server.Context, storage.TriggerValue, string, string) (model.TriggerInfo, error) { + return model.TriggerInfo{}, nil + }) + defer patch2.Reset() + _, err := GetTriggerList(ctx, 0, 0, "mock-fid") + So(err, ShouldBeNil) + }) + }) +} + +func TestDeleteTriggerByID(t *testing.T) { + ctx := fakecontext.NewMockContext() + txn := &storage.Txn{} + patches := gomonkey.ApplyFunc(storage.NewTxn, func(c server.Context) *storage.Txn { + return txn + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Cancel", func(_ *storage.Txn) {}) + defer patches.Reset() + Convey("Test DeleteTriggerByID with GetFunctionInfo err", t, func() { + patch := gomonkey.ApplyFunc(storage.GetFunctionInfo, func( + *storage.Txn, string) (storage.TriggerFunctionIndexValue, error) { + return storage.TriggerFunctionIndexValue{}, errmsg.KeyNotFoundError + }) + err := DeleteTriggerByID(ctx, "mock-tid") + So(err, ShouldNotBeNil) + patch.Reset() + + patch.ApplyFunc(storage.GetFunctionInfo, func( + *storage.Txn, string) (storage.TriggerFunctionIndexValue, error) { + return storage.TriggerFunctionIndexValue{}, errors.New("mock err") + }) + err = DeleteTriggerByID(ctx, "mock-tid") + So(err, ShouldNotBeNil) + patch.Reset() + }) + patches.ApplyFunc(storage.GetFunctionInfo, func( + *storage.Txn, string) (storage.TriggerFunctionIndexValue, error) { + return storage.TriggerFunctionIndexValue{}, nil + }) + Convey("Test DeleteTriggerByID with GetTriggerInfo err", t, func() { + patch := gomonkey.ApplyFunc(storage.GetTriggerInfo, func( + *storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{}, errors.New("mock err") + }) + defer patch.Reset() + err := DeleteTriggerByID(ctx, "mock-tid") + So(err, ShouldNotBeNil) + }) + patches.ApplyFunc(storage.GetTriggerInfo, func( + *storage.Txn, string, string, string) (storage.TriggerValue, error) { + return storage.TriggerValue{}, nil + }) + Convey("Test DeleteTriggerByID with storage.DeleteTrigger err", t, func() { + patch := gomonkey.ApplyFunc(storage.DeleteTrigger, func(*storage.Txn, string, string, string) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := DeleteTriggerByID(ctx, "mock-tid") + So(err, ShouldNotBeNil) + }) + patches.ApplyFunc(storage.DeleteTrigger, func(*storage.Txn, string, string, string) error { + return nil + }) + Convey("Test DeleteTriggerByID with publish.DeleteTrigger err", t, func() { + patch := gomonkey.ApplyFunc(publish.DeleteTrigger, func( + txn storage.Transaction, funcName, verOrAlias string, info storage.TriggerValue) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := DeleteTriggerByID(ctx, "mock-tid") + So(err, ShouldNotBeNil) + }) + patches.ApplyFunc(publish.DeleteTrigger, func( + txn storage.Transaction, funcName, verOrAlias string, info storage.TriggerValue) error { + return nil + }) + Convey("Test DeleteTriggerByID with txn.Commit err", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := DeleteTriggerByID(ctx, "mock-tid") + So(err, ShouldNotBeNil) + }) +} + +func Test_updateTrigger2(t *testing.T) { + Convey("Test updateTrigger 2", t, func() { + req := model.TriggerUpdateRequest{ + TriggerType: "mock-trigger", + } + trigger := triggerSpecTable[model.HTTPType] + trigger.Update = func(raw interface{}) (storage.TriggerSpec, string, error) { + return nil, "", nil + } + triggerSpecTable[req.TriggerType] = trigger + Convey("with CheckAndGetVerOrAlias err", func() { + patch := gomonkey.ApplyFunc(CheckAndGetVerOrAlias, func( + funcID, qualifier string) (model.FunctionQueryInfo, error) { + return model.FunctionQueryInfo{}, errors.New("mock err") + }) + defer patch.Reset() + _, _, err := updateTrigger(req) + So(err, ShouldNotBeNil) + }) + Convey("with success", func() { + patch := gomonkey.ApplyFunc(CheckAndGetVerOrAlias, func( + funcID, qualifier string) (model.FunctionQueryInfo, error) { + return model.FunctionQueryInfo{}, nil + }) + defer patch.Reset() + _, _, err := updateTrigger(req) + So(err, ShouldBeNil) + }) + Convey("with TriggerType not contain in triggerSpecTable", func() { + req.TriggerType = "mock-trigger2" + _, _, err := updateTrigger(req) + So(err, ShouldNotBeNil) + }) + }) +} + +func Test_storeCreateTrigger(t *testing.T) { + Convey("Test storeCreateTrigger", t, func() { + mode := 0 + patches := gomonkey.ApplyFunc(buildModelTriggerSpec, func(info storage.TriggerValue, funcID string) (spec interface{}, err error) { + if mode == 1 { + return nil, errors.New("mock err") + } + return nil, nil + }).ApplyFunc(getVerOrAlias, func(i model.FunctionQueryInfo) string { + if mode == 2 { + return "" + } + return "mock-ver-or-alias" + }).ApplyFunc(storage.SaveTriggerInfo, func(txn *storage.Txn, funcName string, + verOrAlias string, triggerID string, val storage.TriggerValue) error { + if mode == 3 { + return errors.New("mock err") + } + return nil + }).ApplyFunc(publish.AddTrigger, func(txn *storage.Txn, funcName, verOrAlias string, + info storage.TriggerValue, value interface{}) error { + if mode == 4 { + return errors.New("mock err") + } + return nil + }) + defer patches.Reset() + for mode = 0; mode < 5; mode++ { + err := storeCreateTrigger(&storage.Txn{}, storage.TriggerValue{}, model.FunctionQueryInfo{}, "mock-fun-id") + if mode == 0 { + So(err, ShouldBeNil) + } else { + So(err, ShouldNotBeNil) + } + } + }) +} + +func TestDeleteLayerVersion(t *testing.T) { + Convey("Test DeleteLayerVersion", t, func() { + mode := 0 + patches := gomonkey.ApplyFunc(storage.NewTxn, func(c server.Context) *storage.Txn { + return &storage.Txn{} + }).ApplyMethod(reflect.TypeOf(&storage.Txn{}), "Cancel", func(_ *storage.Txn) { + return + }).ApplyFunc(storage.GetLayerVersionTx, func(txn storage.Transaction, layerName string, layerVersion int) (storage.LayerValue, error) { + if mode == 1 { + return storage.LayerValue{}, errmsg.KeyNotFoundError + } else if mode == 2 { + return storage.LayerValue{}, errors.New("mock err") + } + return storage.LayerValue{}, nil + }).ApplyFunc(storage.IsLayerVersionUsed, func(txn *storage.Txn, layerName string, layerVersion int) (bool, error) { + if mode == 3 { + return true, errors.New("mock err") + } else if mode == 4 { + return true, nil + } + return false, nil + }).ApplyFunc(storage.AddUncontrolledTx, func(txn storage.Transaction, bucketID, objectID string) error { + if mode == 5 { + return errors.New("mock err") + } + return nil + }).ApplyFunc(storage.DeleteLayerVersion, func(txn *storage.Txn, layerName string, layerVersion int) error { + if mode == 6 { + return errors.New("mock err") + } + return nil + }).ApplyMethod(reflect.TypeOf(&storage.Txn{}), "Commit", func(_ *storage.Txn) error { + if mode == 7 { + return errors.New("mock err") + } + return nil + }).ApplyFunc(pkgstore.Delete, func(bucketID, objectID string) error { + return nil + }).ApplyFunc(storage.RemoveUncontrolled, func(ctx server.Context, bucketID, objectID string) error { + return nil + }) + defer patches.Reset() + for mode = 0; mode < 8; mode++ { + err := DeleteLayerVersion(fakecontext.NewMockContext(), "mock-layer-name", 1) + if mode == 0 { + So(err, ShouldBeNil) + } else { + So(err, ShouldNotBeNil) + } + } + }) +} + +func TestCreateTrigger(t *testing.T) { + Convey("Test CreateTrigger", t, func() { + mode := 0 + patches := gomonkey.NewPatches() + txn := &storage.Txn{} + patches.ApplyFunc(storage.NewTxn, func(c server.Context) *storage.Txn { + return txn + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Cancel", func(_ *storage.Txn) {}) + defer patches.Reset() + patches.ApplyFunc(ParseFunctionInfo, func(ctx server.Context, queryInfo, qualifier string) (model.FunctionQueryInfo, error) { + if mode == 0 { + return model.FunctionQueryInfo{}, errors.New("mock err") + } + return model.FunctionQueryInfo{}, nil + }) + patches.ApplyFunc(checkFuncInfo, func(txn *storage.Txn, info model.FunctionQueryInfo) error { + if mode == 1 { + return errors.New("mock err") + } + return nil + }) + patches.ApplyFunc(storage.GetTriggerByFunctionNameVersion, func( + txn storage.Transaction, funcName string, funcVer string) ([]storage.TriggerValue, error) { + if mode == 2 { + return nil, errors.New("mock err") + } else if mode == 3 { + return make([]storage.TriggerValue, 34, 34), nil + } + return make([]storage.TriggerValue, 1, 1), nil + }) + patches.ApplyFunc(createTrigger, func(txn *storage.Txn, req model.TriggerCreateRequest, funcInfo model.FunctionQueryInfo, + ) (storage.TriggerValue, error) { + if mode == 4 { + return storage.TriggerValue{}, errors.New("mock err") + } + return storage.TriggerValue{}, nil + }) + patches.ApplyFunc(storeCreateTrigger, func( + txn *storage.Txn, triggerInfo storage.TriggerValue, funcInfo model.FunctionQueryInfo, funcID string, + ) error { + if mode == 5 { + return errors.New("mock err") + } + return nil + }) + patches.ApplyMethod(reflect.TypeOf(txn), "Commit", func(_ *storage.Txn) error { + if mode == 6 { + return errors.New("mock err") + } + return nil + }) + patches.ApplyFunc(buildTriggerResponse, func(ctx server.Context, triggerInfo storage.TriggerValue, funcName, verOrAlias string, + ) (model.TriggerInfo, error) { + if mode == 7 { + return model.TriggerInfo{}, errors.New("mock err") + } + return model.TriggerInfo{}, nil + }) + for mode = 0; mode < 8; mode++ { + _, err := CreateTrigger(fakecontext.NewMockContext(), model.TriggerCreateRequest{}) + So(err, ShouldNotBeNil) + } + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/alias.go b/functionsystem/apps/meta_service/function_repo/storage/alias.go new file mode 100644 index 0000000000000000000000000000000000000000..04af263525ae1aebdf91d836018e0eb6cea1256d --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/alias.go @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" + + "meta_service/common/engine" + "meta_service/common/logger/log" +) + +// IsFuncVersionExist check if function version exist +func IsFuncVersionExist(txn *Txn, funcName, funcVersion string) (bool, error) { + _, err := GetFunctionVersion(txn, funcName, funcVersion) + if err != nil { + if err == errmsg.KeyNotFoundError { + return false, nil + } + log.GetLogger().Errorf("failed to get function version, error: %s", err.Error()) + return false, err + } + return true, nil +} + +// IsAliasNameExist check if alias name exist +func IsAliasNameExist(txn *Txn, fName, aName string) (bool, error) { + _, err := GetAlias(txn, fName, aName) + if err != nil { + if err == errmsg.KeyNotFoundError { + return false, nil + } + log.GetLogger().Errorf("failed to get alias version, error: %s", err.Error()) + return false, err + } + return true, nil +} + +func genAliasKey(t server.TenantInfo, funcName, aliasName string) AliasKey { + return AliasKey{ + TenantInfo: t, + FunctionName: funcName, + AliasName: aliasName, + } +} + +func genAliasRoutingIndexKey(t server.TenantInfo, funcName, funcVersion string) AliasRoutingIndexKey { + return AliasRoutingIndexKey{ + TenantInfo: t, + FunctionName: funcName, + FunctionVersion: funcVersion, + } +} + +// CreateAlias write data to alias table and alias routing index table on storage within a transaction +func CreateAlias(txn *Txn, alias AliasValue) error { + t, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + if err := txn.txn.AliasPut(genAliasKey(t, alias.FunctionName, alias.Name), alias); err != nil { + log.GetLogger().Errorf("alias failed to put entry, error: %s", err.Error()) + return err + } + + for k := range alias.RoutingConfig { + if err := txn.txn.AliasRoutingIndexPut( + genAliasRoutingIndexKey(t, alias.FunctionName, k), + AliasRoutingIndexValue{Name: alias.Name}); err != nil { + + log.GetLogger().Errorf("alias failed to put routing index, error: %s", err.Error()) + return err + } + } + + return nil +} + +// DeleteAliasByFunctionName deletes alias metadata entry and related index entries within a transaction +func DeleteAliasByFunctionName(txn Transaction, funcName string) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + + if err := txn.GetTxn().AliasDeleteRange(genAliasKey(t, funcName, "")); err != nil { + log.GetLogger().Errorf("alias failed to delete range, error: %s", err.Error()) + return err + } + + if err := txn.GetTxn().AliasRoutingIndexDeleteRange(genAliasRoutingIndexKey(t, funcName, "")); err != nil { + log.GetLogger().Errorf("alias failed to delete routing index range, error: %s", err.Error()) + return err + } + + return nil +} + +// DeleteAlias deletes alias metadata and related index entries by specified alias name +func DeleteAlias(txn *Txn, funcName string, aliasName string, routingVers []string) error { + t, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + if err := txn.txn.AliasDelete(genAliasKey(t, funcName, aliasName)); err != nil { + log.GetLogger().Errorf("alias failed to delete, error: %s", err.Error()) + return err + } + for _, v := range routingVers { + if err := txn.txn.AliasRoutingIndexDelete(genAliasRoutingIndexKey(t, funcName, v)); err != nil { + log.GetLogger().Errorf("alias failed to delete routing index, error: %s", err.Error()) + return err + } + } + return nil +} + +// GetAliasNumByFunctionName returns alias number of specified function name within a transaction +func GetAliasNumByFunctionName(txn Transaction, funcName string) (int, error) { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return 0, err + } + + tuples, err := txn.GetTxn().AliasGetRange(genAliasKey(t, funcName, "")) + if err != nil { + log.GetLogger().Errorf("alias failed to get range, error: %s", err.Error()) + return 0, err + } + return len(tuples), nil +} + +// AliasRoutingExist returns true if any alias routings is referred to specified function name and version +// within a transaction +func AliasRoutingExist(txn Transaction, funcName, funcVer string) (bool, error) { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return false, err + } + + _, err = txn.GetTxn().AliasRoutingIndexGet(genAliasRoutingIndexKey(t, funcName, funcVer)) + if err != nil { + if err == errmsg.KeyNotFoundError { + return false, nil + } + log.GetLogger().Errorf("alias failed to get routing index, error: %s", err.Error()) + return false, err + } + return true, nil +} + +// GetAlias get alias information from storage within a transaction +func GetAlias(txn *Txn, funcName, aliasName string) (AliasValue, error) { + t, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return AliasValue{}, err + } + + res, err := txn.txn.AliasGet(genAliasKey(t, funcName, aliasName)) + if err != nil { + log.GetLogger().Debugf("failed to get alias, error: %s", err.Error()) + return AliasValue{}, err + } + return res, nil +} + +// AliasNameExist returns true if alias can be found in alias version index entry. +func AliasNameExist(txn *Txn, funcName, aliasName string) (bool, error) { + _, err := GetAlias(txn, funcName, aliasName) + if err != nil { + if err == errmsg.KeyNotFoundError { + return false, nil + } + log.GetLogger().Errorf("failed to get alias version index, error: %s", err.Error()) + return false, err + } + return true, nil +} + +// GetAliasesByPage get aliases list in page range +func GetAliasesByPage(ctx server.Context, funcName string, pageIndex, pageSize int) ([]AliasValue, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return nil, err + } + + by := engine.SortBy{ + Order: engine.Descend, + Target: engine.SortModify, + } + tuples, _, err := db.AliasStream(ctx.Context(), + genAliasKey(t, funcName, ""), by).ExecuteWithPage(pageIndex, pageSize) + if err != nil { + if err == errmsg.KeyNotFoundError { + return nil, nil + } + log.GetLogger().Errorf(" failed to get aliases from stream, error: %s", err.Error()) + return nil, err + } + res := make([]AliasValue, len(tuples), len(tuples)) + for i, tuple := range tuples { + res[i] = tuple.Value + } + return res, nil +} + +// GetAliasValue get alias value from storage +func GetAliasValue(c server.Context, funcName, aliasName string) (AliasValue, error) { + t, err := c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return AliasValue{}, err + } + res, err := db.AliasGet(c.Context(), genAliasKey(t, funcName, aliasName)) + if err != nil { + log.GetLogger().Debugf("alias get failed, error: %s", err.Error()) + return AliasValue{}, err + } + return res, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/alias_test.go b/functionsystem/apps/meta_service/function_repo/storage/alias_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7d8a51e73aaa32fefed9d7ffb4efc748c76b79d6 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/alias_test.go @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "context" + "errors" + "reflect" + "testing" + + "meta_service/function_repo/errmsg" + "meta_service/function_repo/test/fakecontext" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" +) + +func TestAliasNameExist(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(GetAlias, func(_ *Txn, _, _ string) (AliasValue, error) { + return AliasValue{}, errmsg.KeyNotFoundError + }) + exist, err := AliasNameExist(&Txn{}, "testFunc", "testAlias") + assert.Equal(t, false, exist) + assert.Nil(t, err) + + patches.Reset() + patches.ApplyFunc(GetAlias, func(_ *Txn, _, _ string) (AliasValue, error) { + return AliasValue{}, errors.New("fake error! ") + }) + exist, err = AliasNameExist(&Txn{}, "testFunc", "testAlias") + assert.Equal(t, false, exist) + assert.NotNil(t, err) + + patches.Reset() + patches.ApplyFunc(GetAlias, func(_ *Txn, _, _ string) (AliasValue, error) { + return AliasValue{}, nil + }) + exist, err = AliasNameExist(&Txn{}, "testFunc", "testAlias") + assert.Equal(t, true, exist) +} + +func TestIsAliasNameExist(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(GetAlias, func(_ *Txn, _, _ string) (AliasValue, error) { + return AliasValue{}, errmsg.KeyNotFoundError + }) + exist, err := IsAliasNameExist(&Txn{}, "testFunc", "testAlias") + assert.Equal(t, false, exist) + assert.Nil(t, err) + + patches.Reset() + patches.ApplyFunc(GetAlias, func(_ *Txn, _, _ string) (AliasValue, error) { + return AliasValue{}, errors.New("fake error! ") + }) + exist, err = IsAliasNameExist(&Txn{}, "testFunc", "testAlias") + assert.Equal(t, false, exist) + assert.NotNil(t, err) + + patches.Reset() + patches.ApplyFunc(GetAlias, func(_ *Txn, _, _ string) (AliasValue, error) { + return AliasValue{}, nil + }) + exist, err = IsAliasNameExist(&Txn{}, "testFunc", "testAlias") + assert.Equal(t, true, exist) +} + +func TestGetAliasValue(t *testing.T) { + Convey("Test GetAliasValue mock db AliasGet err", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&generatedKV{}), "AliasGet", func(_ *generatedKV, _ context.Context, _ AliasKey) (AliasValue, error) { + return AliasValue{}, errors.New("mock db AliasGet err") + }) + defer patch.Reset() + ctx := fakecontext.NewMockContext() + _, err := GetAliasValue(ctx, "mockFunc", "mockAlias") + So(err, ShouldNotBeNil) + }) + Convey("Test GetAliasValue", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&generatedKV{}), "AliasGet", func(_ *generatedKV, _ context.Context, _ AliasKey) (AliasValue, error) { + return AliasValue{}, nil + }) + defer patch.Reset() + ctx := fakecontext.NewMockContext() + _, err := GetAliasValue(ctx, "mockFunc", "mockAlias") + So(err, ShouldBeNil) + }) + Convey("Test GetAliasValue tenant info nil err", t, func() { + patch := gomonkey.ApplyMethod(reflect.TypeOf(&generatedKV{}), "AliasGet", func(_ *generatedKV, _ context.Context, _ AliasKey) (AliasValue, error) { + return AliasValue{}, nil + }) + defer patch.Reset() + ctx := fakecontext.NewContext() + _, err := GetAliasValue(ctx, "mockFunc", "mockAlias") + So(err, ShouldNotBeNil) + }) +} + +func TestDeleteAliasByFunctionName(t *testing.T) { + Convey("Test DeleteAliasByFunctionName when tenant nil", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewContext(), + } + err := DeleteAliasByFunctionName(txn, "mockName") + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteAliasByFunctionName when AliasDeleteRange err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "AliasDeleteRange", + func(t *generatedTx, prefix AliasKey) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := DeleteAliasByFunctionName(txn, "mockName") + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteAliasByFunctionName when AliasRoutingIndexDeleteRange err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "AliasDeleteRange", + func(t *generatedTx, prefix AliasKey) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "AliasRoutingIndexDeleteRange", + func(t *generatedTx, prefix AliasRoutingIndexKey) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := DeleteAliasByFunctionName(txn, "mockName") + So(err, ShouldNotBeNil) + }) +} + +func TestDeleteAlias2(t *testing.T) { + Convey("Test DeleteAlias with TenantInfo nil", t, func() { + txn := &Txn{txn: &generatedTx{}, c: fakecontext.NewContext()} + err := DeleteAlias(txn, "mock-func", "mock-alias", nil) + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteAlias with AliasDelete err", t, func() { + txn := &Txn{txn: &generatedTx{}, c: fakecontext.NewMockContext()} + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "AliasDelete", func( + t *generatedTx, key AliasKey, + ) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := DeleteAlias(txn, "mock-func", "mock-alias", nil) + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteAlias with AliasRoutingIndexDelete err", t, func() { + txn := &Txn{txn: &generatedTx{}, c: fakecontext.NewMockContext()} + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "AliasDelete", func( + t *generatedTx, key AliasKey, + ) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "AliasRoutingIndexDelete", func(t *generatedTx, key AliasRoutingIndexKey) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := DeleteAlias(txn, "mock-func", "mock-alias", []string{"a"}) + So(err, ShouldNotBeNil) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/function.go b/functionsystem/apps/meta_service/function_repo/storage/function.go new file mode 100644 index 0000000000000000000000000000000000000000..90a343b8bb6e7cfe83669d4fcd8617680c17a50f --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/function.go @@ -0,0 +1,506 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "strings" + + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" + "meta_service/function_repo/utils" + + "meta_service/common/constants" + "meta_service/common/engine" + "meta_service/common/logger/log" +) + +var EngineSortBy = engine.SortBy{ + Order: engine.Descend, + Target: engine.SortModify, +} + +// GenFunctionVersionKey Gen Function Version Key +func GenFunctionVersionKey(t server.TenantInfo, funcName string, funcVer string) FunctionVersionKey { + return FunctionVersionKey{ + TenantInfo: t, + FunctionName: funcName, + FunctionVersion: funcVer, + } +} + +func genFunctionStatusKey(t server.TenantInfo, funcName string, funcVer string) FunctionStatusKey { + var funcStatus FunctionStatusKey + funcVersion := FunctionVersionKey{ + TenantInfo: t, + FunctionName: funcName, + FunctionVersion: funcVer, + } + + funcStatus.FunctionVersionKey = funcVersion + return funcStatus +} + +func genLayerFunctionIndexKey( + t server.TenantInfo, layerName string, layerVer int, funcName, funcVer string, +) LayerFunctionIndexKey { + return LayerFunctionIndexKey{ + TenantInfo: t, + LayerName: layerName, + LayerVersion: layerVer, + FunctionName: funcName, + FunctionVersion: funcVer, + } +} + +func genObjectRefIndexKey(t server.TenantInfo, bucketID string, objectID string) ObjectRefIndexKey { + return ObjectRefIndexKey{ + TenantInfo: t, + BucketID: bucketID, + ObjectID: objectID, + } +} + +// DeleteFunctionVersions deletes all function versions under a function name. +func DeleteFunctionVersions(txn Transaction, funcName string) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + + prefix := GenFunctionVersionKey(t, funcName, "") + tuples, err := txn.GetTxn().FunctionVersionGetRange(prefix) + if err != nil { + log.GetLogger().Errorf("failed to get function version from range, error: %s", err.Error()) + return err + } + + for _, tuple := range tuples { + for _, layer := range tuple.Value.FunctionLayer { + k := genLayerFunctionIndexKey( + t, layer.Name, layer.Version, tuple.Value.Function.Name, tuple.Value.FunctionVersion.Version) + if err := txn.GetTxn().LayerFunctionIndexDelete(k); err != nil { + log.GetLogger().Errorf("failed to delete layer function index, error: %s", err.Error()) + return err + } + } + if tuple.Value.FunctionVersion.Package.CodeUploadType != constants.S3StorageType { + continue + } + if err := delObjRefCntTx(txn, tuple.Value.FunctionVersion.Package); err != nil { + log.GetLogger().Errorf("failed to delete object reference count, error: %s", err.Error()) + return err + } + } + + if err := txn.GetTxn().FunctionVersionDeleteRange(prefix); err != nil { + log.GetLogger().Errorf("failed to delete function version from range, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteFunctionStatuses deletes all function statuses under a function name. +func DeleteFunctionStatuses(txn Transaction, funcName string) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + + prefix := genFunctionStatusKey(t, funcName, "") + if err := txn.GetTxn().FunctionStatusDeleteRange(prefix); err != nil { + log.GetLogger().Errorf("failed to delete function status from range, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteFunctionVersion deletes function version entries by function name and version, +// related layer function version index entries also will be deleted. +func DeleteFunctionVersion(txn Transaction, funcName string, funcVer string) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + + key := GenFunctionVersionKey(t, funcName, funcVer) + val, err := txn.GetTxn().FunctionVersionGet(key) + if err != nil { + log.GetLogger().Debugf("failed to get function version, error: %s", err.Error()) + return err + } + + for _, layer := range val.FunctionLayer { + k := genLayerFunctionIndexKey(t, layer.Name, layer.Version, val.Function.Name, val.FunctionVersion.Version) + if err := txn.GetTxn().LayerFunctionIndexDelete(k); err != nil { + log.GetLogger().Errorf("failed to delete layer function index, error: %s", err.Error()) + return err + } + } + + if err := delObjRefCntTx(txn, val.FunctionVersion.Package); err != nil { + log.GetLogger().Errorf("failed to delete object reference count, error: %s", err.Error()) + return err + } + if err := txn.GetTxn().FunctionVersionDelete(key); err != nil { + log.GetLogger().Errorf("failed to delete function version, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteFunctionStatus deletes all function status under a function name. +func DeleteFunctionStatus(txn Transaction, funcName string, funcVer string) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + key := genFunctionStatusKey(t, funcName, funcVer) + if err := txn.GetTxn().FunctionStatusDelete(key); err != nil { + log.GetLogger().Errorf("failed to delete function status, error: %s", err.Error()) + return err + } + return nil +} + +// GetFunctionVersion reads function version entry by function name and version from storage within a txn. +func GetFunctionVersion(txn Transaction, funcName, funcVer string) (FunctionVersionValue, error) { + if funcVer == "" { + funcVer = utils.GetDefaultVersion() + } + t, err := txn.GetCtx().TenantInfo() + var res FunctionVersionValue + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return FunctionVersionValue{}, err + } + res, err = txn.GetTxn().FunctionVersionGet(GenFunctionVersionKey(t, funcName, funcVer)) + if err != nil { + log.GetLogger().Debugf("failed to get function version, error: %s", err.Error()) + return FunctionVersionValue{}, err + } + return res, nil +} + +// GetFunctionVersions reads function version entries by function name from storage within a txn. +func GetFunctionVersions(txn Transaction, funcName string) ([]FunctionVersionValue, error) { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return nil, err + } + + tuples, err := txn.GetTxn().FunctionVersionGetRange(GenFunctionVersionKey(t, funcName, "")) + if err != nil { + log.GetLogger().Errorf("failed to get function version from range, error: %s", err.Error()) + return nil, err + } + if len(tuples) == 0 { + return nil, errmsg.New(errmsg.FunctionNotFound, utils.RemoveServiceID(funcName)) + } + + fvs := make([]FunctionVersionValue, len(tuples), len(tuples)) + for i, tuple := range tuples { + fvs[i] = tuple.Value + } + return fvs, nil +} + +// CreateFunctionVersion saves function version entry to storage. +// It also saves 2 indexes: layer function version index and object reference count index. +func CreateFunctionVersion(txn Transaction, fv FunctionVersionValue) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + if err := txn.GetTxn().FunctionVersionPut( + GenFunctionVersionKey(t, fv.Function.Name, fv.FunctionVersion.Version), fv); err != nil { + log.GetLogger().Errorf("failed to put function version, error: %s", err.Error()) + return err + } + + for _, layer := range fv.FunctionLayer { + k := genLayerFunctionIndexKey(t, layer.Name, layer.Version, fv.Function.Name, fv.FunctionVersion.Version) + if err := txn.GetTxn().LayerFunctionIndexPut(k, LayerFunctionIndexValue{}); err != nil { + log.GetLogger().Errorf("failed to put layer function index, error: %s", err.Error()) + return err + } + } + // if package is upload by repo, storage type will be s3 + if fv.FunctionVersion.Package.CodeUploadType != constants.S3StorageType { + return nil + } + if err := addObjRefCntTx(txn, fv.FunctionVersion.Package); err != nil { + log.GetLogger().Errorf("failed to add object reference count, error: %s", err.Error()) + return err + } + return nil +} + +// CreateFunctionStatus saves function status entry to storage. +func CreateFunctionStatus(txn Transaction, funcName string, funcVer string) error { + statusValue := FunctionStatusValue{ + Status: constants.FunctionStatusUnavailable, + } + + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + + if err := txn.GetTxn().FunctionStatusPut( + genFunctionStatusKey(t, funcName, funcVer), statusValue); err != nil { + log.GetLogger().Errorf("failed to put function status, error: %s", err.Error()) + return err + } + + return nil +} + +// UpdateFunctionVersion updates function version entry to storage. +// It also updates 2 indexes: layer function version index and object reference count index. +func UpdateFunctionVersion(txn Transaction, fv FunctionVersionValue) error { + if err := DeleteFunctionVersion(txn, fv.Function.Name, fv.FunctionVersion.Version); err != nil { + if err != errmsg.KeyNotFoundError { + log.GetLogger().Errorf("failed to delete function version, error: %s", err.Error()) + } + return err + } + + if err := CreateFunctionVersion(txn, fv); err != nil { + log.GetLogger().Errorf("failed to create function version, error: %s", err.Error()) + return err + } + return nil +} + +func addObjRefCntTx(txn Transaction, pkg Package) error { + if pkg.BucketID == "" || pkg.ObjectID == "" { + return nil + } + + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + key := genObjectRefIndexKey(t, pkg.BucketID, pkg.ObjectID) + val, err := txn.GetTxn().ObjectRefIndexGet(key) + if err != nil { + if err != errmsg.KeyNotFoundError { + log.GetLogger().Errorf("failed to get object reference index, error: %s", err.Error()) + return err + } + } + val.RefCnt++ + if err := txn.GetTxn().ObjectRefIndexPut(key, val); err != nil { + log.GetLogger().Errorf("failed to put object reference index, error: %s", err.Error()) + return err + } + return nil +} + +func delObjRefCntTx(txn Transaction, pkg Package) error { + if pkg.CodeUploadType != constants.S3StorageType { + return nil + } + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + key := genObjectRefIndexKey(t, pkg.BucketID, pkg.ObjectID) + val, err := txn.GetTxn().ObjectRefIndexGet(key) + if err != nil { + if err == errmsg.KeyNotFoundError { + return nil + } + log.GetLogger().Errorf("failed to get object reference index, error: %s", err.Error()) + return err + } + val.RefCnt-- + if val.RefCnt <= 0 { + if err := txn.GetTxn().ObjectRefIndexDelete(key); err != nil { + log.GetLogger().Errorf("failed to delete object reference index, error: %s", err.Error()) + return err + } + return nil + } + if err := txn.GetTxn().ObjectRefIndexPut(key, val); err != nil { + log.GetLogger().Errorf("failed to put object reference index, error: %s", err.Error()) + return err + } + return nil +} + +// IsObjectReferred returns whether an object is referred by other function version. +// This method should be called only in upload function package transaction. +func IsObjectReferred(txn Transaction, bucketID string, objectID string) (bool, error) { + if bucketID == "" || objectID == "" { + return false, nil + } + + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return false, err + } + val, err := txn.GetTxn().ObjectRefIndexGet(genObjectRefIndexKey(t, bucketID, objectID)) + if err != nil { + if err == errmsg.KeyNotFoundError { + return false, nil + } + log.GetLogger().Errorf("failed to get object reference index, error: %s", err.Error()) + return false, err + } + return val.RefCnt > 0, nil +} + +// GetFunctionVersionSizeByName returns size of function version entry by specified function name. +func GetFunctionVersionSizeByName(ctx server.Context, funcName string) (int64, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return 0, err + } + + res, err := db.FunctionVersionCount(ctx.Context(), GenFunctionVersionKey(t, funcName, "")) + if err != nil { + log.GetLogger().Errorf("failed to count function version, error: %s", err.Error()) + return 0, err + } + return res, nil +} + +// GetFunctionList returns a function version entry list by a fuzzy function name. +// Returned list is only the page of specified page index and size. +func GetFunctionList( + ctx server.Context, funcName string, funcVer string, kind string, pageIndex int, pageSize int, +) ([]FunctionVersionTuple, int, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return nil, 0, err + } + stream := db.FunctionVersionStream(ctx.Context(), GenFunctionVersionKey(t, "", ""), EngineSortBy) + if kind == constants.Faas { + stream = metaDB.FunctionVersionStream(ctx.Context(), GenFunctionVersionKey(t, "", ""), EngineSortBy) + } + + if funcVer != "" || funcName != "" { + stream = stream.Filter(func(key FunctionVersionKey, val FunctionVersionValue) bool { + if funcVer != "" && key.FunctionVersion != funcVer { + return false + } + if funcName != "" && !strings.Contains(val.Function.Name, funcName) { + return false + } + return true + }) + } + + tuples, total, err := stream.ExecuteWithPage(pageIndex, pageSize) + if err != nil { + if err == errmsg.KeyNotFoundError { + return nil, 0, nil + } + log.GetLogger().Errorf("failed to get function version stream with page, error: %s", err.Error()) + return nil, 0, err + } + return tuples, total, nil +} + +// GetFunctionVersionList returns function version entry list by specified function name. +// Returned list is only the page of specified page index and size. +func GetFunctionVersionList( + ctx server.Context, funcName string, funcVer string, pageIndex int, pageSize int, +) ([]FunctionVersionTuple, int, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return nil, 0, err + } + tuples, total, err := db. + FunctionVersionStream(ctx.Context(), GenFunctionVersionKey(t, funcName, funcVer), EngineSortBy). + ExecuteWithPage(pageIndex, pageSize) + if err != nil { + if err == errmsg.KeyNotFoundError { + return nil, 0, nil + } + log.GetLogger().Errorf("failed to get function version stream with page, error: %s", err.Error()) + return nil, 0, err + } + + return tuples, total, nil +} + +// GetFunctionByFunctionNameAndVersion gets function version entry by function name and version. +// It returns snerror(4115) if entry does not exist. +func GetFunctionByFunctionNameAndVersion(ctx server.Context, name string, + version string, kind string, +) (FunctionVersionValue, error) { + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return FunctionVersionValue{}, err + } + + currentDB := GetDB(kind) + val, err := currentDB.FunctionVersionGet(ctx.Context(), GenFunctionVersionKey(t, name, version)) + if err != nil { + if err == errmsg.KeyNotFoundError { + return FunctionVersionValue{}, errmsg.New(errmsg.FunctionNotFound, utils.RemoveServiceID(name)) + } + log.GetLogger().Errorf("failed to get function version, error: %s", err.Error()) + return FunctionVersionValue{}, err + } + + funcStatus, err := currentDB.FunctionStatusGet(ctx.Context(), genFunctionStatusKey(t, name, version)) + if err != nil { + if err == errmsg.KeyNotFoundError { + val.FunctionVersion.Status = constants.FunctionStatusUnavailable + return val, nil + } + log.GetLogger().Errorf("failed to get function version, error: %s", err.Error()) + return FunctionVersionValue{}, err + } + val.FunctionVersion.Status = funcStatus.Status + val.FunctionVersion.InstanceNum = funcStatus.InstanceNum + return val, nil +} + +// GetFunctionByFunctionNameAndAlias returns the function version from the alias version index table. +func GetFunctionByFunctionNameAndAlias(c server.Context, funcName string, aliasName string) string { + value, err := GetAliasValue(c, funcName, aliasName) + if err != nil { + return "" + } + return value.FunctionVersion +} + +// GetDB get db +func GetDB(kind string) *generatedKV { + if kind == constants.Faas { + return metaDB + } + return db +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/function_test.go b/functionsystem/apps/meta_service/function_repo/storage/function_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9aba2c84d6054078f93f2fb7ef65e72236b353be --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/function_test.go @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "context" + "errors" + "reflect" + "testing" + + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" + "meta_service/function_repo/test/fakecontext" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" +) + +func TestGetFunctionByFunctionNameAndAlias(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(GetAliasValue, func(_ server.Context, _, _ string) (AliasValue, error) { + return AliasValue{ + FunctionVersion: "v1.0", + }, nil + }) + assert.Equal(t, "v1.0", GetFunctionByFunctionNameAndAlias( + &fakecontext.Context{}, "", "")) + + patches.Reset() + patches.ApplyFunc(GetAliasValue, func(_ server.Context, _, _ string) (AliasValue, error) { + return AliasValue{}, errors.New("fake error! ") + }) + assert.Equal(t, "", GetFunctionByFunctionNameAndAlias( + &fakecontext.Context{}, "", "")) +} + +func TestDeleteFunctionStatus(t *testing.T) { + Convey("Test DeleteFunctionStatus with tenantInfo nil", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewContext(), + } + err := DeleteFunctionStatus(txn, "mockname", "mockvar") + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteFunctionStatus with delete err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionStatusDelete", + func(t *generatedTx, key FunctionStatusKey) error { + return errmsg.MarshalError + }) + defer patch.Reset() + err := DeleteFunctionStatus(txn, "mockname", "mockvar") + So(err, ShouldEqual, errmsg.MarshalError) + }) + Convey("Test DeleteFunctionStatus", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionStatusDelete", + func(t *generatedTx, key FunctionStatusKey) error { + return nil + }) + defer patch.Reset() + err := DeleteFunctionStatus(txn, "mockname", "mockvar") + So(err, ShouldEqual, nil) + }) +} + +func TestUpdateFunctionVersion(t *testing.T) { + functionVersionValue := FunctionVersionValue{ + Function: Function{Name: "mockName"}, + FunctionVersion: FunctionVersion{Version: "mockVersion"}, + FunctionLayer: nil, + } + Convey("Test UpdateFunctionVersion with mock Delete err", t, func() { + patch := gomonkey.ApplyFunc(DeleteFunctionVersion, func(txn Transaction, funcName string, funcVer string) error { + return errors.New("mock Delete err") + }) + defer patch.Reset() + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + err := UpdateFunctionVersion(txn, functionVersionValue) + So(err, ShouldNotBeNil) + }) + Convey("Test UpdateFunctionVersion with mock Create err", t, func() { + patch := gomonkey.ApplyFunc(DeleteFunctionVersion, func(txn Transaction, funcName string, funcVer string) error { + return nil + }).ApplyFunc(CreateFunctionVersion, func(txn Transaction, fv FunctionVersionValue) error { + return errors.New("mock create err") + }) + defer patch.Reset() + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + err := UpdateFunctionVersion(txn, functionVersionValue) + So(err, ShouldNotBeNil) + }) + Convey("Test UpdateFunctionVersion", t, func() { + patch := gomonkey.ApplyFunc(DeleteFunctionVersion, func(txn Transaction, funcName string, funcVer string) error { + return nil + }).ApplyFunc(CreateFunctionVersion, func(txn Transaction, fv FunctionVersionValue) error { + return nil + }) + defer patch.Reset() + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + err := UpdateFunctionVersion(txn, functionVersionValue) + So(err, ShouldBeNil) + }) +} + +func TestCreateFunctionVersion(t *testing.T) { + functionVersionValue := FunctionVersionValue{ + Function: Function{Name: "mockName"}, + FunctionVersion: FunctionVersion{Version: "mockVersion", Package: Package{CodeUploadType: "s3"}}, + FunctionLayer: []FunctionLayer{{Name: "mockLayer", Version: 0, Order: 0}}, + } + Convey("Test CreateFunctionVersion with tenant nil", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewContext(), + } + err := CreateFunctionVersion(txn, functionVersionValue) + So(err, ShouldNotBeNil) + }) + Convey("Test CreateFunctionVersion with FunctionVersionPut err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionVersionPut", + func(t *generatedTx, key FunctionVersionKey, val FunctionVersionValue) error { + return errors.New("mock FunctionVersionPut err") + }) + defer patch.Reset() + err := CreateFunctionVersion(txn, functionVersionValue) + So(err, ShouldNotBeNil) + }) + Convey("Test CreateFunctionVersion with FunctionLayer err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionVersionPut", + func(t *generatedTx, key FunctionVersionKey, val FunctionVersionValue) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "LayerFunctionIndexPut", + func(t *generatedTx, key LayerFunctionIndexKey, val LayerFunctionIndexValue) error { + return errors.New("mock FunctionLayer err") + }) + defer patch.Reset() + err := CreateFunctionVersion(txn, functionVersionValue) + So(err, ShouldNotBeNil) + }) + Convey("Test CreateFunctionVersion with addObjRefCntTx err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionVersionPut", + func(t *generatedTx, key FunctionVersionKey, val FunctionVersionValue) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "LayerFunctionIndexPut", + func(t *generatedTx, key LayerFunctionIndexKey, val LayerFunctionIndexValue) error { + return nil + }).ApplyFunc(addObjRefCntTx, func(txn Transaction, pkg Package) error { + return errors.New("mock addObjRefCntTx err") + }) + defer patch.Reset() + err := CreateFunctionVersion(txn, functionVersionValue) + So(err, ShouldNotBeNil) + }) + Convey("Test CreateFunctionVersion with local codeUploadType", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionVersionPut", + func(t *generatedTx, key FunctionVersionKey, val FunctionVersionValue) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "LayerFunctionIndexPut", + func(t *generatedTx, key LayerFunctionIndexKey, val LayerFunctionIndexValue) error { + return nil + }) + defer patch.Reset() + functionVersionValue.FunctionVersion.Package.CodeUploadType = "" + err := CreateFunctionVersion(txn, functionVersionValue) + So(err, ShouldBeNil) + }) +} + +func TestGetFunctionByFunctionNameAndVersion(t *testing.T) { + Convey("Test GetFunctionByFunctionNameAndVersion with tenant err", t, func() { + ctx := fakecontext.NewContext() + _, err := GetFunctionByFunctionNameAndVersion(ctx, "mockName", "mockVersion", "") + So(err, ShouldNotBeNil) + }) + Convey("Test GetFunctionByFunctionNameAndVersion with FunctionVersionGet err", t, func() { + ctx := fakecontext.NewMockContext() + dbBackup := db + db = &generatedKV{} + patch := gomonkey.ApplyMethod(reflect.TypeOf(db), "FunctionVersionGet", + func(kv *generatedKV, ctx context.Context, key FunctionVersionKey) (FunctionVersionValue, error) { + return FunctionVersionValue{}, errmsg.KeyNotFoundError + }) + defer func() { + patch.Reset() + db = dbBackup + }() + _, err := GetFunctionByFunctionNameAndVersion(ctx, "mockName", "mockVersion", "") + So(err, ShouldNotBeNil) + }) + Convey("Test GetFunctionByFunctionNameAndVersion with FunctionVersionGet err2", t, func() { + ctx := fakecontext.NewMockContext() + dbBackup := db + db = &generatedKV{} + patch := gomonkey.ApplyMethod(reflect.TypeOf(db), "FunctionVersionGet", + func(kv *generatedKV, ctx context.Context, key FunctionVersionKey) (FunctionVersionValue, error) { + return FunctionVersionValue{}, errors.New("mock err") + }) + defer func() { + patch.Reset() + db = dbBackup + }() + _, err := GetFunctionByFunctionNameAndVersion(ctx, "mockName", "mockVersion", "") + So(err, ShouldNotBeNil) + }) + Convey("Test GetFunctionByFunctionNameAndVersion with FunctionStatusGet err", t, func() { + ctx := fakecontext.NewMockContext() + dbBackup := db + db = &generatedKV{} + var mockErr error + mockErr = errmsg.KeyNotFoundError + patch := gomonkey.ApplyMethod(reflect.TypeOf(db), "FunctionVersionGet", + func(kv *generatedKV, ctx context.Context, key FunctionVersionKey) (FunctionVersionValue, error) { + return FunctionVersionValue{}, nil + }).ApplyMethod(reflect.TypeOf(db), "FunctionStatusGet", + func(kv *generatedKV, ctx context.Context, key FunctionStatusKey) (FunctionStatusValue, error) { + return FunctionStatusValue{}, mockErr + }) + defer func() { + patch.Reset() + db = dbBackup + }() + _, err := GetFunctionByFunctionNameAndVersion(ctx, "mockName", "mockVersion", "") + So(err, ShouldBeNil) + mockErr = errors.New("mock err") + _, err = GetFunctionByFunctionNameAndVersion(ctx, "mockName", "mockVersion", "") + So(err, ShouldNotBeNil) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/layer.go b/functionsystem/apps/meta_service/function_repo/storage/layer.go new file mode 100644 index 0000000000000000000000000000000000000000..4ee9a4624928479ad279bd0e49d8a98c68fdccf9 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/layer.go @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "sort" + + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" + + "meta_service/common/engine" + "meta_service/common/logger/log" +) + +// NewLayerVersion says a new layer begins with version 1. +const NewLayerVersion = 1 + +func genLayerCountIndexKey(t server.TenantInfo, layerName string) LayerCountIndexKey { + return LayerCountIndexKey{ + TenantInfo: t, + LayerName: layerName, + } +} + +func genLayerKey(t server.TenantInfo, layerName string, layerVersion int) LayerKey { + return LayerKey{ + TenantInfo: t, + LayerName: layerName, + LayerVersion: layerVersion, + } +} + +// GetLayerLatestVersion gets layer's newest version. It also returns the newest version number. +func GetLayerLatestVersion(ctx server.Context, layerName string) (LayerValue, int, error) { + tenant, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return LayerValue{}, 0, err + } + _, err = db.LayerCountIndexGet(ctx.Context(), genLayerCountIndexKey(tenant, layerName)) + if err != nil { + log.GetLogger().Debugf("failed to get layer count index, error: %s", err.Error()) + return LayerValue{}, 0, err + } + k, v, err := db.LayerLastInRange(ctx.Context(), genLayerKey(tenant, layerName, 0)) + if err != nil { + log.GetLogger().Debugf("failed to check layer last in range, error: %s", err.Error()) + return LayerValue{}, 0, err + } + return v, k.LayerVersion, nil +} + +// GetLayerVersion gets a specific layer by its name and version +func GetLayerVersion(ctx server.Context, layerName string, layerVersion int) (LayerValue, error) { + tenant, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return LayerValue{}, err + } + res, err := db.LayerGet(ctx.Context(), genLayerKey(tenant, layerName, layerVersion)) + if err != nil { + log.GetLogger().Debugf("failed to get layer, error: %s", err.Error()) + return LayerValue{}, err + } + return res, nil +} + +// GetLayerStream returns a range stream of layers +func GetLayerStream(ctx server.Context, layerName string) (*LayerPrepareStmt, error) { + tenant, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return nil, err + } + by := engine.SortBy{ + Order: engine.Descend, + Target: engine.SortModify, + } + return db.LayerStream(ctx.Context(), genLayerKey(tenant, layerName, 0), by), nil +} + +// GetLayerSizeAndLatestVersion is the same as "GetLayerLatestVersion" but in transaction. +func GetLayerSizeAndLatestVersion(txn *Txn, layerName string) (int, int, error) { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return 0, 0, err + } + _, err = txn.txn.LayerCountIndexGet(genLayerCountIndexKey(tenant, layerName)) + if err != nil { + log.GetLogger().Debugf("failed to get layer count index, error: %s", err.Error()) + return 0, 0, err + } + tuples, err := txn.txn.LayerGetRange(genLayerKey(tenant, layerName, 0)) + if err != nil { + log.GetLogger().Errorf("failed to get layers from range, error: %s", err.Error()) + return 0, 0, err + } + if len(tuples) == 0 { + return 0, 0, errmsg.KeyNotFoundError + } + sort.Slice(tuples, func(i, j int) bool { + return tuples[i].Key.LayerVersion > tuples[j].Key.LayerVersion + }) + return len(tuples), tuples[0].Key.LayerVersion, nil +} + +// CountLayerTx returns the number of all layers. +func CountLayerTx(txn *Txn) (int, error) { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return 0, err + } + tuples, err := txn.txn.LayerCountIndexGetRange(genLayerCountIndexKey(tenant, "")) + if err != nil { + log.GetLogger().Errorf("failed to get layer count indexes from range, error: %s", err.Error()) + return 0, err + } + return len(tuples), nil +} + +// GetLayerVersionTx is the same as "GetLayerVersion" but in transaction. +func GetLayerVersionTx(txn Transaction, layerName string, layerVersion int) (LayerValue, error) { + tenant, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return LayerValue{}, err + } + res, err := txn.GetTxn().LayerGet(genLayerKey(tenant, layerName, layerVersion)) + if err != nil { + log.GetLogger().Debugf("failed to get layer, error: %s", err.Error()) + return LayerValue{}, err + } + return res, nil +} + +// GetLayerTx retrieves all layer versions of a layer. +func GetLayerTx(txn *Txn, layerName string) ([]LayerTuple, error) { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return nil, err + } + tuples, err := txn.txn.LayerGetRange(genLayerKey(tenant, layerName, 0)) + if err != nil { + log.GetLogger().Errorf("failed to get layers from range, error: %s", err.Error()) + return nil, err + } + if len(tuples) == 0 { + return nil, errmsg.New(errmsg.LayerNotFound, layerName) + } + return tuples, nil +} + +// CreateLayer creates a specific layer version. +func CreateLayer(txn *Txn, layerName string, layerVersion int, layer LayerValue) error { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + if layerVersion == NewLayerVersion { + if err := txn.txn.LayerCountIndexPut(genLayerCountIndexKey(tenant, layerName), + LayerCountIndexValue{}); err != nil { + log.GetLogger().Errorf("failed to put layer count index, error: %s", err.Error()) + return err + } + } + + if err := txn.txn.LayerPut(genLayerKey(tenant, layerName, layerVersion), layer); err != nil { + log.GetLogger().Errorf("failed to put layer, error: %s", err.Error()) + return err + } + return nil +} + +// UpdateLayer updates a specific layer version. +func UpdateLayer(txn *Txn, layerName string, layerVersion int, layer LayerValue) error { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + if err := txn.txn.LayerPut(genLayerKey(tenant, layerName, layerVersion), layer); err != nil { + log.GetLogger().Errorf("failed to put layer, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteLayer deletes all layer versions. +func DeleteLayer(txn *Txn, layerName string) error { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + if err := txn.txn.LayerDeleteRange(genLayerKey(tenant, layerName, 0)); err != nil { + log.GetLogger().Errorf("failed to delete layers from range, error: %s", err.Error()) + return err + } + if err := txn.txn.LayerCountIndexDelete(genLayerCountIndexKey(tenant, layerName)); err != nil { + log.GetLogger().Errorf("failed to delete layer count index, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteLayerVersion deletes a specific layer version. +func DeleteLayerVersion(txn *Txn, layerName string, layerVersion int) error { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + if err := txn.txn.LayerDelete(genLayerKey(tenant, layerName, layerVersion)); err != nil { + log.GetLogger().Errorf("failed to delete layer, error: %s", err.Error()) + return err + } + tuples, err := txn.txn.LayerGetRange(genLayerKey(tenant, layerName, 0)) + if err != nil { + log.GetLogger().Errorf("failed to get layers from range, error: %s", err.Error()) + return err + } + if len(tuples) == 0 { + if err := txn.txn.LayerCountIndexDelete(genLayerCountIndexKey(tenant, layerName)); err != nil { + log.GetLogger().Errorf("failed to delete layer count index, error: %s", err.Error()) + return err + } + } + return nil +} + +// IsLayerUsed returns the first used layer version, 0 if layer is not being used. +func IsLayerUsed(txn *Txn, layerName string) (int, error) { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return 0, err + } + + prefix := genLayerFunctionIndexKey(tenant, layerName, 0, "", "") + tuples, err := txn.txn.LayerFunctionIndexGetRange(prefix) + if err != nil { + log.GetLogger().Errorf("failed to get layer function indexes from range, error: %s", err.Error()) + return 0, err + } + + if len(tuples) == 0 { + return 0, nil + } + + return tuples[0].Key.LayerVersion, nil +} + +// IsLayerVersionUsed returns true if the specific layer version is being used. +func IsLayerVersionUsed(txn *Txn, layerName string, layerVersion int) (bool, error) { + tenant, err := txn.c.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return false, err + } + + prefix := genLayerFunctionIndexKey(tenant, layerName, layerVersion, "", "") + tuples, err := txn.txn.LayerFunctionIndexGetRange(prefix) + if err != nil { + log.GetLogger().Errorf("failed to get layer function indexes from range, error: %s", err.Error()) + return false, err + } + return len(tuples) > 0, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/layer_test.go b/functionsystem/apps/meta_service/function_repo/storage/layer_test.go new file mode 100644 index 0000000000000000000000000000000000000000..75a2fd52bacd49351b4b50594d4fca2d563df1c9 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/layer_test.go @@ -0,0 +1,246 @@ +package storage + +import ( + "context" + "errors" + "reflect" + "testing" + + "meta_service/function_repo/test/fakecontext" + + "github.com/agiledragon/gomonkey" + . "github.com/smartystreets/goconvey/convey" +) + +func TestUpdateLayer(t *testing.T) { + Convey("Test UpdateLayer with tenant nil", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewContext(), + } + err := UpdateLayer(txn, "mockLayer", 1, LayerValue{}) + So(err, ShouldNotBeNil) + }) + Convey("Test UpdateLayer with LayerPut err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerPut", + func(t *generatedTx, key LayerKey, val LayerValue) error { + return errors.New("mock LayerPut err") + }) + defer patch.Reset() + err := UpdateLayer(txn, "mockLayer", 1, LayerValue{}) + So(err, ShouldNotBeNil) + }) +} + +func TestDeleteLayer(t *testing.T) { + Convey("Test DeleteLayer with tenant nil", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewContext(), + } + err := DeleteLayer(txn, "mockLayer") + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteLayer with LayerDeleteRange err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerDeleteRange", + func(t *generatedTx, prefix LayerKey) error { + return errors.New("mock LayerDeleteRange err") + }) + defer patch.Reset() + err := DeleteLayer(txn, "mockLayer") + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteLayer with LayerCountIndexDelete err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerDeleteRange", + func(t *generatedTx, prefix LayerKey) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "LayerCountIndexDelete", + func(t *generatedTx, key LayerCountIndexKey) error { + return errors.New("mock LayerCountIndexDelete err") + }) + defer patch.Reset() + err := DeleteLayer(txn, "mockLayer") + So(err, ShouldNotBeNil) + }) +} + +func TestDeleteLayerVersion(t *testing.T) { + Convey("Test DeleteLayerVersion with tenant nil", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewContext(), + } + err := DeleteLayerVersion(txn, "mockLayer", 0) + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteLayerVersion with LayerDelete err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerDelete", + func(t *generatedTx, key LayerKey) error { + return errors.New("mock LayerDelete err") + }) + defer patch.Reset() + + err := DeleteLayerVersion(txn, "mockLayer", 0) + So(err, ShouldNotBeNil) + }) + + Convey("Test DeleteLayerVersion with LayerGetRange err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerDelete", + func(t *generatedTx, key LayerKey) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "LayerGetRange", + func(t *generatedTx, _ LayerKey) ([]LayerTuple, error) { + return []LayerTuple{}, errors.New("mock LayerGetRange err") + }) + defer patch.Reset() + + err := DeleteLayerVersion(txn, "mockLayer", 0) + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteLayerVersion with LayerCountIndexDelete err", t, func() { + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerDelete", + func(t *generatedTx, key LayerKey) error { + return nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "LayerGetRange", + func(t *generatedTx, _ LayerKey) ([]LayerTuple, error) { + return []LayerTuple{}, nil + }).ApplyMethod(reflect.TypeOf(txn.txn), "LayerCountIndexDelete", + func(t *generatedTx, key LayerCountIndexKey) error { + return errors.New("mock LayerCountIndexDelete err") + }) + defer patch.Reset() + + err := DeleteLayerVersion(txn, "mockLayer", 0) + So(err, ShouldNotBeNil) + }) +} + +func TestHTTPTriggerEtcdSpecDelete(t *testing.T) { + Convey("Test HTTPTriggerEtcdSpec.delete with tenant nil", t, func() { + httpTriggerEtcdSpec := &HTTPTriggerEtcdSpec{} + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewContext(), + } + err := httpTriggerEtcdSpec.delete(txn, "mockFunc", "mockVerOrAlias") + So(err, ShouldNotBeNil) + }) + Convey("Test HTTPTriggerEtcdSpec.delete with FunctionResourceIDIndexDelete err", t, func() { + httpTriggerEtcdSpec := &HTTPTriggerEtcdSpec{} + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionResourceIDIndexDelete", + func(t *generatedTx, key FunctionResourceIDIndexKey) error { + return errors.New("mock FunctionResourceIDIndexDelete err") + }) + defer patch.Reset() + err := httpTriggerEtcdSpec.delete(txn, "mockFunc", "mockVerOrAlias") + So(err, ShouldNotBeNil) + }) + Convey("Test HTTPTriggerEtcdSpec.delete with FunctionResourceIDIndexDeleteRange err", t, func() { + httpTriggerEtcdSpec := &HTTPTriggerEtcdSpec{} + txn := &Txn{ + txn: &generatedTx{}, + c: fakecontext.NewMockContext(), + } + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "FunctionResourceIDIndexDeleteRange", + func(t *generatedTx, prefix FunctionResourceIDIndexKey) error { + return errors.New("mock FunctionResourceIDIndexDeleteRange err") + }) + defer patch.Reset() + err := httpTriggerEtcdSpec.delete(txn, "mockFunc", "") + So(err, ShouldNotBeNil) + }) +} + +func TestCreateLayer(t *testing.T) { + Convey("Test CreateLayer with TenantInfo err", t, func() { + txn := &Txn{txn: &generatedTx{}, c: fakecontext.NewContext()} + err := CreateLayer(txn, "mock-layer", 1, LayerValue{}) + So(err, ShouldNotBeNil) + }) + Convey("Test CreateLayer with LayerCountIndexPut err", t, func() { + txn := &Txn{txn: &generatedTx{}, c: fakecontext.NewMockContext()} + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerCountIndexPut", func( + t *generatedTx, key LayerCountIndexKey, val LayerCountIndexValue, + ) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := CreateLayer(txn, "mock-layer", 1, LayerValue{}) + So(err, ShouldNotBeNil) + }) + Convey("Test CreateLayer with LayerPut err", t, func() { + txn := &Txn{txn: &generatedTx{}, c: fakecontext.NewMockContext()} + patch := gomonkey.ApplyMethod(reflect.TypeOf(txn.txn), "LayerPut", func( + t *generatedTx, key LayerKey, val LayerValue, + ) error { + return errors.New("mock err") + }) + defer patch.Reset() + err := CreateLayer(txn, "mock-layer", 2, LayerValue{}) + So(err, ShouldNotBeNil) + }) +} + +func TestGetLayerLatestVersion(t *testing.T) { + Convey("Test GetLayerLatestVersion", t, func() { + Convey("with TenantInfo err", func() { + ctx := fakecontext.NewContext() + _, _, err := GetLayerLatestVersion(ctx, "mock-layer-name") + So(err, ShouldNotBeNil) + }) + Convey("with LayerCountIndexGet err", func() { + ctx := fakecontext.NewMockContext() + patches := gomonkey.ApplyGlobalVar(&db, &generatedKV{}).ApplyMethod(reflect.TypeOf(&generatedKV{}), "LayerCountIndexGet", func( + kv *generatedKV, ctx context.Context, key LayerCountIndexKey, + ) (LayerCountIndexValue, error) { + return LayerCountIndexValue{}, errors.New("mock err") + }) + defer patches.Reset() + _, _, err := GetLayerLatestVersion(ctx, "mock-layer-name") + So(err, ShouldNotBeNil) + }) + Convey("with LayerLastInRange err", func() { + ctx := fakecontext.NewMockContext() + patches := gomonkey.ApplyGlobalVar(&db, &generatedKV{}).ApplyMethod(reflect.TypeOf(&generatedKV{}), "LayerCountIndexGet", func( + kv *generatedKV, ctx context.Context, key LayerCountIndexKey, + ) (LayerCountIndexValue, error) { + return LayerCountIndexValue{}, nil + }).ApplyMethod(reflect.TypeOf(&generatedKV{}), "LayerLastInRange", func( + kv *generatedKV, ctx context.Context, prefix LayerKey, + ) (LayerKey, LayerValue, error) { + return LayerKey{}, LayerValue{}, errors.New("mock err") + }) + defer patches.Reset() + _, _, err := GetLayerLatestVersion(ctx, "mock-layer-name") + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/podpool.go b/functionsystem/apps/meta_service/function_repo/storage/podpool.go new file mode 100644 index 0000000000000000000000000000000000000000..6c59bd720a216cd2e510d284fda7f0bc4b43cec7 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/podpool.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "strings" + + "meta_service/function_repo/errmsg" + "meta_service/function_repo/server" + "meta_service/function_repo/utils/constants" + + "meta_service/common/logger/log" +) + +func buildPodPoolRegisterKey(id string) string { + // format: /yr/podpools/info// + keys := []string{constants.PodPoolPrefix} + keys = append(keys, id) + return strings.Join(keys, constants.ETCDKeySeparator) +} + +// DeletePodPool delete a pod pool. +func DeletePodPool(txn *Txn, id string) { + key := buildPodPoolRegisterKey(id) + txn.Delete(key) + return +} + +// GetPodPoolList get pool pool list by id/group +func GetPodPoolList(ctx server.Context, id string, group string, pageIndex int, pageSize int) ([]PoolTuple, int, + error, +) { + stream := db.PoolStream(ctx.Context(), PoolKey{ID: id}, EngineSortBy) + if id != "" || group != "" { + stream = stream.Filter(func(key PoolKey, val PoolValue) bool { + if id != "" && val.Id != id { + return false + } + if group != "" && val.Group != group { + return false + } + return true + }) + } + var tuples []PoolTuple + var err error + total := 0 + if pageIndex >= 0 && pageSize >= 0 { + tuples, total, err = stream.ExecuteWithPage(pageIndex, pageSize) + } else { + tuples, err = stream.Execute() + } + if err != nil { + if err == errmsg.KeyNotFoundError { + return nil, 0, nil + } + log.GetLogger().Errorf("failed to get pool stream with page, error: %s", err.Error()) + return nil, 0, err + } + return tuples, total, nil +} + +// CreateUpdatePodPool create or update pod pool +func CreateUpdatePodPool(txn Transaction, value PoolValue) error { + if err := txn.GetTxn().PoolPut(PoolKey{ID: value.Id}, value); err != nil { + log.GetLogger().Errorf("failed to put pool info, error: %s", err.Error()) + return err + } + return nil +} + +// GetPodPool get a pod pool. +func GetPodPool(txn Transaction, id string) (PoolValue, error) { + res, err := txn.GetTxn().PoolGet(PoolKey{ID: id}) + if err != nil { + log.GetLogger().Warnf("failed to get pod pool, error: %s", err.Error()) + return PoolValue{}, err + } + return res, nil +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/publish/builder.go b/functionsystem/apps/meta_service/function_repo/storage/publish/builder.go new file mode 100644 index 0000000000000000000000000000000000000000..c248a197916cee5596ff679a874da1bcb94d4fa1 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/publish/builder.go @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package publish + +import ( + "strings" + + "meta_service/function_repo/server" + "meta_service/function_repo/utils/constants" + + "meta_service/common/urnutils" +) + +func appendTenantInfo(keys []string, info server.TenantInfo) []string { + keys = append(keys, constants.BusinessKey, info.BusinessID, constants.TenantKey) + if info.ProductID != "" { + keys = append(keys, info.TenantID+urnutils.TenantProductSplitStr+info.ProductID) + } else { + keys = append(keys, info.TenantID) + } + return keys +} + +// BuildFunctionRegisterKey returns the etcd key of the registered function information. +func BuildFunctionRegisterKey(info server.TenantInfo, name, funcVersion, kind string) string { + // format(faas): /sn/functions/business//tenant//function//version/ + // format(yrlib): /yr/functions/business//tenant//function//version/ + keys := []string{constants.YRFunctionPrefix} + if kind == constants.Faas { + keys = []string{constants.FunctionPrefix} + } + keys = appendTenantInfo(keys, info) + keys = append(keys, constants.ResourceKey, name) + if name == "" { + return strings.Join(keys, constants.ETCDKeySeparator) + } + keys = append(keys, constants.VersionKey, funcVersion) + return strings.Join(keys, constants.ETCDKeySeparator) +} + +// BuildInstanceRegisterKey return the etcd key of the registered function instance meta information. +func BuildInstanceRegisterKey(info server.TenantInfo, name, funcVersion, clusterID string) string { + // format: /instances/business/yrk/cluster/cluster001/tenant//function//version/ + keys := []string{ + constants.InstancePrefix, constants.BusinessKey, "yrk", constants.ClusterKey, clusterID, + constants.TenantKey, + } + keys = append(keys, info.TenantID, constants.ResourceKey, name) + if name == "" { + return strings.Join(keys, constants.ETCDKeySeparator) + } + keys = append(keys, constants.VersionKey, funcVersion) + return strings.Join(keys, constants.ETCDKeySeparator) +} + +// BuildTriggerRegisterKey return the etcd key of the registered Trigger information. +func BuildTriggerRegisterKey( + info server.TenantInfo, funcName, triggerType, verOrAlias, triggerID string, +) string { + // format: /sn/triggers/triggerType//business//tenant/\ + // /function//version// + keys := []string{constants.TriggerPrefix, constants.TriggerTypeKey, triggerType} + if triggerType == "" { + return strings.Join(keys, constants.ETCDKeySeparator) + } + keys = appendTenantInfo(keys, info) + keys = append(keys, constants.ResourceKey, funcName) + if funcName == "" { + return strings.Join(keys, constants.ETCDKeySeparator) + } + keys = append(keys, constants.VersionKey, verOrAlias) + if verOrAlias == "" { + return strings.Join(keys, constants.ETCDKeySeparator) + } + keys = append(keys, triggerID) + return strings.Join(keys, constants.ETCDKeySeparator) +} + +// BuildAliasRegisterKey return the etcd key of the registered Alias information. +func BuildAliasRegisterKey(info server.TenantInfo, funcName, aliasName string) string { + // format: /sn/aliases/business//Tenant//function// + keys := []string{constants.AliasPrefix} + keys = appendTenantInfo(keys, info) + keys = append(keys, constants.ResourceKey, funcName) + if funcName == "" { + return strings.Join(keys, constants.ETCDKeySeparator) + } + keys = append(keys, aliasName) + return strings.Join(keys, constants.ETCDKeySeparator) +} + +// BuildTraceChainRegisterKey return the etcd key of the registered TraceChain information. +func BuildTraceChainRegisterKey(name string, info server.TenantInfo) string { + // format: /sn/functionchains/business//Tenant//function/ + keys := []string{constants.ChainPrefix} + keys = appendTenantInfo(keys, info) + keys = append(keys, constants.ResourceKey, name) + return strings.Join(keys, constants.ETCDKeySeparator) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/publish/builder_test.go b/functionsystem/apps/meta_service/function_repo/storage/publish/builder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d3196de7591cd3d966f981be18ba6d6449c98cff --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/publish/builder_test.go @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package publish + +import ( + "testing" + + "meta_service/function_repo/server" + + "github.com/stretchr/testify/assert" +) + +func genTenantInfo(productID string) server.TenantInfo { + return server.TenantInfo{ + BusinessID: "a", + TenantID: "a", + ProductID: productID, + } +} + +func TestBuildFunctionRegisterKey(t *testing.T) { + info := genTenantInfo("") + key := BuildFunctionRegisterKey(info, "a", "", "faas") + assert.Equal(t, "/sn/functions/business/a/tenant/a/function/a/version/", key) + key = BuildFunctionRegisterKey(info, "a", "", "yrlib") + assert.Equal(t, "/yr/functions/business/a/tenant/a/function/a/version/", key) +} + +func TestBuildFunctionRegisterKeyWithProductID(t *testing.T) { + info := genTenantInfo("a") + key := BuildFunctionRegisterKey(info, "a", "", "faas") + assert.Equal(t, "/sn/functions/business/a/tenant/a@a/function/a/version/", key) +} + +func TestBuildFunctionRegisterKeyWithVersion(t *testing.T) { + info := genTenantInfo("") + key := BuildFunctionRegisterKey(info, "a", "a", "faas") + assert.Equal(t, "/sn/functions/business/a/tenant/a/function/a/version/a", key) +} + +func TestBuildFunctionRegisterKeyWithoutName(t *testing.T) { + info := genTenantInfo("") + key := BuildFunctionRegisterKey(info, "", "a", "faas") + assert.Equal(t, "/sn/functions/business/a/tenant/a/function/", key) +} + +func TestBuildTriggerRegisterKey(t *testing.T) { + info := genTenantInfo("") + key := BuildTriggerRegisterKey(info, "a", "a", "a", "a") + assert.Equal(t, "/sn/triggers/triggerType/a/business/a/tenant/a/function/a/version/a/a", key) +} + +func TestBuildTriggerRegisterKeyWithProductID(t *testing.T) { + info := genTenantInfo("a") + key := BuildTriggerRegisterKey(info, "a", "a", "a", "a") + assert.Equal(t, "/sn/triggers/triggerType/a/business/a/tenant/a@a/function/a/version/a/a", key) +} + +func TestBuildTriggerRegisterKeyWithoutVersion(t *testing.T) { + info := genTenantInfo("") + key := BuildTriggerRegisterKey(info, "a", "a", "", "") + assert.Equal(t, "/sn/triggers/triggerType/a/business/a/tenant/a/function/a/version/", key) +} + +func TestBuildTriggerRegisterKeyWithoutTriggerID(t *testing.T) { + info := genTenantInfo("") + key := BuildTriggerRegisterKey(info, "a", "a", "a", "") + assert.Equal(t, "/sn/triggers/triggerType/a/business/a/tenant/a/function/a/version/a/", key) +} + +func TestBuildTriggerRegisterKeyWithoutVersionWithTriggerID(t *testing.T) { + info := genTenantInfo("") + key := BuildTriggerRegisterKey(info, "a", "a", "", "a") + assert.Equal(t, "/sn/triggers/triggerType/a/business/a/tenant/a/function/a/version/", key) +} + +func TestBuildTriggerRegisterKeyWithoutTriggerType(t *testing.T) { + info := genTenantInfo("") + key := BuildTriggerRegisterKey(info, "a", "", "a", "a") + assert.Equal(t, "/sn/triggers/triggerType/", key) +} + +func TestBuildTriggerRegisterKeyWithoutName(t *testing.T) { + info := genTenantInfo("") + key := BuildTriggerRegisterKey(info, "", "a", "a", "a") + assert.Equal(t, "/sn/triggers/triggerType/a/business/a/tenant/a/function/", key) +} + +func TestBuildAliasRegisterKey(t *testing.T) { + info := genTenantInfo("") + key := BuildAliasRegisterKey(info, "a", "a") + assert.Equal(t, "/sn/aliases/business/a/tenant/a/function/a/a", key) +} + +func TestBuildAliasRegisterKeyWithoutAliasName(t *testing.T) { + info := genTenantInfo("") + key := BuildAliasRegisterKey(info, "a", "") + assert.Equal(t, "/sn/aliases/business/a/tenant/a/function/a/", key) +} + +func TestBuildAliasRegisterKeyWithoutName(t *testing.T) { + info := genTenantInfo("") + key := BuildAliasRegisterKey(info, "", "a") + assert.Equal(t, "/sn/aliases/business/a/tenant/a/function/", key) +} + +func TestBuildAliasRegisterKeyWithProductID(t *testing.T) { + info := genTenantInfo("a") + key := BuildAliasRegisterKey(info, "a", "a") + assert.Equal(t, "/sn/aliases/business/a/tenant/a@a/function/a/a", key) +} + +func TestBuildTraceChainRegisterKey(t *testing.T) { + info := genTenantInfo("") + key := BuildTraceChainRegisterKey("a", info) + assert.Equal(t, "/sn/functionchains/business/a/tenant/a/function/a", key) +} + +func TestBuildTraceChainRegisterKeyWithProductID(t *testing.T) { + info := genTenantInfo("a") + key := BuildTraceChainRegisterKey("a", info) + assert.Equal(t, "/sn/functionchains/business/a/tenant/a@a/function/a", key) +} + +func TestBuildTraceChainRegisterKeyWithoutName(t *testing.T) { + info := genTenantInfo("a") + key := BuildTraceChainRegisterKey("", info) + assert.Equal(t, "/sn/functionchains/business/a/tenant/a@a/function/", key) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/publish/publish.go b/functionsystem/apps/meta_service/function_repo/storage/publish/publish.go new file mode 100644 index 0000000000000000000000000000000000000000..ce594bc5068b6a587243e3104873027c832533f0 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/publish/publish.go @@ -0,0 +1,568 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package publish + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + + common "meta_service/common/constants" + "meta_service/common/functionhandler" + "meta_service/common/logger/log" + "meta_service/common/metadata" + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/utils" + "meta_service/function_repo/utils/constants" +) + +const ( + // the length of envMap + envMapLength = 50 + // default init handler timeout + defaultInitializerTimeout = 30 + // default pre_stop handler timeout + defaultPreStopTimeout = 0 + maxPreStopTimeout = 180 +) + +// AddTrigger published a create event of trigger by function name, version and trigger id in a transaction. +func AddTrigger(txn *storage.Txn, funcName, verOrAlias string, info storage.TriggerValue, value interface{}) error { + ctx := txn.GetCtx() + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + tKey := BuildTriggerRegisterKey(t, funcName, info.TriggerType, verOrAlias, info.TriggerID) + b, err := json.Marshal(value) + if err != nil { + log.GetLogger().Errorf("failed to marshal version value: %s", err.Error()) + return errmsg.MarshalError + } + txn.Put(tKey, string(b)) + return nil +} + +// DeleteTrigger published a delete event of trigger by function name, version and trigger id in a transaction. +func DeleteTrigger(txn storage.Transaction, funcName, verOrAlias string, info storage.TriggerValue) error { + ctx := txn.GetCtx() + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + tKey := BuildTriggerRegisterKey(t, funcName, info.TriggerType, verOrAlias, info.TriggerID) + txn.Delete(tKey) + return nil +} + +// DeleteTriggerByFuncName published a delete event of trigger by function name in a transaction. +func DeleteTriggerByFuncName(txn storage.Transaction, funcName string) error { + ctx := txn.GetCtx() + t, err := ctx.TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + tKey := BuildTriggerRegisterKey(t, funcName, model.HTTPType, "", "") + txn.DeleteRange(tKey) + return nil +} + +// SavePublishFuncVersion save the published function version value +func SavePublishFuncVersion(txn storage.Transaction, + funcVersionValue storage.FunctionVersionValue, +) error { + info, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenantInfo error: %s", err.Error()) + return err + } + + key := BuildFunctionRegisterKey(info, funcVersionValue.Function.Name, + funcVersionValue.FunctionVersion.Version, funcVersionValue.FunctionVersion.Kind) + value, err := getVersionValue(info, txn, funcVersionValue) + if err != nil { + return err + } + + txn.Put(key, string(value)) + return nil +} + +func getVersionValue(info server.TenantInfo, txn storage.Transaction, + funcVersionValue storage.FunctionVersionValue, +) ([]byte, error) { + var ( + value interface{} + err error + ) + if funcVersionValue.FunctionVersion.Kind == common.Faas { + value, err = buildFaaSFunctionVersionValue(txn, funcVersionValue, info) + if err != nil { + log.GetLogger().Errorf("failed to buildFaaSFunctionVersionValue: %s", err.Error()) + return nil, err + } + } else { + value, err = buildFunctionVersionValue(txn, funcVersionValue, info) + if err != nil { + log.GetLogger().Errorf("failed to buildFunctionVersionValue: %s", err.Error()) + return nil, err + } + } + versionValue, err := json.Marshal(value) + if err != nil { + log.GetLogger().Errorf("failed to marshal version value: %s", err.Error()) + return nil, errmsg.MarshalError + } + return versionValue, nil +} + +func buildFaaSFunctionVersionValue(txn storage.Transaction, fv storage.FunctionVersionValue, + tenantInfo server.TenantInfo, +) (metadata.FaaSFuncMeta, error) { + var info metadata.FaaSFuncMeta + if err := buildEnv(fv, &info.EnvMetaData); err != nil { + log.GetLogger().Errorf("failed to build environment: %s", err.Error()) + return info, err + } + return buildFaaSFuncMetaData(txn, fv, tenantInfo, info) +} + +func buildFaaSFuncMetaData(txn storage.Transaction, fv storage.FunctionVersionValue, tenantInfo server.TenantInfo, + info metadata.FaaSFuncMeta, +) (metadata.FaaSFuncMeta, error) { + var err error + info.FuncMetaData.Name = fv.Function.Name + info.FuncMetaData.TenantID = tenantInfo.TenantID + info.FuncMetaData.BusinessID = tenantInfo.BusinessID + info.FuncMetaData.FunctionDescription = fv.Function.Description + info.FuncMetaData.FunctionURN, tenantInfo.TenantID = buildFunctionURN(tenantInfo, fv) + info.FuncMetaData.FunctionVersionURN = utils.BuildFunctionVersionURN(tenantInfo.BusinessID, tenantInfo.TenantID, + tenantInfo.ProductID, fv.Function.Name, fv.FunctionVersion.Version) + info.FuncMetaData.ReversedConcurrency = fv.Function.ReversedConcurrency + info.FuncMetaData.CreationTime = fv.Function.CreateTime + info.FuncMetaData.Handler = fv.FunctionVersion.Handler + info.FuncMetaData.Runtime = fv.FunctionVersion.Runtime + info.FuncMetaData.Tags = fv.Function.Tag + info.FuncMetaData.RevisionID = fv.FunctionVersion.RevisionID + info.FuncMetaData.CodeSize = int(fv.FunctionVersion.Package.Size) + info.FuncMetaData.CodeSha512 = fv.FunctionVersion.Package.Signature + info.FuncMetaData.Timeout = fv.FunctionVersion.Timeout + info.FuncMetaData.Version = fv.FunctionVersion.Version + info.FuncMetaData.FuncName = fv.FunctionVersion.FuncName + info.FuncMetaData.Service = fv.FunctionVersion.Service + info.FuncMetaData.VersionDescription = fv.FunctionVersion.Description + info.FuncMetaData.IsStatefulFunction = fv.FunctionVersion.StatefulFlag != 0 + info.FuncMetaData.Layers, err = getFaaSLayerBucket(storage.GetTxnByKind(txn.GetCtx(), ""), + fv.FunctionLayer, tenantInfo) + if err != nil { + log.GetLogger().Errorf("failed to get bucket layer info: %s", err.Error()) + return metadata.FaaSFuncMeta{}, err + } + info.FuncMetaData.IsBridgeFunction = false + info.FuncMetaData.EnableAuthInHeader = false + info.CodeMetaData, err = buildCodeMetaData(fv, tenantInfo.BusinessID) + if err != nil { + log.GetLogger().Errorf("failed to get bucket layer info: %s", err.Error()) + return metadata.FaaSFuncMeta{}, err + } + info.ResourceMetaData.CPU = fv.FunctionVersion.CPU + info.ResourceMetaData.Memory = fv.FunctionVersion.Memory + info.ResourceMetaData.CustomResources = fv.FunctionVersion.CustomResources + info.ResourceMetaData.EnableDynamicMemory = false + info.ExtendedMetaData.Initializer = getInitializer(fv.FunctionVersion.ExtendedHandler, + fv.FunctionVersion.ExtendedTimeout) + info.ExtendedMetaData.PreStop = getPreStop(fv.FunctionVersion.ExtendedHandler, + fv.FunctionVersion.ExtendedTimeout) + err = checkPreStopTime(info.ExtendedMetaData.PreStop) + if err != nil { + return metadata.FaaSFuncMeta{}, err + } + err = buildFaaSInstanceMetaData(txn, fv, tenantInfo, &info.InstanceMetaData) + if err != nil { + log.GetLogger().Errorf("failed to build faas instance meta data, error: %s", err.Error()) + return metadata.FaaSFuncMeta{}, err + } + return info, nil +} + +func buildFaaSInstanceMetaData(txn storage.Transaction, fv storage.FunctionVersionValue, tenantInfo server.TenantInfo, + instance *metadata.FaaSInstanceMetaData, +) error { + instance.MaxInstance = fv.FunctionVersion.MaxInstance + instance.MinInstance = fv.FunctionVersion.MinInstance + instance.ConcurrentNum = fv.FunctionVersion.ConcurrentNum + instance.PoolID = fv.FunctionVersion.PoolID + instance.PoolLabel = fv.FunctionVersion.PoolLabel + instance.IdleMode = false + key := BuildInstanceRegisterKey(tenantInfo, fv.Function.Name, + fv.FunctionVersion.Version, constants.DefaultClusterID) + data := make(map[string]interface{}) + data["instanceMetaData"] = *instance + value, err := json.Marshal(data) + if err != nil { + log.GetLogger().Errorf("failed to marshal version value: %s", err.Error()) + return err + } + txn.Put(key, string(value)) + return nil +} + +func getFaaSLayerBucket(txn storage.Transaction, layers []storage.FunctionLayer, + info server.TenantInfo, +) ([]*metadata.FaaSLayer, error) { + res := make([]*metadata.FaaSLayer, len(layers), len(layers)) + for i, layer := range layers { + val, bucket, err := getBucketInfo(txn, layer, info) + if err != nil { + return nil, err + } + res[i] = &metadata.FaaSLayer{ + BucketURL: bucket.URL, + ObjectID: val.Package.ObjectID, + BucketID: val.Package.BucketID, + AppID: bucket.AppID, + Sha256: val.Package.Signature, + } + } + return res, nil +} + +func getInitializer(extendHandler map[string]string, extendTimeout map[string]int) metadata.Initializer { + if len(extendHandler) == 0 { + return metadata.Initializer{} + } + initHandler, ok := extendHandler[functionhandler.ExtendedInitializer] + if !ok { + return metadata.Initializer{} + } + + if len(extendTimeout) == 0 { + return metadata.Initializer{Handler: initHandler, Timeout: int64(defaultInitializerTimeout)} + } + initTimeout, ok := extendTimeout[functionhandler.ExtendedInitializer] + if !ok { + return metadata.Initializer{Handler: initHandler, Timeout: int64(defaultInitializerTimeout)} + } + + return metadata.Initializer{ + Handler: initHandler, + Timeout: int64(initTimeout), + } +} + +func getPreStop(extendHandler map[string]string, extendTimeout map[string]int) metadata.PreStop { + if len(extendHandler) == 0 { + return metadata.PreStop{} + } + preStopHandler, ok := extendHandler[functionhandler.ExtendedPreStop] + if !ok { + return metadata.PreStop{} + } + + if len(extendTimeout) == 0 { + return metadata.PreStop{Handler: preStopHandler, Timeout: int64(defaultPreStopTimeout)} + } + + preStopTimeout, ok := extendTimeout[functionhandler.ExtendedPreStop] + if !ok { + return metadata.PreStop{Handler: preStopHandler, Timeout: int64(defaultPreStopTimeout)} + } + + return metadata.PreStop{ + Handler: preStopHandler, + Timeout: int64(preStopTimeout), + } +} + +func checkPreStopTime(preStop metadata.PreStop) error { + if preStop.Timeout > maxPreStopTimeout || preStop.Timeout < defaultPreStopTimeout { + return errors.New("preStop timeOut must between 0 and 180") + } + return nil +} + +func buildFunctionVersionValue(txn storage.Transaction, fv storage.FunctionVersionValue, + tenantInfo server.TenantInfo, +) (metadata.Function, error) { + var info metadata.Function + if err := buildEnv(fv, &info.EnvMetaData); err != nil { + log.GetLogger().Errorf("failed to build environment: %s", err.Error()) + return info, err + } + return buildFuncMetaData(txn, fv, tenantInfo, info) +} + +func buildFuncMetaData(txn storage.Transaction, fv storage.FunctionVersionValue, tenantInfo server.TenantInfo, + info metadata.Function, +) (metadata.Function, error) { + info.FuncMetaData.Name = fv.Function.Name + info.FuncMetaData.FunctionDescription = fv.Function.Description + info.FuncMetaData.FunctionURN, tenantInfo.TenantID = buildFunctionURN(tenantInfo, fv) + info.FuncMetaData.TenantID = tenantInfo.TenantID + info.FuncMetaData.BusinessID = tenantInfo.BusinessID + info.FuncMetaData.FunctionVersionURN = utils.BuildFunctionVersionURN(tenantInfo.BusinessID, tenantInfo.TenantID, + tenantInfo.ProductID, fv.Function.Name, fv.FunctionVersion.Version) + info.FuncMetaData.ReversedConcurrency = fv.Function.ReversedConcurrency + info.FuncMetaData.CreationTime = fv.Function.CreateTime + info.FuncMetaData.Tags = fv.Function.Tag + info.FuncMetaData.RevisionID = fv.FunctionVersion.RevisionID + info.FuncMetaData.CodeSize = fv.FunctionVersion.Package.Size + info.FuncMetaData.CodeSha512 = fv.FunctionVersion.Package.Signature + info.FuncMetaData.Handler = fv.FunctionVersion.Handler + info.ResourceMetaData.CPU = fv.FunctionVersion.CPU + info.ResourceMetaData.Memory = fv.FunctionVersion.Memory + info.ResourceMetaData.CustomResources = fv.FunctionVersion.CustomResources + info.FuncMetaData.Runtime = fv.FunctionVersion.Runtime + info.FuncMetaData.Timeout = fv.FunctionVersion.Timeout + info.FuncMetaData.Version = fv.FunctionVersion.Version + info.FuncMetaData.VersionDescription = fv.FunctionVersion.Description + info.FuncMetaData.StatefulFlag = fv.FunctionVersion.StatefulFlag != 0 + info.FuncMetaData.HookHandler = fv.FunctionVersion.HookHandler + var err error + info.CodeMetaData, err = buildCodeMetaData(fv, tenantInfo.BusinessID) + if err != nil { + log.GetLogger().Errorf("failed to get bucket layer info: %s", err.Error()) + return metadata.Function{}, err + } + info.FuncMetaData.Layers, err = getLayerBucket(txn, fv.FunctionLayer, tenantInfo) + if err != nil { + log.GetLogger().Errorf("failed to get bucket layer info: %s", err.Error()) + return metadata.Function{}, err + } + info.ExtendedMetaData.InstanceMetaData.MaxInstance = fv.FunctionVersion.MaxInstance + info.ExtendedMetaData.InstanceMetaData.MinInstance = fv.FunctionVersion.MinInstance + info.ExtendedMetaData.InstanceMetaData.ConcurrentNum = fv.FunctionVersion.ConcurrentNum + info.ExtendedMetaData.InstanceMetaData.CacheInstance = fv.FunctionVersion.CacheInstance + info.ExtendedMetaData.ExtendedHandler = fv.FunctionVersion.ExtendedHandler + info.ExtendedMetaData.ExtendedTimeout = fv.FunctionVersion.ExtendedTimeout + info.ExtendedMetaData.Device = fv.FunctionVersion.Device + return info, nil +} + +func buildCodeMetaData(fv storage.FunctionVersionValue, businessID string) (metadata.CodeMetaData, error) { + codeMetaData := metadata.CodeMetaData{ + CodeUploadType: fv.FunctionVersion.Package.CodeUploadType, + Sha512: fv.FunctionVersion.Package.Signature, + } + codeMetaData.StorageType = fv.FunctionVersion.Package.StorageType + if codeMetaData.StorageType == common.LocalStorageType || codeMetaData.StorageType == common.CopyStorageType { + codeMetaData.LocalMetaData.CodePath = fv.FunctionVersion.Package.CodePath + } else { + pkg := fv.FunctionVersion.Package + codeMetaData.S3MetaData.ObjectID = fv.FunctionVersion.Package.ObjectID + codeMetaData.S3MetaData.BucketID = fv.FunctionVersion.Package.BucketID + if fv.FunctionVersion.Package.BucketUrl != "" { + codeMetaData.S3MetaData.BucketURL = fv.FunctionVersion.Package.BucketUrl + } else { + bucketCfg, err := pkgstore.FindBucket(businessID, pkg.BucketID) + if err != nil { + log.GetLogger().Errorf("failed to find bucket info: %s", err.Error()) + return metadata.CodeMetaData{}, err + } + codeMetaData.S3MetaData.BucketURL = bucketCfg.URL + codeMetaData.S3MetaData.AppID = bucketCfg.AppID + } + } + return codeMetaData, nil +} + +func buildFunctionURN(tenantInfo server.TenantInfo, fv storage.FunctionVersionValue) (string, string) { + return utils.BuildFunctionURN(tenantInfo.BusinessID, tenantInfo.TenantID, + tenantInfo.ProductID, fv.Function.Name), tenantInfo.TenantID +} + +func buildEnv(fv storage.FunctionVersionValue, env *metadata.EnvMetaData) error { + if config.RepoCfg.DecryptAlgorithm == "NO_CRYPTO" { + env.CryptoAlgorithm = "NO_CRYPTO" + env.EnvKey = "" + environment, err := getEnvironmentText(fv) + if err != nil { + return errors.New("failed to getEnvironment: " + err.Error()) + } + env.Environment = environment + } else { + key, err := generateEnvKey() + if err != nil { + return errors.New("failed to generate EnvKey: " + err.Error()) + } + env.Environment, err = getEnvironment([]byte(key), fv) + if err != nil { + return errors.New("failed to getEnvironment: " + err.Error()) + } + env.EnvKey, err = utils.EncryptETCDValue(key) + if err != nil { + return errors.New("failed to encryptKey key: " + err.Error()) + } + env.CryptoAlgorithm = "GCM" + } + return nil +} + +func getLayerBucket(txn storage.Transaction, layers []storage.FunctionLayer, + info server.TenantInfo, +) ([]metadata.CodeMetaData, error) { + res := make([]metadata.CodeMetaData, len(layers), len(layers)) + for i, layer := range layers { + val, bucket, err := getBucketInfo(txn, layer, info) + if err != nil { + return nil, err + } + res[i] = metadata.CodeMetaData{ + Sha512: val.Package.Signature, + S3MetaData: metadata.S3MetaData{ + BucketURL: bucket.URL, + ObjectID: val.Package.ObjectID, + BucketID: val.Package.BucketID, + AppID: bucket.AppID, + }, + } + } + return res, nil +} + +func getBucketInfo(txn storage.Transaction, layer storage.FunctionLayer, + info server.TenantInfo, +) (storage.LayerValue, config.BucketConfig, error) { + val, err := storage.GetLayerVersionTx(txn, layer.Name, layer.Version) + if err != nil { + if err == errmsg.KeyNotFoundError { + log.GetLogger().Errorf( + "failed to get layer version, layer name %s, layer version %d does not exist", + layer.Name, layer.Version) + return storage.LayerValue{}, config.BucketConfig{}, err + } + log.GetLogger().Errorf("failed to get layer version: %s", err.Error()) + return storage.LayerValue{}, config.BucketConfig{}, err + } + bucketID := val.Package.BucketID + objectID := val.Package.ObjectID + bucket, err := pkgstore.FindBucket(info.BusinessID, bucketID) + if err != nil { + log.GetLogger().Errorf( + "failed to get bucket, layer name %s, layer version %s, bucketId %s objectID %s", + layer.Name, layer.Version, bucketID, objectID) + return storage.LayerValue{}, config.BucketConfig{}, err + } + return val, bucket, nil +} + +func generateEnvKey() (string, error) { + k := make([]byte, constants.RandomKeySize, constants.RandomKeySize) + if _, err := rand.Read(k); err != nil { + log.GetLogger().Errorf("failed to generate randomKey by rand.Read(): %s", err.Error()) + return "", err + } + return hex.EncodeToString(k[:]), nil +} + +func getEnvironment(key []byte, funcVersion storage.FunctionVersionValue) (string, error) { + envText, err := getEnvironmentText(funcVersion) + if err != nil { + log.GetLogger().Errorf("failed to getEnvironmentText: %s", err.Error()) + return "", err + } + return envText, nil +} + +func getEnvironmentText(funcVersion storage.FunctionVersionValue) (string, error) { + envMap := make(map[string]string, envMapLength) + envPrefix := config.RepoCfg.FunctionCfg.DefaultCfg.EnvPrefix + if funcVersion.FunctionVersion.Kind == common.Faas { + envPrefix = "" + } + if len(funcVersion.FunctionVersion.Environment) != 0 { + env, err := utils.DecodeEnv(funcVersion.FunctionVersion.Environment) + if err != nil { + log.GetLogger().Errorf("failed to decode environment: %s", err.Error()) + return "", err + } + for key, value := range env { + envMap[envPrefix+key] = value + } + } + + envMapValue, err := json.Marshal(envMap) + if err != nil { + log.GetLogger().Errorf("failed to marshal environment: %s", err.Error()) + return "", err + } + return string(envMapValue), nil +} + +// DeleteAllPublishFunction delete all published version by function name +func DeleteAllPublishFunction(txn storage.Transaction, name, kind string, tenantInfo server.TenantInfo) { + path := BuildFunctionRegisterKey(tenantInfo, name, "", kind) + txn.DeleteRange(path) + path = BuildInstanceRegisterKey(tenantInfo, name, "", constants.DefaultClusterID) + txn.DeleteRange(path) + for cluster := range config.RepoCfg.ClusterID { + path = BuildInstanceRegisterKey(tenantInfo, name, "", cluster) + txn.DeleteRange(path) + } +} + +// DeletePublishFunction delete a specified version by function version and function name +func DeletePublishFunction(txn storage.Transaction, name string, tenantInfo server.TenantInfo, funcVersion, + kind string, +) { + path := BuildFunctionRegisterKey(tenantInfo, name, funcVersion, kind) + txn.Delete(path) + path = BuildInstanceRegisterKey(tenantInfo, name, funcVersion, constants.DefaultClusterID) + txn.Delete(path) + for cluster := range config.RepoCfg.ClusterID { + path = BuildInstanceRegisterKey(tenantInfo, name, funcVersion, cluster) + txn.DeleteRange(path) + } +} + +// DeleteAliasByFuncNameEtcd - +func DeleteAliasByFuncNameEtcd(txn storage.Transaction, funcName string) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + + txn.DeleteRange(BuildAliasRegisterKey(t, funcName, "")) + return nil +} + +// CreateAliasEtcd - +func CreateAliasEtcd(txn *storage.Txn, funcName string, aliasEtcd AliasEtcd) error { + t, err := txn.GetCtx().TenantInfo() + if err != nil { + log.GetLogger().Errorf("failed to get tenant info, error: %s", err.Error()) + return err + } + str, err := json.Marshal(aliasEtcd) + if err != nil { + log.GetLogger().Errorf("marshal aliasEtcd value failed, error: %s", err.Error()) + return errmsg.MarshalError + } + txn.Put(BuildAliasRegisterKey(t, funcName, aliasEtcd.Name), string(str)) + + return nil +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/publish/publish_test.go b/functionsystem/apps/meta_service/function_repo/storage/publish/publish_test.go new file mode 100644 index 0000000000000000000000000000000000000000..6bc385641861e1bcc8a0466f862403e7c6af4c45 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/publish/publish_test.go @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package publish + +import ( + "crypto/rand" + "encoding/json" + "errors" + "net/http" + "reflect" + "testing" + + "meta_service/function_repo/config" + "meta_service/function_repo/errmsg" + "meta_service/function_repo/pkgstore" + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + "meta_service/function_repo/test/fakecontext" + + "meta_service/common/crypto" + "meta_service/common/engine" + "meta_service/common/metadata" + + codec2 "meta_service/common/codec" + + "github.com/agiledragon/gomonkey" + "github.com/gin-gonic/gin" + "github.com/golang/mock/gomock" + . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" +) + +func getStorage(ctrl *gomock.Controller) (*engine.MockEngine, *engine.MockTransaction, *storage.Txn) { + transaction := engine.NewMockTransaction(ctrl) + eng := engine.NewMockEngine(ctrl) + eng.EXPECT().BeginTx(gomock.Any()).Return(transaction) + _ = storage.InitStorageByEng(eng, "test") + + ctx := fakecontext.NewMockContext() + txn := storage.NewTxn(ctx) + return eng, transaction, txn +} + +func TestSavePublishFuncVersion(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + if err := config.InitConfig("/home/sn/repo/config.json"); err != nil { + t.Fatalf(err.Error()) + } + + _, tran, txn := getStorage(ctrl) + + layer, err := codec2.NewJSONCodec().Encode(storage.LayerValue{}) + if err != nil { + t.Fatalf(err.Error()) + } + tran.EXPECT().Get(gomock.Any()).Return(layer, nil) + // can not assert value because of the environment encrypt + tran.EXPECT().Put(gomock.Eq("/sn/functions/business/yrk/tenant/i1fe539427b24702acc11fbb4e134e17/function/Name/version/Version"), gomock.Any()).Return() + + functionVersionValue := storage.FunctionVersionValue{ + Function: storage.Function{Name: "Name"}, + FunctionVersion: storage.FunctionVersion{Version: "Version", Environment: "7da374d89a8e872bac706113:d7fb221574a0cf6c8e04b0647545f42c83f5"}, + FunctionLayer: []storage.FunctionLayer{}, + } + functionVersionValue.FunctionLayer = append(functionVersionValue.FunctionLayer, storage.FunctionLayer{Name: "Name", Version: 0, Order: 1}) + if err := SavePublishFuncVersion(txn, functionVersionValue); err != nil { + t.Fatalf("Save error %s", err) + } +} + +func TestCreateAliasEtcd(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + _, tran, txn := getStorage(ctrl) + + info := AliasEtcd{Name: "a"} + + tran.EXPECT().Put(gomock.Eq("/sn/aliases/business/yrk/tenant/i1fe539427b24702acc11fbb4e134e17/function/funcName/a"), gomock.Any()) + err := CreateAliasEtcd(txn, "funcName", info) + assert.Nil(t, err) +} + +func TestSaveTraceChainInfo(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + eng := engine.NewMockEngine(ctrl) + _ = storage.InitStorageByEng(eng, "test") + + ctx := fakecontext.NewMockContext() + info, err := ctx.TenantInfo() + assert.Nil(t, err) + + eng.EXPECT().Put(gomock.Any(), gomock.Eq("/sn/functionchains/business/yrk/tenant/i1fe539427b24702acc11fbb4e134e17/function/name"), gomock.Eq("value")).Return(nil) + err = SaveTraceChainInfo(info, "name", "value") + assert.Nil(t, err) + + eng.EXPECT().Put(gomock.Any(), gomock.Eq("/sn/functionchains/business/yrk/tenant/i1fe539427b24702acc11fbb4e134e17/function/name"), gomock.Eq("value")).Return(errmsg.EtcdInternalError) + err = SaveTraceChainInfo(info, "name", "value") + assert.Equal(t, errmsg.EtcdInternalError, err) +} + +func TestAddTrigger(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + _, tran, txn := getStorage(ctrl) + tran.EXPECT().Put(gomock.Any(), gomock.Any()).Return() + Convey("Test AddTrigger", t, func() { + err := AddTrigger(txn, "mock-func", "veroralias", + storage.TriggerValue{TriggerID: "mockID", TriggerType: "mockType"}, + struct{}{}, + ) + So(err, ShouldBeNil) + }) +} + +func TestDeleteTrigger(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + _, tran, txn := getStorage(ctrl) + tran.EXPECT().Del(gomock.Any()).Return() + Convey("Test DeleteTrigger", t, func() { + err := DeleteTrigger(txn, "mock-func", "veroralias", + storage.TriggerValue{TriggerID: "mockID", TriggerType: "mockType"}) + So(err, ShouldBeNil) + }) +} + +func Test_getEnvironment(t *testing.T) { + key := make([]byte, 1) + funcVer := storage.FunctionVersionValue{} + Convey("Test getEnvironment", t, func() { + patch := gomonkey.ApplyFunc(getEnvironmentText, func(_ storage.FunctionVersionValue) (string, error) { + return "", errors.New("mock err") + }) + defer patch.Reset() + _, err := getEnvironment(key, funcVer) + So(err, ShouldNotBeNil) + }) + Convey("Test getEnvironment 2", t, func() { + patch := gomonkey.ApplyFunc(getEnvironmentText, func(_ storage.FunctionVersionValue) (string, error) { + return "", nil + }).ApplyFunc(crypto.Encrypt, func(content string, secret []byte) ([]byte, error) { + return nil, errors.New("mock err 2") + }) + defer patch.Reset() + _, err := getEnvironment(key, funcVer) + So(err, ShouldNotBeNil) + }) +} + +func Test_buildFunctionVersionValue(t *testing.T) { + Convey("Test buildFunctionVersionValue", t, func() { + fv := storage.FunctionVersionValue{} + tInfo := server.TenantInfo{} + patch := gomonkey.ApplyFunc(buildEnv, func(fv storage.FunctionVersionValue, env *metadata.EnvMetaData) error { + return errors.New("mock err") + }) + defer patch.Reset() + _, err := buildFunctionVersionValue(nil, fv, tInfo) + So(err, ShouldNotBeNil) + }) +} + +func Test_buildFaaSFunctionVersionValue(t *testing.T) { + Convey("Test buildFaaSFunctionVersionValue", t, func() { + patches := gomonkey.NewPatches() + defer patches.Reset() + fv := storage.FunctionVersionValue{ + FunctionVersion: storage.FunctionVersion{ + MaxInstance: 100, + MinInstance: 0, + ConcurrentNum: 100, + Version: "latest", + }, + Function: storage.Function{ + Name: "0@faaspy@hello", + }, + FunctionLayer: []storage.FunctionLayer{}, + } + tInfo := server.TenantInfo{ + TenantID: "12345678901234561234567890123456", + } + Convey("failed to buildFaaSEnv", func() { + Convey("failed to getEnvironment", func() { + patches.ApplyFunc(generateEnvKey, func() (string, error) { + return "", nil + }) + patches.ApplyFunc(getEnvironment, func(key []byte, + funcVersion storage.FunctionVersionValue, + ) (string, error) { + return "", errors.New("get env failed") + }) + _, err := buildFaaSFunctionVersionValue(nil, fv, tInfo) + So(err, ShouldNotBeNil) + }) + Convey("failed to generateEnvKey", func() { + patches.ApplyFunc(generateEnvKey, func() (string, error) { + return "", errors.New("generateEnvKey failed") + }) + _, err := buildFaaSFunctionVersionValue(nil, fv, tInfo) + So(err, ShouldNotBeNil) + }) + }) + Convey("succeed to buildFaaSFunctionVersionValue", func() { + patches.ApplyFunc(buildEnv, func(fv storage.FunctionVersionValue, env *metadata.EnvMetaData) error { + return nil + }) + ctx := server.NewContext(&gin.Context{Request: &http.Request{}}) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + _, tran, txn := getStorage(ctrl) + tran.EXPECT().Put(gomock.Any(), gomock.Any()).Return() + patches.ApplyFunc((*storage.Txn).GetCtx, func(_ *storage.Txn) server.Context { + return ctx + }) + patches.ApplyFunc(storage.GetTxnByKind, func(ctx server.Context, kind string) storage.Transaction { + return txn + }) + Convey("succeed to build value", func() { + _, err := buildFaaSFunctionVersionValue(txn, fv, tInfo) + So(err, ShouldBeNil) + }) + Convey("succeed to getFaaSLayerBucket", func() { + fv.FunctionLayer = []storage.FunctionLayer{ + {Name: "name", Version: 2, Order: 2}, + } + patches.ApplyFunc(storage.GetLayerVersionTx, func(txn storage.Transaction, layerName string, + layerVersion int, + ) (storage.LayerValue, error) { + return storage.LayerValue{}, nil + }) + patches.ApplyFunc(pkgstore.FindBucket, func(businessID string, + bucketID string, + ) (config.BucketConfig, error) { + return config.BucketConfig{}, nil + }) + _, err := buildFaaSFunctionVersionValue(txn, fv, tInfo) + So(err, ShouldBeNil) + }) + }) + }) +} + +func Test_buildCodeMeteData(t *testing.T) { + Convey("Test buildCodeMeteData", t, func() { + fv := storage.FunctionVersionValue{ + FunctionVersion: storage.FunctionVersion{ + Package: storage.Package{}, + }, + } + fv.FunctionVersion.Package.StorageType = "s3" + fv.FunctionVersion.Package.CodeUploadType = "" + fv.FunctionVersion.Package.BucketID = "bucket01" + fv.FunctionVersion.Package.BucketUrl = "bucket-url" + fv.FunctionVersion.Package.ObjectID = "obj1" + fv.FunctionVersion.Package.Signature = "aaa" + codeMetaData, err := buildCodeMetaData(fv, "yrk") + So(err, ShouldBeNil) + So(codeMetaData.Sha512, ShouldEqual, "aaa") + fv.FunctionVersion.Package.BucketUrl = "" + _, err = buildCodeMetaData(fv, "yrk") + So(err, ShouldBeNil) + fv.FunctionVersion.Package.StorageType = "local" + _, err = buildCodeMetaData(fv, "yrk") + So(err, ShouldBeNil) + fv.FunctionVersion.Package.StorageType = "copy" + _, err = buildCodeMetaData(fv, "yrk") + So(err, ShouldBeNil) + fv.FunctionVersion.Package.StorageType = "copy" + }) +} + +func Test_generateEnvKey(t *testing.T) { + Convey("Test generateEnvKey", t, func() { + patch := gomonkey.ApplyFunc(rand.Read, func(b []byte) (n int, err error) { + return 0, errors.New("mock err") + }) + defer patch.Reset() + _, err := generateEnvKey() + So(err, ShouldNotBeNil) + }) +} + +func TestCreateAliasEtcd2(t *testing.T) { + Convey("Test CreateAliasEtcd 2", t, func() { + txn := &storage.Txn{} + ctx := fakecontext.NewMockContext() + patch := gomonkey.ApplyFunc(json.Marshal, func(v interface{}) ([]byte, error) { + return nil, errors.New("mock err") + }).ApplyMethod(reflect.TypeOf(txn), "GetCtx", func(tx *storage.Txn) server.Context { + return ctx + }) + defer patch.Reset() + Convey("with TenantInfo err", func() { + ctx = fakecontext.NewContext() + err := CreateAliasEtcd(txn, "mock-fun", AliasEtcd{}) + So(err, ShouldNotBeNil) + }) + Convey("with json Marshal err", func() { + ctx = fakecontext.NewMockContext() + err := CreateAliasEtcd(txn, "mock-fun", AliasEtcd{}) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/publish/trace_chain.go b/functionsystem/apps/meta_service/function_repo/storage/publish/trace_chain.go new file mode 100644 index 0000000000000000000000000000000000000000..7c63b495bce24461e069379e49bb8e0d53c4cfaa --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/publish/trace_chain.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package publish + +import ( + "context" + + "meta_service/function_repo/server" + "meta_service/function_repo/storage" + + "meta_service/common/logger/log" +) + +// SaveTraceChainInfo save trace chain info to etcd +func SaveTraceChainInfo(info server.TenantInfo, name, value string) error { + key := BuildTraceChainRegisterKey(name, info) + if err := storage.Put(context.Background(), key, value); err != nil { + log.GetLogger().Errorf("failed to save trace chain info to etcd, error: %s", err.Error()) + return err + } + return nil +} + +// DeleteTraceChainInfo delete trace chain info from etcd +func DeleteTraceChainInfo(txn storage.Transaction, name string, info server.TenantInfo) { + key := BuildTraceChainRegisterKey(name, info) + txn.Delete(key) +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/publish/type.go b/functionsystem/apps/meta_service/function_repo/storage/publish/type.go new file mode 100644 index 0000000000000000000000000000000000000000..4ca43821e82304dabac37cd7f2537761f1348daf --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/publish/type.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package publish implements an event publisher to publish create, update and delete events of functions, +// triggers and aliases. +package publish + +// AliasEtcd alias info was watched by workerManager etc. +type AliasEtcd struct { + AliasURN string `json:"aliasUrn"` + FunctionURN string `json:"functionUrn"` + FunctionVersionURN string `json:"functionVersionUrn"` + Name string `json:"name"` + FunctionVersion string `json:"functionVersion"` + RevisionID string `json:"revisionId"` + Description string `json:"description"` + // original java code use "reoutingconfig" as json key + RoutingConfig []AliasRoutingEtcd `json:"routingconfig"` +} + +// AliasRoutingEtcd - +type AliasRoutingEtcd struct { + FunctionVersionURN string `json:"functionVersionUrn"` + Weight int `json:"weight"` +} diff --git a/functionsystem/apps/meta_service/function_repo/storage/reserve_instance.go b/functionsystem/apps/meta_service/function_repo/storage/reserve_instance.go new file mode 100644 index 0000000000000000000000000000000000000000..2ec203e7b856191de8de7d4e0dd2860b79d1c14f --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/reserve_instance.go @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "encoding/json" + "io" + "strings" + + "meta_service/function_repo/errmsg" + "meta_service/function_repo/model" + "meta_service/function_repo/server" + "meta_service/function_repo/utils/constants" + + "meta_service/common/logger/log" + "meta_service/common/metadata" +) + +const labelKeyLen int = 14 + +func buildReserveInstanceWithLabelKey(clusterID, tenantID, name, funcVersion, label string) string { + // format: /instances/business/yrk/cluster/cluster001/tenant//function//version\ + // //label/