From b1a0ea9edc4e8848a5afe81f22a79a1d86abcdb9 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 18 Dec 2025 05:08:45 +0000 Subject: [PATCH 1/2] cli: remove old code Signed-off-by: Yuhang Wei --- functionsystem/apps/cli/.gitignore | 29 - functionsystem/apps/cli/cmd/main.go | 36 - functionsystem/apps/cli/constant/build.go | 27 - functionsystem/apps/cli/constant/constants.go | 59 - functionsystem/apps/cli/go.mod | 67 - functionsystem/apps/cli/go.sum | 96 - functionsystem/apps/cli/internal/root.go | 73 - functionsystem/apps/cli/internal/root_test.go | 26 - .../apps/cli/internal/start/yr_start.go | 751 --- .../apps/cli/internal/start/yr_start_test.go | 406 -- .../apps/cli/internal/status/yr_status.go | 152 - .../apps/cli/internal/stop/yr_stop.go | 300 -- .../apps/cli/internal/stop/yr_stop_test.go | 92 - .../apps/cli/internal/version/version.go | 57 - .../apps/cli/internal/version/version_test.go | 62 - functionsystem/apps/cli/pkg/cmdio/cmdio.go | 39 - .../apps/cli/pkg/progress/progress.go | 84 - functionsystem/apps/cli/pkg/test/cmdio.go | 50 - functionsystem/apps/cli/utils/cmd_args.go | 81 - .../apps/cli/utils/colorprint/ac.go | 144 - .../apps/cli/utils/colorprint/colorprint.go | 148 - .../cli/utils/colorprint/colorprint_unix.go | 86 - .../apps/cli/utils/colorprint/types.go | 86 - .../apps/cli/utils/colorprint/types_unix.go | 49 - functionsystem/apps/cli/utils/file.go | 38 - functionsystem/apps/cli/utils/file_test.go | 39 - functionsystem/apps/cli/utils/localip.go | 53 - functionsystem/apps/cli/utils/masterinfo.go | 134 - functionsystem/apps/cli/utils/memory.go | 66 - functionsystem/apps/cli/utils/memory_test.go | 60 - functionsystem/apps/cli/utils/raw.go | 53 - functionsystem/apps/cli/utils/raw_test.go | 35 - functionsystem/apps/cli/utils/reader.go | 70 - functionsystem/apps/cli/utils/reader_test.go | 63 - functionsystem/apps/cli/utils/symbollink.go | 55 - functionsystem/apps/cli/utils/util.go | 78 - functionsystem/apps/cli/utils/util_test.go | 83 - .../meta_service/function_repo/storage/kv.go | 4594 +++++++++++++++++ 38 files changed, 4594 insertions(+), 3827 deletions(-) delete mode 100644 functionsystem/apps/cli/.gitignore delete mode 100644 functionsystem/apps/cli/cmd/main.go delete mode 100644 functionsystem/apps/cli/constant/build.go delete mode 100644 functionsystem/apps/cli/constant/constants.go delete mode 100644 functionsystem/apps/cli/go.mod delete mode 100644 functionsystem/apps/cli/go.sum delete mode 100644 functionsystem/apps/cli/internal/root.go delete mode 100644 functionsystem/apps/cli/internal/root_test.go delete mode 100644 functionsystem/apps/cli/internal/start/yr_start.go delete mode 100644 functionsystem/apps/cli/internal/start/yr_start_test.go delete mode 100644 functionsystem/apps/cli/internal/status/yr_status.go delete mode 100644 functionsystem/apps/cli/internal/stop/yr_stop.go delete mode 100644 functionsystem/apps/cli/internal/stop/yr_stop_test.go delete mode 100644 functionsystem/apps/cli/internal/version/version.go delete mode 100644 functionsystem/apps/cli/internal/version/version_test.go delete mode 100644 functionsystem/apps/cli/pkg/cmdio/cmdio.go delete mode 100644 functionsystem/apps/cli/pkg/progress/progress.go delete mode 100644 functionsystem/apps/cli/pkg/test/cmdio.go delete mode 100644 functionsystem/apps/cli/utils/cmd_args.go delete mode 100644 functionsystem/apps/cli/utils/colorprint/ac.go delete mode 100644 functionsystem/apps/cli/utils/colorprint/colorprint.go delete mode 100644 functionsystem/apps/cli/utils/colorprint/colorprint_unix.go delete mode 100644 functionsystem/apps/cli/utils/colorprint/types.go delete mode 100644 functionsystem/apps/cli/utils/colorprint/types_unix.go delete mode 100644 functionsystem/apps/cli/utils/file.go delete mode 100644 functionsystem/apps/cli/utils/file_test.go delete mode 100644 functionsystem/apps/cli/utils/localip.go delete mode 100644 functionsystem/apps/cli/utils/masterinfo.go delete mode 100644 functionsystem/apps/cli/utils/memory.go delete mode 100644 functionsystem/apps/cli/utils/memory_test.go delete mode 100644 functionsystem/apps/cli/utils/raw.go delete mode 100644 functionsystem/apps/cli/utils/raw_test.go delete mode 100644 functionsystem/apps/cli/utils/reader.go delete mode 100644 functionsystem/apps/cli/utils/reader_test.go delete mode 100644 functionsystem/apps/cli/utils/symbollink.go delete mode 100644 functionsystem/apps/cli/utils/util.go delete mode 100644 functionsystem/apps/cli/utils/util_test.go create mode 100644 functionsystem/apps/meta_service/function_repo/storage/kv.go diff --git a/functionsystem/apps/cli/.gitignore b/functionsystem/apps/cli/.gitignore deleted file mode 100644 index 8f0f6e3..0000000 --- a/functionsystem/apps/cli/.gitignore +++ /dev/null @@ -1,29 +0,0 @@ -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Code coverage profiles and other test artifacts -*.out -coverage.* -*.coverprofile -profile.cov - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go file -go.work -go.work.sum - -# env file -.env - -# Editor/IDE -.idea/ -.vscode/ diff --git a/functionsystem/apps/cli/cmd/main.go b/functionsystem/apps/cli/cmd/main.go deleted file mode 100644 index d51c544..0000000 --- a/functionsystem/apps/cli/cmd/main.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package main is main function of serverless native -package main - -import ( - "os" - - "cli/internal" -) - -const errorExitCode = 1 - -func main() { - rootCmd := cmd.NewCmdRoot() - - err := rootCmd.Execute() - if err != nil { - os.Exit(errorExitCode) - return - } -} diff --git a/functionsystem/apps/cli/constant/build.go b/functionsystem/apps/cli/constant/build.go deleted file mode 100644 index d65d4fa..0000000 --- a/functionsystem/apps/cli/constant/build.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package build is build info of cli -package constant - -const ( - // Version is version of cli, it is dynamically set by the Makefile. - Version = "DEV" - // Date is build time of cli, it is dynamically set by the Makefile. - Date = "" - // CustomizeVer is build time of cli, it is dynamically set by the Makefile. - CustomizeVer = "" -) diff --git a/functionsystem/apps/cli/constant/constants.go b/functionsystem/apps/cli/constant/constants.go deleted file mode 100644 index 52c8904..0000000 --- a/functionsystem/apps/cli/constant/constants.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package constant is constant of CLI -package constant - -// Build input parameter variables -var ( - // CliName cli tool name - CliName = "yr" - // PlatformName service platform name - PlatformName = "OpenYuanrong" -) - -// Usage -const ( - // DeployReadyCheckIntervalMs - - DeployReadyCheckIntervalMs = 500 - // DeployStdLogRedirectIntervalMs - - DeployStdLogRedirectIntervalMs = 100 - // DeployReadyCheckTimeoutS - - DeployReadyCheckTimeoutS = 120 -) - -const ( - // DefaultYuanRongInstallationDir is the default installation dir to yuanrong - DefaultYuanRongInstallationDir = "/opt/yuanrong" - // DefaultYuanRongDeployDir is the default deploy path - DefaultYuanRongDeployDir = "/tmp/yr_sessions/latest" - // DefaultYuanRongCurrentMasterInfoPath is the default master info path - DefaultYuanRongCurrentMasterInfoPath = "/tmp/yr_sessions/yr_current_master_info" - - // EnvYuanRongInstallationDir - - EnvYuanRongInstallationDir = "YR_INSTALLATION_DIR" - // EnvYuanRongDeployDir - - EnvYuanRongDeployDir = "YR_DEPLOY_DIR" -) - -var ( - // YuanRongInstallationDir - - YuanRongInstallationDir = DefaultYuanRongInstallationDir - // YuanRongDeployDir - - YuanRongDeployDir = DefaultYuanRongDeployDir - // YuanRongCurrentMasterInfoPath - - YuanRongCurrentMasterInfoPath = DefaultYuanRongCurrentMasterInfoPath -) diff --git a/functionsystem/apps/cli/go.mod b/functionsystem/apps/cli/go.mod deleted file mode 100644 index 03a3180..0000000 --- a/functionsystem/apps/cli/go.mod +++ /dev/null @@ -1,67 +0,0 @@ -module cli - -go 1.21.4 - -require ( - github.com/agiledragon/gomonkey v2.0.1+incompatible - github.com/smartystreets/goconvey v1.6.4 - github.com/spf13/cobra v1.8.0 - github.com/stretchr/testify v1.9.0 - go.etcd.io/etcd/client/v3 v3.5.11 -) - -require ( - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect - github.com/spf13/pflag v1.0.5 // indirect - go.etcd.io/etcd/api/v3 v3.5.11 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect - go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect -) - -replace ( - github.com/agiledragon/gomonkey => github.com/agiledragon/gomonkey v2.0.1+incompatible - github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.7.0 - // for test or internal use - github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.10.0 - github.com/golang/mock => github.com/golang/mock v1.3.1 - github.com/google/uuid => github.com/google/uuid v1.6.0 - github.com/olekukonko/tablewriter => github.com/olekukonko/tablewriter v0.0.5 - github.com/operator-framework/operator-lib => github.com/operator-framework/operator-lib v0.4.0 - github.com/prashantv/gostub => github.com/prashantv/gostub v1.0.0 - github.com/robfig/cron/v3 => github.com/robfig/cron/v3 v3.0.1 - github.com/smartystreets/goconvey => github.com/smartystreets/goconvey v1.6.4 - github.com/spf13/cobra => github.com/spf13/cobra v1.8.0 - github.com/stretchr/testify => github.com/stretchr/testify v1.5.1 - github.com/valyala/fasthttp => github.com/valyala/fasthttp v1.55.0 - go.etcd.io/etcd/api/v3 => go.etcd.io/etcd/api/v3 v3.5.11 - go.etcd.io/etcd/client/v3 => go.etcd.io/etcd/client/v3 v3.5.11 - go.uber.org/zap => go.uber.org/zap v1.27.0 - golang.org/x/crypto => golang.org/x/crypto v0.12.0 - // affects VPC plugin building, will cause error if not pinned - golang.org/x/net => golang.org/x/net v0.21.0 - golang.org/x/sync => golang.org/x/sync v0.0.0-20190423024810-112230192c58 - golang.org/x/sys => golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c - golang.org/x/text => golang.org/x/text v0.9.0 - golang.org/x/time => golang.org/x/time v0.3.0 - gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1 -) diff --git a/functionsystem/apps/cli/go.sum b/functionsystem/apps/cli/go.sum deleted file mode 100644 index 95263f5..0000000 --- a/functionsystem/apps/cli/go.sum +++ /dev/null @@ -1,96 +0,0 @@ -github.com/agiledragon/gomonkey v2.0.1+incompatible h1:DIQT3ZshgGz9pTwBddRSZWDutIRPx2d7UzmjzgWo9q0= -github.com/agiledragon/gomonkey v2.0.1+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/etcd/api/v3 v3.5.11 h1:B54KwXbWDHyD3XYAwprxNzTe7vlhR69LuBgZnMVvS7E= -go.etcd.io/etcd/api/v3 v3.5.11/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= -go.etcd.io/etcd/client/pkg/v3 v3.5.11 h1:bT2xVspdiCj2910T0V+/KHcVKjkUrCZVtk8J2JF2z1A= -go.etcd.io/etcd/client/pkg/v3 v3.5.11/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= -go.etcd.io/etcd/client/v3 v3.5.11 h1:ajWtgoNSZJ1gmS8k+icvPtqsqEav+iUorF7b0qozgUU= -go.etcd.io/etcd/client/v3 v3.5.11/go.mod h1:a6xQUEqFJ8vztO1agJh/KQKOMfFI8og52ZconzcDJwE= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/functionsystem/apps/cli/internal/root.go b/functionsystem/apps/cli/internal/root.go deleted file mode 100644 index 9319a67..0000000 --- a/functionsystem/apps/cli/internal/root.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package cmd is cmd of YuanRong -package cmd - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" - - "cli/constant" - "cli/internal/start" - "cli/internal/status" - "cli/internal/stop" - "cli/internal/version" - "cli/pkg/cmdio" -) - -// NewCmdRoot new root cmd -func NewCmdRoot() *cobra.Command { - initYuanRongDeployAndInstallVars() - cmd := &cobra.Command{ - Use: fmt.Sprintf("%s", constant.CliName), - Short: fmt.Sprintf("used on %s Platform", constant.PlatformName), - Long: fmt.Sprintf("used on %s Platform.", constant.PlatformName), - Example: fmt.Sprintf(` - %s start --master - %s stop - %s version - %s status - source <(%s completion bash) - `, constant.CliName, constant.CliName, constant.CliName, constant.CliName, constant.CliName), - } - - cio := cmdio.System() - - cobra.OnInitialize() - - cmd.AddCommand(start.InitYrCMD(cio)) - cmd.AddCommand(stop.InitYrCMD(cio)) - cmd.AddCommand(status.InitYrCMD(cio)) - cmd.AddCommand(version.InitVersionCMD(cio)) - - return cmd -} - -func initYuanRongDeployAndInstallVars() { - // 1. use default values - // Do nothing, already did - - // 2. from environment - if installDir := os.Getenv(constant.EnvYuanRongInstallationDir); installDir != "" { - constant.YuanRongInstallationDir = installDir - } - if deployDir := os.Getenv(constant.EnvYuanRongDeployDir); deployDir != "" { - constant.YuanRongDeployDir = deployDir - } -} diff --git a/functionsystem/apps/cli/internal/root_test.go b/functionsystem/apps/cli/internal/root_test.go deleted file mode 100644 index 8dce1d2..0000000 --- a/functionsystem/apps/cli/internal/root_test.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package cmd - -import ( - "testing" -) - -func TestNewCmdRoot(t *testing.T) { - t.Run("New Cmd Root", func(t *testing.T) { - NewCmdRoot() - }) -} diff --git a/functionsystem/apps/cli/internal/start/yr_start.go b/functionsystem/apps/cli/internal/start/yr_start.go deleted file mode 100644 index 5cb84a9..0000000 --- a/functionsystem/apps/cli/internal/start/yr_start.go +++ /dev/null @@ -1,751 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package start is start cmd -package start - -import ( - "bufio" - "context" - "errors" - "fmt" - "io" - "os" - "os/exec" - "os/signal" - "path/filepath" - "regexp" - "runtime" - "strconv" - "strings" - "syscall" - "time" - - "github.com/spf13/cobra" - - "cli/constant" - "cli/pkg/cmdio" - "cli/utils" - "cli/utils/colorprint" -) - -type yrStartOptions struct { - cmdIO *cmdio.CmdIO - - // General Options: - ipAddress string - cpuNum string - deployPath string - memoryNum string - sharedMemoryNum string - masterInfoOutput string - master bool - masterInfo string - servicesPath string - masterIp string - enableInheritEnv string - etcdAddrList string - dsNodeDeadTimeoutS string - dsNodeTimeoutS string - dsHeartbeatIntervalMs string - - // startup options - disableNcCheck bool - - // Others - verbose int - block bool -} - -var yrOpts yrStartOptions - -const ( - fileMode = 0o600 - dirMode = os.ModeDir | 0o750 - deployFailureWaitTime = 2 * constant.DeployStdLogRedirectIntervalMs * time.Millisecond -) - -var ipRegex *regexp.Regexp = regexp.MustCompile(`(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})`) - -type currentMasterInfoDumpPolicy int - -const ( - dumpPolicyNothing currentMasterInfoDumpPolicy = iota + 1 - dumpPolicyNew - dumpPolicyReplace - dumpPolicyInvalid -) - -var yrStartCmd = &cobra.Command{ - Use: "start", - Short: fmt.Sprintf("start %s Platform", constant.PlatformName), - Long: fmt.Sprintf(`start %s Platform`, constant.PlatformName), - Example: utils.RawFormat(fmt.Sprintf(` - %s start in non-interactive mode: - $ %s start --master - `, constant.CliName, constant.CliName)), - Args: cobra.ArbitraryArgs, - RunE: yrStartDeploy, -} - -// InitYrCMD init cmd -func InitYrCMD(cio *cmdio.CmdIO) *cobra.Command { - yrOpts.cmdIO = cio - mem := &runtime.MemStats{} - runtime.ReadMemStats(mem) - - totalMem := mem.Sys - if physicalMem, err := utils.ReadMemoryStats(); err == nil { - totalMem = physicalMem.MemTotal - } - - yuanRongCpu := runtime.NumCPU() * 1000 // 这里cpu的单位是千分之一核，默认使用所有系统资源 - yuanRongMem := totalMem / 1024 // openYuanRong 默认纳管集群所有内存 - dataSystemMem := yuanRongMem / 3 // 共享内存默认使用1/3的内存 - - yrStartCmd.SilenceUsage = true - yrStartCmd.FParseErrWhitelist.UnknownFlags = true - - defineParams(yuanRongCpu, yuanRongMem, dataSystemMem) - - // Logging Options: None - // Function System Options: None - // Data System Options: None - yrStartCmd.SetHelpFunc(func(command *cobra.Command, strings []string) { - helpInfo, err := yrDeployShowHelp(constant.YuanRongInstallationDir) - if err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "failed to get script help info", err.Error(), "\n") - } - colorprint.PrintSuccess(yrOpts.cmdIO.Out, "", helpInfo) - }) - return yrStartCmd -} - -func defineParams(yuanRongCpu int, yuanRongMem uint64, dataSystemMem uint64) { - // In the tools, we override some options to provide better usability - // General Options: - yrStartCmd.Flags().BoolVarP(&yrOpts.master, "master", "", false, - "deploy control plane, otherwise only deploy data plane") - yrStartCmd.Flags().IntVarP(&yrOpts.verbose, "verbose", "", 2, "no output") - yrStartCmd.Flags().BoolVarP(&yrOpts.block, "block", "", false, - "block, not running background") - yrStartCmd.Flags().StringVarP(&yrOpts.ipAddress, "ip_address", "a", "", - "node ip address") - yrStartCmd.Flags().StringVarP(&yrOpts.cpuNum, "cpu_num", "c", strconv.Itoa(yuanRongCpu), - "overall cpu cores (1/1000 core) in current script context") - yrStartCmd.Flags().StringVarP(&yrOpts.deployPath, "deploy_path", "d", "", - "it is recommended to use an empty directory as deploy_path") - yrStartCmd.Flags().StringVarP(&yrOpts.memoryNum, "memory_num", "m", - strconv.FormatUint(yuanRongMem, 10), - "overall memory (MB) in current script context(should be larger than shared_memory_num)") - yrStartCmd.Flags().StringVarP(&yrOpts.sharedMemoryNum, "shared_memory_num", "s", - strconv.FormatUint(dataSystemMem, 10), - "data system shared memory (MB) should be reserved in current script context") - yrStartCmd.Flags().StringVarP(&yrOpts.masterInfoOutput, "master_info_output", "o", "", - "master info output file, absolute path") - yrStartCmd.Flags().StringVarP(&yrOpts.masterInfo, "master_info", "", "", "master info") - yrStartCmd.Flags().StringVarP(&yrOpts.servicesPath, "services_path", "p", "", - "auto load function path") - yrStartCmd.Flags().StringVarP(&yrOpts.masterIp, "master_ip", "", "", - "master ip for data plane component") - yrStartCmd.Flags().StringVarP(&yrOpts.etcdAddrList, "etcd_addr_list", "", "", - "etcd cluster address list, such as ip1,ip2 or ip1:port1,ip2:port2 or "+ - "ip1:port1:peer_port1,ip2:port2:peer_port2. if use this param, master_info is not required.") - yrStartCmd.Flags().StringVarP(&yrOpts.enableInheritEnv, "enable_inherit_env", "", "true", - "enable runtime to inherit env from runtime-manager, default is true") - yrStartCmd.Flags().StringVarP(&yrOpts.dsNodeDeadTimeoutS, "ds_node_dead_timeout_s", "", "30", - "datasystem node dead timeout (in second, default is 30)") - yrStartCmd.Flags().StringVarP(&yrOpts.dsNodeTimeoutS, "ds_node_timeout_s", "", "10", - "datasystem node timeout (in second, default is 10)") - yrStartCmd.Flags().StringVarP(&yrOpts.dsHeartbeatIntervalMs, "ds_heartbeat_interval_ms", "", "3000", - "datasystem heartbeat interval (in ms, default is 3000)") - - yrStartCmd.Flags().BoolVarP(&yrOpts.disableNcCheck, "disable_nc_check", "", true, - "disable nc check port (default true)") -} - -func yrDeployShowHelp(path string) (string, error) { - // make sure fresh deploy.sh script exists and valid - deployScriptPath := filepath.Join(path, "deploy", "process", "deploy.sh") - if _, err := os.Stat(deployScriptPath); err != nil { - return "", errors.New( - fmt.Sprintf("maybe check if you have installed yuanrong (%s)", err.Error())) - } - - execName, _ := utils.GetGOOSType() - args := make([]string, 0) - args = append(args, deployScriptPath) - args = append(args, "--help") - masterShowHelpCommand := exec.Command(execName, args...) - output, err := masterShowHelpCommand.Output() - return string(output), err -} - -func yrStartDeploy(cmd *cobra.Command, args []string) error { - if err := yrStartYuanRong(cmd, args); err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "", "Deploy OpenYuanrong failed.", "\n") - return err - } - colorprint.PrintSuccess(yrOpts.cmdIO.Out, "Succeeded", " to start!") - return nil -} - -func validateStartParams(cmd *cobra.Command) error { - if yrOpts.master && cmd.Flags().Changed("master_info") { - return errors.New("--master, --master_info cannot be specified together") - } - if cmd.Flags().Changed("etcd_addr_list") && cmd.Flags().Changed("master_info") { - return errors.New("--etcd_addr_list, --master_info cannot be specified together") - } - if !yrOpts.master && yrOpts.etcdAddrList == "" && yrOpts.masterInfo == "" { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "", - "none of --master, --etcd_addr_list or --master_info is specified", ` -* trying to start your first yuanrong node, please use --master - yr start --master - -* trying to start your second yuanrong node, please use --master_info as the instruction shown by the above command -`) - return errors.New("") - } - return nil -} - -func yrStartYuanRong(cmd *cobra.Command, args []string) error { - if !utils.SupportSystem() { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "Failed to start, ", - fmt.Sprintf("unsupported os: %s", runtime.GOOS), ".\n") - return errors.New(fmt.Sprintf("unsupported os: %s", runtime.GOOS)) - } - - policy, err := checkIfNodeAlreadyStartedYuanRong() - if err != nil { - return err - } - - yrProcessDeployArgs() - yuanRongDir := constant.YuanRongInstallationDir - - if err := validateStartParams(cmd); err != nil { - return err - } - - if err := yrMakeDeployPathSymbolLink(); err != nil { - return err - } - if yrOpts.master { // start master - return yrDeployMaster(yuanRongDir, policy) - } - if err := yrDeployAgent(yuanRongDir); err != nil { - return err - } - if err := yrMakeSymbolLink(policy); err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "warn: make symbol link may failed. ", err.Error(), "\n") - } - return nil -} - -func checkIfNodeAlreadyStartedYuanRong() (currentMasterInfoDumpPolicy, error) { - // Overall: always allow user start yuanrong over an existing cluster, - // if start another agent in already belonged cluster's master node, do nothing - // if start anything else, log the replace, point to the newer yuanrong processes - // - // if not exist, ok, just return - // if exist, then - // * if existing info is master, and incoming is also in the same yuanrong cluster - // then do nothing - // * else, make a replacement, log a warning about the replacement, and then point to the new processes - // - // to check if belongs to same master, we use etcd_addr_list(if exists) or master_ip(if exists) - var err error - info, err := utils.GetMasterInfoFromFile(yrOpts.masterInfoOutput) - if err != nil { - if os.IsNotExist(err) { - return dumpPolicyNew, nil - } - return dumpPolicyInvalid, errors.New("maybe you have started a yuanrong cluster in your environment, and we " + - "failed to read its info, run `yr stop` to make sure it is fully cleared, and then you can start again") - } - - if !info.IsMasterNode() { - // if current is agent, just replace it - return dumpPolicyReplace, nil - } - - var incomingMi *utils.MasterInfo - if yrOpts.masterInfo != "" { - // start a single-master cluster's agent - incomingMi = utils.ExtractMasterInfo(yrOpts.masterInfo) - } else { - // maybe, starting another single-master cluster's master -- no etcdAddrList, always should return REPLACE - // maybe, starting a multi-master cluster's master -- has etcdAddrList, check if the same - // maybe, starting a multi-master cluster's agent -- has etcdAddrList, check if the same - incomingMi = &utils.MasterInfo{ - EtcdAddrList: splitEtcdAddrListString(yrOpts.etcdAddrList), - } - } - - if info.IsSameCluster(incomingMi) { - return dumpPolicyNothing, nil - } - return dumpPolicyReplace, nil -} - -func splitEtcdAddrListString(etcdAddrList string) []string { - var list []string - if ipRegex == nil { - return list - } - for _, ip := range ipRegex.FindAllString(etcdAddrList, -1) { - list = append(list, ip) - } - return list -} - -func printMasterStartedHelpMsg() error { - if masterInfo, err := utils.GetMasterInfoString(yrOpts.masterInfoOutput); err == nil && - yrOpts.verbose > 0 { - var startNextHelp string - if len(yrOpts.etcdAddrList) > 0 { - startNextHelp = "" - } else { - startNextHelp = fmt.Sprintf(` * to add another node into yuanrong cluster - yr start --master_info "%s" -`, masterInfo) - } - - colorprint.PrintSuccess(yrOpts.cmdIO.Out, "Yuanrong deployed succeed", - fmt.Sprintf(` -Cluster master info: - %s - -Next Steps: -%s - * to check if yuanrong cluster is running normally - yr status - - * to stop running yuanrong processes - yr stop -`, masterInfo, startNextHelp)) - return nil - } - if yrOpts.verbose > 0 { - colorprint.PrintFail(yrOpts.cmdIO.Out, "Yuanrong deployed may failed", - fmt.Sprintf(` -failed to find master info file at %s, the deployment may failed, you can also run - yr stop -to make sure no yuanrong processes left on you machine -`, yrOpts.masterInfoOutput), ".\n") - } - return errors.New("failed to deploy yuanrong master") -} - -func redirectFileContentToStd(ctx context.Context, deployStdFile *os.File) { - var lastSize int64 = 0 - for { - select { - case <-ctx.Done(): - return - default: - time.Sleep(constant.DeployStdLogRedirectIntervalMs * time.Millisecond) - fi, err := deployStdFile.Stat() - if err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "failed to stat log file: ", err.Error(), ".\n") - return - } - if fi.Size() <= lastSize { - time.Sleep(constant.DeployStdLogRedirectIntervalMs * time.Millisecond) - continue - } - if _, err := deployStdFile.Seek(lastSize, io.SeekStart); err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "failed to seek log file: ", err.Error(), ".\n") - return - } - buffer := make([]byte, fi.Size()-lastSize) - n, err := deployStdFile.Read(buffer) - if err != nil && err != io.EOF { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "failed to read log file: ", err.Error(), ".\n") - return - } - if n > 0 { - fmt.Print(string(buffer[:n])) - } - lastSize = fi.Size() - } - } -} - -func prepareDeployWorkingDir() (*os.File, error) { - // make sure fresh new `master_pid_port.txt` - if err := os.Remove(filepath.Join(yrOpts.deployPath, "master_pid_port.txt")); err != nil { - // do nothing - } - - // make sure master info output file parent dir exists - if err := os.MkdirAll(filepath.Dir(yrOpts.masterInfoOutput), dirMode); err != nil { - return nil, err - } - // make sure the master info output file fresh new - if err := os.RemoveAll(yrOpts.masterInfoOutput); err != nil { - return nil, err - } - // make sure deploy path exists - if err := os.MkdirAll(yrOpts.deployPath, dirMode); err != nil { - return nil, err - } - // make sure /tmp/yr_sessions dir exists - if err := os.MkdirAll(filepath.Dir(constant.DefaultYuanRongCurrentMasterInfoPath), dirMode); err != nil { - return nil, err - } - - // DO NOT CLOSE THIS FILE - deployStdFile, err := os.OpenFile(filepath.Join(yrOpts.deployPath, "deploy_std.log"), - os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileMode) - if err != nil { - return nil, err - } - - return deployStdFile, nil -} - -func yrDeployMaster(path string, dumpPolicy currentMasterInfoDumpPolicy) error { - var err error - err, masterDeployCmd := concatMasterDeployCmd(path) - if err != nil { - return err - } - - deployStdFile, err := prepareDeployWorkingDir() - if err != nil { - return err - } - masterDeployCmd.Stdout = deployStdFile - masterDeployCmd.Stderr = deployStdFile - - masterRunningCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if yrOpts.verbose > 1 { - go redirectFileContentToStd(masterRunningCtx, deployStdFile) - } - - // block, run the command directly - // no block, run the command until master_info_output file generated - if yrOpts.block { - blockRun(masterDeployCmd, cancel) - } - - var waitChan chan error - if err, waitChan = utils.ExecCommandUntil(masterDeployCmd, untilMasterDeploySucceed, - constant.DeployReadyCheckTimeoutS, yrOpts.block); err != nil { - // wait some time so the std output will be redirect to the stdout - time.Sleep(deployFailureWaitTime) - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, - fmt.Sprintf("maybe you can go check the deploy log file %s for further details", deployStdFile.Name()), - "", ".\n") - clearCmdProcess(masterDeployCmd) - return err - } - - if err := yrMakeSymbolLink(dumpPolicy); err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, - fmt.Sprintf("warn: make symbol link failed: %s", err.Error()), "", ".\n") - return nil - } - - if yrOpts.block { - if err := printMasterStartedHelpMsg(); err != nil { - return err - } - // when block, wait here - if waitChan != nil { - return <-waitChan - } - return errors.New("wait failed, maybe the yr start failed") - } - - return printMasterStartedHelpMsg() -} - -func concatMasterDeployCmd(path string) (error, *exec.Cmd) { - var err error - - // make sure fresh deploy.sh script exists and valid - deployScriptPath := filepath.Join(path, "deploy", "process", "deploy.sh") - if _, err = os.Stat(deployScriptPath); err != nil { - return errors.New(fmt.Sprintf("maybe check if you have installed yuanrong: %s(%s)", - deployScriptPath, err.Error())), nil - } - - execName, _ := utils.GetGOOSType() - args := make([]string, 0) - args = append(args, deployScriptPath) - err, args = concatArgs(args) - if err != nil { - return err, nil - } - masterDeployCmd := exec.Command(execName, args...) - return nil, masterDeployCmd -} - -func blockRun(masterDeployCmd *exec.Cmd, cancel context.CancelFunc) { - // when block, use pdeathsig to make sure when this command exit, send sigterm to the deploy.sh - // when no blocked, will just run - masterDeployCmd.SysProcAttr = &syscall.SysProcAttr{ - Pdeathsig: syscall.SIGTERM, - } - - ch := make(chan os.Signal, 2) - signal.Notify(ch, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM) - go func() { - select { - case <-ch: - cancel() - if err := os.RemoveAll(constant.DefaultYuanRongCurrentMasterInfoPath); err != nil { - // do nothing - } - os.Exit(0) - } - }() -} - -func yrProcessDeployArgs() { - if yrOpts.deployPath == "" { - yrOpts.deployPath = "/tmp/yr_sessions/" + time.Now().Format("20060102150405") - } - if yrOpts.masterInfoOutput == "" { - yrOpts.masterInfoOutput = filepath.Join(yrOpts.deployPath, "master.info") - } -} - -func yrMakeDeployPathSymbolLink() error { - if yrOpts.deployPath != constant.DefaultYuanRongDeployDir { - if !utils.Exists(filepath.Dir(constant.DefaultYuanRongDeployDir)) { - if err := os.MkdirAll(filepath.Dir(constant.DefaultYuanRongDeployDir), dirMode); err != nil { - // do nothing - } - } - if err := utils.CreateSymbolicLinkWithForce(yrOpts.deployPath, constant.DefaultYuanRongDeployDir); err != nil { - colorprint.PrintInteractive(yrOpts.cmdIO.Out, fmt.Sprintf("WARN: failed to link %s to %s. err: %s\n", - yrOpts.deployPath, constant.DefaultYuanRongDeployDir, err.Error())) - return nil - } - } - return nil -} - -func yrMakeSymbolLink(dumpPolicy currentMasterInfoDumpPolicy) error { - if yrOpts.masterInfoOutput != constant.DefaultYuanRongCurrentMasterInfoPath { - switch dumpPolicy { - case dumpPolicyNothing: - case dumpPolicyInvalid: - return nil - case dumpPolicyReplace: - info, err := utils.GetMasterInfoString(constant.YuanRongCurrentMasterInfoPath) - if err != nil { - info = fmt.Sprintf("failed to get the master info due to %s", err.Error()) - } - colorMap := []colorprint.StringColorInfo{ - {Str: fmt.Sprintf("NOTE: dumped yuanrong cluster info at %s is updated.\n", - constant.YuanRongCurrentMasterInfoPath), Color: colorprint.KeywordColor}, - {Str: fmt.Sprintf("There seems an existing yuanrong cluster already running on your node with info:\n"+ - " %s\n", - info), Color: colorprint.Colorless}, - {Str: "The new yuanrong proccesses is running now. The old yuanrong cluster processes will continue" + - " running on the background until `yr stop` is called\n", Color: colorprint.KeywordColor}, - } - colorprint.PrintKeywords(yrOpts.cmdIO.Out, colorMap) - fallthrough - default: - if err := utils.CreateSymbolicLinkWithForce(yrOpts.masterInfoOutput, - constant.DefaultYuanRongCurrentMasterInfoPath); err != nil { - colorprint.PrintInteractive(yrOpts.cmdIO.Out, fmt.Sprintf("WARN: failed to link %s to %s. err: %s \n", - yrOpts.masterInfoOutput, constant.YuanRongCurrentMasterInfoPath, err.Error())) - return nil - } - } - } - - return nil -} - -func clearCmdProcess(cmd *exec.Cmd) { - if cmd.Process == nil { - return - } - if err := cmd.Process.Kill(); err != nil { - // DO NOTHING - } -} - -func untilMasterDeploySucceed(ctx context.Context, block bool) error { - for { - _, err := os.Stat(filepath.Join(yrOpts.deployPath, "master_pid_port.txt")) - if err != nil && os.IsNotExist(err) { - time.Sleep(time.Millisecond * constant.DeployReadyCheckIntervalMs) - continue - } - _, err = os.Stat(yrOpts.masterInfoOutput) - if err != nil && os.IsNotExist(err) { - time.Sleep(time.Millisecond * constant.DeployReadyCheckIntervalMs) - continue - } - if err == nil { - return untilAgentDeploySucceed(ctx, false) // make sure agent is start successfully - } - } -} - -func untilAgentDeploySucceed(ctx context.Context, block bool) error { - for { - _, err := os.Stat(filepath.Join(yrOpts.deployPath, "deploy_std.log")) - if err != nil && os.IsNotExist(err) { - time.Sleep(time.Millisecond * constant.DeployReadyCheckIntervalMs) - continue - } - break - } - - file, err := os.Open(filepath.Join(yrOpts.deployPath, "deploy_std.log")) - if err != nil { - return errors.New("failed to open deploy std file") - } - defer file.Close() - - for { - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if !block && strings.Contains(line, "yuanrong data plane components installation") { - return nil - } - } - } -} - -func yrDeployAgent(path string) error { - agentScriptPath := filepath.Join(path, "deploy", "process", "deploy.sh") - if _, err := os.Stat(agentScriptPath); err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "Failed to start, ", - fmt.Sprintf("%s not exists, check if you have installed yuanrong", agentScriptPath), ".\n") - return err - } - - execName, _ := utils.GetGOOSType() - args := make([]string, 0) - args = append(args, agentScriptPath) - err, args := concatArgs(args) - if err != nil { - return err - } - agentDeployCmd := exec.Command(execName, args...) - - deployStdFile, err := prepareDeployWorkingDir() - if err != nil { - return err - } - agentDeployCmd.Stdout = deployStdFile - agentDeployCmd.Stderr = deployStdFile - - if yrOpts.block { - // when block, use pdeathsig to make sure when this command exit, send sigterm to the deploy.sh - // when no blocked, will just run - agentDeployCmd.SysProcAttr = &syscall.SysProcAttr{ - Pdeathsig: syscall.SIGTERM, - } - } - - if err, _ = utils.ExecCommandUntil(agentDeployCmd, untilAgentDeploySucceed, - constant.DeployReadyCheckTimeoutS, yrOpts.block); err != nil { - clearCmdProcess(agentDeployCmd) - return err - } - - return nil -} - -func addIfNot(value, name, ignore string, result []string) []string { - if value == ignore { - return result - } - result = append(result, fmt.Sprintf("--%s", name)) - result = append(result, value) - return result -} - -func addIfNotAndAddMore(value, name, ignore, more string, result []string) []string { - if value == ignore { - return result - } - result = append(result, fmt.Sprintf("--%s", name)) - result = append(result, value) - result = append(result, more) - return result -} - -func addIfNotBool(value bool, name string, ignore bool, result []string, noArg bool) []string { - if value == ignore { - return result - } - - if noArg { - if value { - result = append(result, fmt.Sprintf("--%s", name)) - } - } else { - strValue := "false" - if value { - strValue = "true" - } - result = append(result, fmt.Sprintf("--%s", name)) - result = append(result, strValue) - } - return result -} - -func concatArgs(result []string) (error, []string) { - if yrOpts.ipAddress == "" { - yrOpts.ipAddress = utils.GetLocalIP() - } - result = addIfNot(yrOpts.ipAddress, "ip_address", "", result) - result = addIfNot(yrOpts.cpuNum, "cpu_num", "", result) - result = addIfNot(yrOpts.deployPath, "deploy_path", "", result) - result = addIfNot(yrOpts.memoryNum, "memory_num", "", result) - result = addIfNot(yrOpts.sharedMemoryNum, "shared_memory_num", "", result) - result = addIfNot(yrOpts.masterInfoOutput, "master_info_output", "", result) - result = addIfNotBool(yrOpts.master, "master", false, result, true) - result = addIfNot(yrOpts.masterInfo, "master_info", "", result) - result = addIfNot(yrOpts.servicesPath, "services_path", "", result) - result = addIfNot(yrOpts.masterIp, "master_ip", "", result) - result = addIfNot(yrOpts.enableInheritEnv, "enable_inherit_env", "", result) - // if use etcd_addr_list, must enable multi master - result = addIfNotAndAddMore(yrOpts.etcdAddrList, "etcd_addr_list", "", "-e", result) - - result = addIfNot(yrOpts.dsNodeTimeoutS, "ds_node_timeout_s", "", result) - result = addIfNot(yrOpts.dsNodeDeadTimeoutS, "ds_node_dead_timeout_s", "", result) - result = addIfNot(yrOpts.dsHeartbeatIntervalMs, "ds_heartbeat_interval_ms", "", result) - - result = addIfNotBool(yrOpts.disableNcCheck, "disable_nc_check", false, result, true) - - restArgsIdx := 2 - result = append(result, os.Args[restArgsIdx:]...) - - for _, param := range result { - if err := utils.ParamCheck(param); err != nil { - return err, nil - } - } - return nil, result -} diff --git a/functionsystem/apps/cli/internal/start/yr_start_test.go b/functionsystem/apps/cli/internal/start/yr_start_test.go deleted file mode 100644 index a780e97..0000000 --- a/functionsystem/apps/cli/internal/start/yr_start_test.go +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package start - -import ( - "cli/constant" - "context" - "errors" - "fmt" - "os" - "os/exec" - "path" - "runtime" - "testing" - "time" - - "cli/pkg/cmdio" - "cli/utils" - - "github.com/agiledragon/gomonkey" - . "github.com/smartystreets/goconvey/convey" -) - -func prepareMockDeployDirAndShellScripts(yuanrongDir string, deployDir string) { - infoContentStr := "master_ip:172.17.0.17,etcd_ip:172.17.0.17,etcd_port:13917,global_scheduler_port:14935,ds_master_port:15269,etcd_peer_port:18275,bus-proxy:27893,bus:24860,ds-worker:33615," - content := fmt.Sprintf(`#!/bin/bash -while [[ "$#" -gt 0 ]]; do - case $1 in - --master_info_output) masterinfooutput="$2"; shift ;; - esac - shift -done -echo "%s" > "%s" -echo "%s" > "$masterinfooutput" -`, - infoContentStr, path.Join(deployDir, "master_pid_port.txt"), infoContentStr) - - if err := os.MkdirAll(path.Join(yuanrongDir, "deploy", "process"), os.ModeDir); err != nil { - fmt.Printf("failed to create mock deploy dir, err: %s\n", err.Error()) - } - if err := os.MkdirAll(path.Dir(constant.DefaultYuanRongCurrentMasterInfoPath), os.ModeDir); err != nil { - fmt.Printf("failed to create mock DefaultYuanRongCurrentMasterInfoPath dir, err: %s\n", err.Error()) - } - if err := os.WriteFile(path.Join(yuanrongDir, "deploy", "process", "deploy.sh"), []byte(content), - 0644); err != nil { - fmt.Printf("failed to write mock deploy shell, err: %s\n", err.Error()) - } -} - -func clearMockDeployDirAndShellScripts(deployDir string) { - _ = os.RemoveAll(deployDir) -} - -type mockFileInfo struct{} - -func (m *mockFileInfo) Name() string { return "" } -func (m *mockFileInfo) Size() int64 { return 0 } -func (m *mockFileInfo) Mode() os.FileMode { return 0 } -func (m *mockFileInfo) ModTime() time.Time { return time.Time{} } -func (m *mockFileInfo) IsDir() bool { return false } -func (m *mockFileInfo) Sys() interface{} { return nil } - -type mockCmd struct { - *exec.Cmd - started bool - waitErr error -} - -func (m *mockCmd) Start() error { - m.started = true - return nil -} - -func (m *mockCmd) Wait() error { - return m.waitErr -} - -func (m *mockCmd) ProcessState() *os.ProcessState { - return &os.ProcessState{} -} - -var ( - workdir, _ = os.Getwd() -) - -func ATestYrStartYuanRong(t *testing.T) { - Convey("Test yrStartYuanRong", t, func() { - Convey("When system is supported and master is true, no error should be returned", func() { - patches := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return true - }) - defer patches.Reset() - - patches.ApplyFunc(yrDeployMaster, func(path string, _ currentMasterInfoDumpPolicy) error { - return nil - }) - - yrOpts = yrStartOptions{ - master: true, - } - - err := yrStartYuanRong(nil, nil) - So(err, ShouldBeNil) - }) - - Convey("When system is supported and master_info is provided, no error should be returned", func() { - patches := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return true - }) - defer patches.Reset() - - patches.ApplyFunc(yrDeployAgent, func(path string) error { - return nil - }) - - yrOpts = yrStartOptions{ - masterInfo: "some_master_info", - } - - err := yrStartYuanRong(nil, nil) - So(err, ShouldBeNil) - }) - - Convey("When system is not supported, an error should be returned", func() { - patches := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return false - }) - defer patches.Reset() - yrOpts = yrStartOptions{ - verbose: 1, - cmdIO: &cmdio.CmdIO{ - Out: os.Stdout, - ErrOut: os.Stderr, - }, - } - err := yrStartYuanRong(nil, nil) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, "unsupported os: "+runtime.GOOS) - }) - - Convey("When both master and master_info are provided, an error should be returned", func() { - patches := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return true - }) - defer patches.Reset() - - yrOpts = yrStartOptions{ - master: true, - masterInfo: "some_master_info", - } - - err := yrStartYuanRong(nil, nil) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, "only one of `master` and `master_info` can be specified") - }) - - Convey("When neither master nor master_info is provided, an error should be returned", func() { - patches := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return true - }) - defer patches.Reset() - - yrOpts = yrStartOptions{} - - err := yrStartYuanRong(nil, nil) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, "one of `master` and `master_info` must be specified") - }) - - Convey("When yrDeployMaster returns an error, the error should be propagated", func() { - patches := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return true - }) - defer patches.Reset() - - expectedError := errors.New("deploy master error") - patches.ApplyFunc(yrDeployMaster, func(path string, _ currentMasterInfoDumpPolicy) error { - return expectedError - }) - - yrOpts = yrStartOptions{ - master: true, - } - - err := yrStartYuanRong(nil, nil) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, expectedError) - }) - - Convey("When yrDeployAgent returns an error, the error should be propagated", func() { - patches := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return true - }) - defer patches.Reset() - - expectedError := errors.New("deploy agent error") - patches.ApplyFunc(yrDeployAgent, func(path string) error { - return expectedError - }) - - yrOpts = yrStartOptions{ - masterInfo: "some_master_info", - } - - err := yrStartYuanRong(nil, nil) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, expectedError) - }) - }) -} - -func prepareMockFiles(files ...string) { - for _, file := range files { - _ = os.MkdirAll(path.Dir(file), os.ModeDir) - _ = os.WriteFile(file, []byte{}, os.ModePerm) - } -} - -func removePreparedMockFiles(files ...string) { - for _, file := range files { - _ = os.RemoveAll(file) - } -} - -func removeSymbolLinks() { - _ = os.RemoveAll(constant.DefaultYuanRongDeployDir) - _ = os.RemoveAll(constant.DefaultYuanRongCurrentMasterInfoPath) -} - -func TestYrDeployMaster(t *testing.T) { - // mock files - mockDeployPath := path.Join("/tmp", "deploy") - mockDeployShellPath := path.Join(constant.YuanRongInstallationDir, "deploy", "process", "deploy.sh") - mockMasterInfoOutputPath := path.Join(mockDeployPath, "master.info") - mockMasterInfoPidOutputPath := path.Join(mockDeployPath, "master_pid_port.txt") - - // clear previous mock files - removePreparedMockFiles(mockDeployShellPath, mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - clearMockDeployDirAndShellScripts(constant.YuanRongInstallationDir) - removeSymbolLinks() - - Convey("Test yrDeployMaster", t, func() { - Convey("When deploy.sh script exists and no existing master info, no error should be returned", func() { - patches := gomonkey.ApplyFunc(utils.GetGOOSType, func() (string, string) { - return "bash", "-c" - }) - - defer patches.Reset() - - yrOpts = yrStartOptions{ - verbose: 1, - block: false, - deployPath: mockDeployPath, - masterInfoOutput: mockMasterInfoOutputPath, - cmdIO: &cmdio.CmdIO{ - Out: os.Stdout, - ErrOut: os.Stderr, - }, - } - - removePreparedMockFiles(mockDeployShellPath, mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - clearMockDeployDirAndShellScripts(constant.YuanRongInstallationDir) - removeSymbolLinks() - prepareMockFiles(mockDeployShellPath, mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - prepareMockDeployDirAndShellScripts(constant.YuanRongInstallationDir, mockDeployPath) - - err := yrDeployMaster(constant.YuanRongInstallationDir, dumpPolicyNew) - So(err, ShouldBeNil) - }) - - Convey("When deploy.sh script does not exist, an error should be returned", func() { - removePreparedMockFiles(mockDeployShellPath, mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - clearMockDeployDirAndShellScripts(constant.YuanRongInstallationDir) - removeSymbolLinks() - prepareMockFiles(mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - - err := yrDeployMaster(constant.YuanRongInstallationDir, dumpPolicyNew) - So(err, ShouldNotBeNil) - So(err.Error(), ShouldContainSubstring, "maybe check if you have installed yuanrong") - }) - - Convey("When utils.ExecCommandUntil returns an error, the error should be propagated", func() { - removePreparedMockFiles(mockDeployShellPath, mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - clearMockDeployDirAndShellScripts(constant.YuanRongInstallationDir) - removeSymbolLinks() - prepareMockFiles(mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - prepareMockDeployDirAndShellScripts(constant.YuanRongInstallationDir, mockDeployPath) - - patches := gomonkey.ApplyFunc(utils.GetGOOSType, func() (string, string) { - return "bash", "-c" - }) - expectedError := errors.New("exec command until error") - patches.ApplyFunc(utils.ExecCommandUntil, func(cmd *exec.Cmd, check func(context.Context) error, timeout int) (error, chan error) { - return expectedError, nil - }) - - patches.ApplyFunc(redirectFileContentToStd, func(ctx context.Context, deployStdFile *os.File) { - return - }) - defer patches.Reset() - - yrOpts = yrStartOptions{ - verbose: 1, - block: false, - deployPath: mockDeployPath, - masterInfoOutput: mockMasterInfoPidOutputPath, - cmdIO: &cmdio.CmdIO{ - Out: os.Stdout, - ErrOut: os.Stderr, - }, - } - - err := yrDeployMaster(constant.YuanRongInstallationDir, dumpPolicyNew) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, expectedError) - }) - - Convey("When yrMakeSymbolLink returns an error, the error should be propagated", func() { - removePreparedMockFiles(mockDeployShellPath, mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - clearMockDeployDirAndShellScripts(constant.YuanRongInstallationDir) - removeSymbolLinks() - prepareMockFiles(mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - prepareMockDeployDirAndShellScripts(constant.YuanRongInstallationDir, mockDeployPath) - - expectedError := errors.New("make symbol link error") - patches := gomonkey.ApplyFunc(utils.GetGOOSType, func() (string, string) { - return "bash", "-c" - }).ApplyFunc(utils.ExecCommandUntil, func(cmd *exec.Cmd, check func(context.Context) error, timeout int) (error, chan error) { - return nil, nil - }).ApplyFunc(redirectFileContentToStd, func(ctx context.Context, deployStdFile *os.File) { - return - }).ApplyFunc(yrMakeSymbolLink, func(dumpPolicy currentMasterInfoDumpPolicy) error { - return expectedError - }) - defer patches.Reset() - - yrOpts = yrStartOptions{ - verbose: 1, - block: false, - deployPath: mockDeployPath, - masterInfoOutput: mockMasterInfoOutputPath, - cmdIO: &cmdio.CmdIO{ - Out: os.Stdout, - ErrOut: os.Stderr, - }, - } - - err := yrDeployMaster(constant.YuanRongInstallationDir, dumpPolicyNew) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, expectedError) - }) - - Convey("When printMasterStartedHelpMsg returns an error, the error should be propagated", func() { - removePreparedMockFiles(mockDeployShellPath, mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - clearMockDeployDirAndShellScripts(constant.YuanRongInstallationDir) - removeSymbolLinks() - prepareMockFiles(mockMasterInfoOutputPath, mockMasterInfoPidOutputPath) - prepareMockDeployDirAndShellScripts(constant.YuanRongInstallationDir, mockDeployPath) - - patches := gomonkey.ApplyFunc(utils.GetGOOSType, func() (string, string) { - return "bash", "-c" - }) - defer patches.Reset() - - patches.ApplyFunc(yrMakeSymbolLink, func(dumpPolicy currentMasterInfoDumpPolicy) error { - return nil - }) - patches.ApplyFunc(redirectFileContentToStd, func(ctx context.Context, deployStdFile *os.File) { - return - }) - expectedError := errors.New("print master started help msg error") - patches.ApplyFunc(printMasterStartedHelpMsg, func() error { - return expectedError - }) - - yrOpts = yrStartOptions{ - verbose: 1, - block: false, - deployPath: mockDeployPath, - masterInfoOutput: mockMasterInfoOutputPath, - cmdIO: &cmdio.CmdIO{ - Out: os.Stdout, - ErrOut: os.Stderr, - }, - } - - err := yrDeployMaster(constant.YuanRongInstallationDir, dumpPolicyNew) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, expectedError) - }) - }) -} diff --git a/functionsystem/apps/cli/internal/status/yr_status.go b/functionsystem/apps/cli/internal/status/yr_status.go deleted file mode 100644 index ef9ecea..0000000 --- a/functionsystem/apps/cli/internal/status/yr_status.go +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package status is start cmd -package status - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "os" - "runtime" - "time" - - "github.com/spf13/cobra" - clientv3 "go.etcd.io/etcd/client/v3" - - "cli/constant" - "cli/pkg/cmdio" - "cli/utils" - "cli/utils/colorprint" -) - -type yrStatusOptions struct { - cmdIO *cmdio.CmdIO - etcdEndpoint string - etcdTlsEnable bool - etcdCaFile string - etcdClientKeyFile string - etcdClientCertFile string -} - -var yrOpts yrStatusOptions - -var yrStatusCmd = &cobra.Command{ - Use: "status", - Short: fmt.Sprintf("status %s Platform", constant.PlatformName), - Long: fmt.Sprintf(`status %s Platform`, constant.PlatformName), - Example: utils.RawFormat(fmt.Sprintf(` - %s status in non-interactive mode: - $ %s status - `, constant.CliName, constant.CliName)), - Args: utils.NoArgs, - RunE: yrStatusYuanRong, -} - -// InitYrCMD init login cmd -func InitYrCMD(cio *cmdio.CmdIO) *cobra.Command { - yrOpts.cmdIO = cio - // 添加命令行参数 - yrStatusCmd.Flags().StringVar(&yrOpts.etcdEndpoint, "etcd_endpoint", "", "ETCD endpoint address") - yrStatusCmd.Flags().BoolVar(&yrOpts.etcdTlsEnable, "etcd_tls_enable", false, "Enable ETCD TLS") - yrStatusCmd.Flags().StringVar(&yrOpts.etcdCaFile, "etcd_ca_file", "", "CA certificate for ETCD service") - yrStatusCmd.Flags().StringVar(&yrOpts.etcdClientKeyFile, "etcd_client_key_file", "", "Client cert for ETCD") - yrStatusCmd.Flags().StringVar(&yrOpts.etcdClientCertFile, "etcd_client_cert_file", "", "Client key for ETCD") - yrStatusCmd.MarkFlagsRequiredTogether("etcd_tls_enable", - "etcd_ca_file", "etcd_client_key_file", "etcd_client_cert_file") - return yrStatusCmd -} - -func yrStatusYuanRong(cmd *cobra.Command, args []string) error { - if !utils.SupportSystem() { - return errors.New(fmt.Sprintf("unsupported os: %s", runtime.GOOS)) - } - - info, err := utils.GetMasterInfoFromFile(constant.YuanRongCurrentMasterInfoPath) - if err != nil || info == nil { - colorprint.PrintFail(yrOpts.cmdIO.Out, "there seems no running yuanrong cluster.\n", err.Error(), ".\n") - return nil - } - if yrOpts.etcdEndpoint == "" { - yrOpts.etcdEndpoint = fmt.Sprintf("%s:%s", info.EtcdIP, info.EtcdPort) - } - - readyAgentCount, err := countAgent() - if err != nil { - return err - } else { - colorprint.PrintSuccess(yrOpts.cmdIO.Out, "YuanRong is running normally,", fmt.Sprintf(` -YuanRong cluster addresses: - functionsystem: %s - datasystem: %s - -YuanRong cluster status: - current running agents: %s -`, info.MasterIP+":"+info.BusPort, info.MasterIP+":"+info.DsWorkerPort, readyAgentCount)) - return nil - } -} - -func countAgent() (string, error) { - cfg := clientv3.Config{ - Endpoints: []string{yrOpts.etcdEndpoint}, - DialTimeout: time.Second, - } - if yrOpts.etcdTlsEnable { - cert, err := tls.LoadX509KeyPair(yrOpts.etcdClientCertFile, yrOpts.etcdClientKeyFile) - if err != nil { - colorprint.PrintFail(yrOpts.cmdIO.Out, "LoadX509KeyPair error", err.Error(), ".\n") - } - - caCert, err := os.ReadFile(yrOpts.etcdCaFile) - if err != nil { - colorprint.PrintFail(yrOpts.cmdIO.Out, "Read ca file error", err.Error(), ".\n") - } - - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - cfg.TLS = &tls.Config{ - RootCAs: caCertPool, - Certificates: []tls.Certificate{cert}, - } - } - - client, err := clientv3.New(cfg) - if err != nil { - colorprint.PrintFail(yrOpts.cmdIO.Out, "Connect to etcd server failed.", err.Error(), ".\n") - return "", err - } - defer client.Close() - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - resp, err := client.Get(ctx, "/yr/readyAgentCount") - if err != nil { - colorprint.PrintFail(yrOpts.cmdIO.Out, "Connect to etcd server failed.", err.Error(), ".\n") - return "", err - } - if resp == nil || len(resp.Kvs) != 1 { - colorprint.PrintFail(yrOpts.cmdIO.Out, "Can not find agent count key", "", ".\n") - return "", errors.New("can not find agent count key") - } - readyAgentCount := resp.Kvs[0].Value - return string(readyAgentCount), nil -} diff --git a/functionsystem/apps/cli/internal/stop/yr_stop.go b/functionsystem/apps/cli/internal/stop/yr_stop.go deleted file mode 100644 index df8834d..0000000 --- a/functionsystem/apps/cli/internal/stop/yr_stop.go +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package stop is start cmd -package stop - -import ( - "errors" - "fmt" - "os" - "path" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "syscall" - "time" - - "github.com/spf13/cobra" - - "cli/constant" - "cli/pkg/cmdio" - "cli/pkg/progress" - "cli/utils" - "cli/utils/colorprint" -) - -type yrStopOptions struct { - cmdIO *cmdio.CmdIO - graceExitTimeout int -} - -var yrOpts yrStopOptions - -var yrStopCmd = &cobra.Command{ - Use: "stop", - Short: fmt.Sprintf("stop %s Platform", constant.PlatformName), - Long: fmt.Sprintf(`stop %s Platform`, constant.PlatformName), - Example: utils.RawFormat(fmt.Sprintf(` - %s stop in non-interactive mode: - $ %s stop - `, constant.CliName, constant.CliName)), - Args: utils.NoArgs, - RunE: yrStopYuanRong, -} - -const ( - defaultGraceExitTimeout = 5 -) - -var ( - errorGetZeroYuanRongProcesses = errors.New("didn't find any yuanrong processes") -) - -// InitYrCMD init login cmd -func InitYrCMD(cio *cmdio.CmdIO) *cobra.Command { - yrOpts.cmdIO = cio - - yrStopCmd.Flags().IntVarP(&yrOpts.graceExitTimeout, "grace_exit_timeout", "g", defaultGraceExitTimeout, - fmt.Sprintf("grace exit timeout given yuanrong processes, "+ - "after this timeout the processes will be killed by force. "+ - "(in seconds, default to %d)", defaultGraceExitTimeout)) - - return yrStopCmd -} - -func yrStopYuanRong(cmd *cobra.Command, args []string) error { - if !utils.SupportSystem() { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, "Failed to stop, ", - fmt.Sprintf("unsupported os: %s", runtime.GOOS), ".\n") - return errors.New(fmt.Sprintf("unsupported os: %s", runtime.GOOS)) - } - yuanRongDir, err := filepath.Abs(constant.YuanRongInstallationDir) - if err != nil { - return err - } - var keywords = []string{ - filepath.Join(yuanRongDir, "deploy", "process", "deploy.sh"), - filepath.Join(yuanRongDir, "functionsystem"), - filepath.Join(yuanRongDir, "datasystem"), - filepath.Join(yuanRongDir, "third_party"), - filepath.Join(yuanRongDir, "runtime"), - } - - clearCurrentMasterInfoSymbolLink() - err = stopYuanRongProcesses(keywords, time.Duration(yrOpts.graceExitTimeout)*time.Second) - if errors.Is(err, errorGetZeroYuanRongProcesses) { - colorprint.PrintSuccess(yrOpts.cmdIO.Out, "Did not find any active Yuanrong processes.", "\n") - } else { - colorprint.PrintSuccess(yrOpts.cmdIO.Out, "Yuanrong stop succeed.", "\n") - } - return nil -} - -func clearCurrentMasterInfoSymbolLink() { - if err := os.RemoveAll(constant.DefaultYuanRongCurrentMasterInfoPath); err != nil { - colorprint.PrintFail(yrOpts.cmdIO.ErrOut, fmt.Sprintf("Failed to remove dumped master info at %s, ", - constant.DefaultYuanRongCurrentMasterInfoPath), - err.Error(), ".\n") - } -} - -type process struct { - PID int - Cmdline string - Comm string - Args []string - Keyword string -} - -// 获取所有进程列表 -func getProcesses() ([]process, error) { - var processes []process - procDir, err := os.ReadDir("/proc") - if err != nil { - return nil, err - } - for _, entry := range procDir { - if !entry.IsDir() { - continue - } - pidStr := entry.Name() - pid, err := strconv.Atoi(pidStr) - if err != nil { - continue - } - // 读取 /proc//cmdline - cmdlinePath := path.Join("/proc", pidStr, "cmdline") - cmdlineBytes, err := os.ReadFile(cmdlinePath) - if err != nil { - continue // 进程可能已退出，跳过 - } - - cmdline := strings.Split(string(cmdlineBytes), "\x00") - if len(cmdline) == 0 { - continue - } - exePath := cmdline[0] - args := cmdline[1:] - processes = append(processes, process{PID: pid, Cmdline: exePath, Comm: path.Base(exePath), Args: args}) - } - return processes, nil -} - -func filterProcesses(processes []process, keywords []string) []process { - var filtered []process - keywordSet := make(map[string]bool) - for _, keyword := range keywords { - keywordSet[keyword] = true - } - for _, p := range processes { - for keyword := range keywordSet { - if strings.Contains(p.Cmdline, keyword) { - p.Keyword = keyword - filtered = append(filtered, p) - break - } - filtered = filterProcessArgs(p, keyword, filtered) - } - } - return filtered -} - -func filterProcessArgs(p process, keyword string, filtered []process) []process { - for _, arg := range p.Args { - if strings.Contains(arg, keyword) { - p.Keyword = keyword - return append(filtered, p) - } - } - return filtered -} - -func groupProcessesByKeywords(processes []process, keywords []string) []process { - var groups [][]process - for _, keyword := range keywords { - var group []process - for _, p := range processes { - if p.Keyword == keyword { - group = append(group, p) - } - } - groups = append(groups, group) - } - - processed := make(map[int]bool) - var orderedProcesses []process - for _, group := range groups { - for _, p := range group { - if !processed[p.PID] { - orderedProcesses = append(orderedProcesses, p) - processed[p.PID] = true - } - } - } - return orderedProcesses -} - -func getProcessName(p process) string { - // bash need more process - if p.Comm == "bash" { - for _, arg := range p.Args { - if strings.Contains(arg, "deploy.sh") { - return "deploy.sh" - } - } - } - return p.Comm -} - -func handleProcess(p process, graceExitTimeout time.Duration) { - proc, err := os.FindProcess(p.PID) - if err != nil { - return - } - - err = proc.Signal(syscall.SIGTERM) - if err != nil { - return - } - - select { - case <-time.After(graceExitTimeout): - colorMap := []colorprint.StringColorInfo{ - {Str: fmt.Sprintf("process %s(%d) is force killed after %ds\n", - getProcessName(p), p.PID, int(graceExitTimeout.Seconds())), Color: colorprint.InteractiveColor}, - } - colorprint.PrintKeywords(yrOpts.cmdIO.Out, colorMap) - err = proc.Signal(syscall.SIGKILL) - if err != nil { - return - } - case <-checkProcessExit(p.PID): - colorprint.PrintSuccess(yrOpts.cmdIO.Out, fmt.Sprintf("process %s(%d) exited\n", - getProcessName(p), p.PID), "") - } -} - -func checkProcessExit(pid int) <-chan bool { - ch := make(chan bool) - go func() { - for { - proc, err := os.FindProcess(pid) - if err != nil { - ch <- true - return - } - err = proc.Signal(syscall.Signal(0)) - if err != nil { - ch <- true - return - } - time.Sleep(time.Millisecond * 100) - } - }() - return ch -} - -func stopYuanRongProcesses(keywords []string, graceExitTimeout time.Duration) error { - processes, err := getProcesses() - if err != nil { - return err - } - - filteredProcesses := filterProcesses(processes, keywords) - orderedProcesses := groupProcessesByKeywords(filteredProcesses, keywords) - total := len(orderedProcesses) - if total == 0 { - return errorGetZeroYuanRongProcesses - } - fmt.Printf("find %d Yuanrong processes\n", total) - - stopProgress := progress.CreateAndDoing("Stopping Yuanrong processes") - var wg sync.WaitGroup - for _, p := range orderedProcesses { - wg.Add(1) - go func(p process) { - defer wg.Done() - handleProcess(p, graceExitTimeout) - }(p) - } - - wg.Wait() - stopProgress.Done("Stop Yuanrong processes") - return nil -} diff --git a/functionsystem/apps/cli/internal/stop/yr_stop_test.go b/functionsystem/apps/cli/internal/stop/yr_stop_test.go deleted file mode 100644 index 5b73e86..0000000 --- a/functionsystem/apps/cli/internal/stop/yr_stop_test.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stop - -import ( - "cli/constant" - "cli/pkg/cmdio" - "cli/utils" - "github.com/agiledragon/gomonkey" - "github.com/smartystreets/goconvey/convey" - "os" - "path" - "path/filepath" - "testing" - - "github.com/spf13/cobra" -) - -// Test function -func TestYrStopYuanRong(t *testing.T) { - convey.Convey("Testing yrStopYuanRong", t, func() { - cmd := &cobra.Command{} - var args []string - yrOpts.cmdIO = &cmdio.CmdIO{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} - yuanRongDir := constant.YuanRongInstallationDir - stopScriptPath := filepath.Join(yuanRongDir, "deploy", "process", "yr_stop.sh") - - convey.Convey("Given an unsupported OS", func() { - patch := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return false - }) - defer patch.Reset() - - convey.Convey("When calling yrStopYuanRong", func() { - err := yrStopYuanRong(cmd, args) - - convey.Convey("Then it should return an error", func() { - convey.So(err, convey.ShouldNotBeNil) - convey.So(err.Error(), convey.ShouldContainSubstring, "unsupported os") - }) - }) - }) - - convey.Convey("Given a missing stop script", func() { - patch := gomonkey.ApplyFunc(utils.SupportSystem, func() bool { - return true - }) - defer patch.Reset() - _ = os.RemoveAll(stopScriptPath) // Ensure file does not exist - - convey.Convey("When calling yrStopYuanRong", func() { - err := yrStopYuanRong(cmd, args) - - convey.Convey("Then it should return an error", func() { - convey.So(err, convey.ShouldNotBeNil) - convey.So(err.Error(), convey.ShouldContainSubstring, "maybe check if you have installed yuanrong") - }) - }) - }) - - convey.Convey("Given a valid stop script", func() { - _ = os.MkdirAll(path.Dir(stopScriptPath), os.ModeDir) - _ = os.WriteFile(stopScriptPath, []byte("#!/bin/bash\necho stop"), 0755) // Create mock script - patch := gomonkey.ApplyFunc(utils.GetGOOSType, func() (string, string) { - return "bash", "-c" - }) - defer patch.Reset() - - convey.Convey("When calling yrStopYuanRong", func() { - err := yrStopYuanRong(cmd, args) - - convey.Convey("Then it should succeed without error", func() { - convey.So(err, convey.ShouldBeNil) - }) - }) - }) - }) -} diff --git a/functionsystem/apps/cli/internal/version/version.go b/functionsystem/apps/cli/internal/version/version.go deleted file mode 100644 index a1e5ccd..0000000 --- a/functionsystem/apps/cli/internal/version/version.go +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package version is version cmd of CLI -package version - -import ( - "github.com/spf13/cobra" - - "cli/constant" - "cli/pkg/cmdio" - "cli/utils" - "cli/utils/colorprint" -) - -type options struct { - cmdIO *cmdio.CmdIO -} - -var opts = &options{} - -var cmd = &cobra.Command{ - Use: "version", - Short: "print the cli's version information", - Long: `print the cli's version information`, - Args: utils.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - colorMap := []colorprint.StringColorInfo{ - {Str: "CLI version: ", Color: colorprint.Colorless}, - {Str: constant.Version, Color: colorprint.KeywordColor}, - {Str: ".\n", Color: colorprint.Colorless}, - {Str: "Using yuanrong at: ", Color: colorprint.Colorless}, - {Str: constant.YuanRongInstallationDir, Color: colorprint.Colorless}, - {Str: "\n", Color: colorprint.Colorless}, - } - colorprint.PrintKeywords(opts.cmdIO.Out, colorMap) - }, -} - -// InitVersionCMD init cmd for version -func InitVersionCMD(cio *cmdio.CmdIO) *cobra.Command { - opts.cmdIO = cio - return cmd -} diff --git a/functionsystem/apps/cli/internal/version/version_test.go b/functionsystem/apps/cli/internal/version/version_test.go deleted file mode 100644 index 42a859d..0000000 --- a/functionsystem/apps/cli/internal/version/version_test.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package version - -import ( - "testing" - - "github.com/agiledragon/gomonkey" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "cli/constant" - "cli/pkg/test" - "cli/utils/colorprint" -) - -func TestInitVersionCMD(t *testing.T) { - testCases := []struct { - name string - version string - buildTime string - wantStd string - }{ - { - name: "get version info", - version: `1.0.0`, - buildTime: `20210218`, - wantStd: "CLI version: " + - colorprint.GetPrintString("1.0.0", colorprint.KeywordColor) + ".\n" + - "Using yuanrong at: " + constant.YuanRongInstallationDir + "\n", - }, - } - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - p := gomonkey.ApplyGlobalVar(&constant.Version, tt.version) - p.ApplyGlobalVar(&constant.Date, tt.buildTime) - defer p.Reset() - - cio, mio := test.MockCmdIO() - cmd := InitVersionCMD(cio) - - cmd.SetArgs([]string{}) - _, err := cmd.ExecuteC() - require.NoError(t, err) - assert.Equal(t, tt.wantStd, mio.Out.String()) - }) - } -} diff --git a/functionsystem/apps/cli/pkg/cmdio/cmdio.go b/functionsystem/apps/cli/pkg/cmdio/cmdio.go deleted file mode 100644 index c943455..0000000 --- a/functionsystem/apps/cli/pkg/cmdio/cmdio.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package cmdio is io of the CLI -package cmdio - -import ( - "io" - "os" -) - -// CmdIO io of cmd -type CmdIO struct { - In io.ReadCloser - Out io.Writer - ErrOut io.Writer -} - -// System io of cmd -func System() *CmdIO { - return &CmdIO{ - In: os.Stdin, - Out: os.Stdout, - ErrOut: os.Stderr, - } -} diff --git a/functionsystem/apps/cli/pkg/progress/progress.go b/functionsystem/apps/cli/pkg/progress/progress.go deleted file mode 100644 index af35026..0000000 --- a/functionsystem/apps/cli/pkg/progress/progress.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package progress use to displays work progress. -package progress - -import ( - "fmt" - "os" - "strings" - "time" -) - -const ( - defaultRepeatCount = 9 - defaultSleepTime = 300 - // FailedDone Failed to done - FailedDone = "fail" -) - -// Progress process info -type Progress struct { - Begin time.Time - DoingMessage string - RepeatCount int - DoneChan chan struct{} -} - -// CreateAndDoing crate a progress and start doing -func CreateAndDoing(doingMessage string) *Progress { - progress := &Progress{ - Begin: time.Now(), - RepeatCount: defaultRepeatCount, - DoingMessage: doingMessage, - DoneChan: make(chan struct{}), - } - go progress.doing() - return progress -} - -// Doing print doing message until done -func (p *Progress) doing() { - for i := 1; ; i++ { - select { - case _, ok := <-p.DoneChan: - if !ok { - // 通道关闭则退出 - return - } - default: - dynamicMessage := strings.Repeat(".", i) - if i == p.RepeatCount { - dynamicMessage = strings.Repeat(" ", i) - i = 0 - } - if _, err := fmt.Fprintf(os.Stdout, "%s %s\r", p.DoingMessage, dynamicMessage); err != nil { - continue - } - time.Sleep(time.Millisecond * defaultSleepTime) - } - } -} - -// Done set Progress done -func (p *Progress) Done(doneMessage string) { - if doneMessage != FailedDone { - elapsed := time.Since(p.Begin) - fmt.Printf("%s costs time %dms\n", doneMessage, elapsed.Milliseconds()) - } - close(p.DoneChan) -} diff --git a/functionsystem/apps/cli/pkg/test/cmdio.go b/functionsystem/apps/cli/pkg/test/cmdio.go deleted file mode 100644 index 328ddb5..0000000 --- a/functionsystem/apps/cli/pkg/test/cmdio.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package test is used for unit tests -package test - -import ( - "bytes" - "io/ioutil" - - "cli/pkg/cmdio" -) - -// CmdIO mock io of cmd -type CmdIO struct { - In *bytes.Buffer - Out *bytes.Buffer - ErrOut *bytes.Buffer -} - -// MockCmdIO mock cmdio for unit tests -func MockCmdIO() (*cmdio.CmdIO, *CmdIO) { - in := &bytes.Buffer{} - out := &bytes.Buffer{} - errOut := &bytes.Buffer{} - cio := &cmdio.CmdIO{ - In: ioutil.NopCloser(in), - Out: out, - ErrOut: errOut, - } - tio := &CmdIO{ - In: in, - Out: out, - ErrOut: errOut, - } - return cio, tio -} diff --git a/functionsystem/apps/cli/utils/cmd_args.go b/functionsystem/apps/cli/utils/cmd_args.go deleted file mode 100644 index 22c6eae..0000000 --- a/functionsystem/apps/cli/utils/cmd_args.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "context" - "fmt" - "os/exec" - "runtime" - "time" - - "github.com/spf13/cobra" -) - -// NoArgs 无参数处理 -func NoArgs(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return nil - } - // 打印未知参数 - return fmt.Errorf("unknown argument %q", args) -} - -// GetGOOSType 返回当前系统应该使用的启动命令 -func GetGOOSType() (string, string) { - if runtime.GOOS == "windows" { - return "CMD", "/C" - } - return "bash", "-c" -} - -// ExecCommandUntil - this function will also check if the command exit, so it will return the channel with wait called -func ExecCommandUntil(cmd *exec.Cmd, stopCond func(ctx context.Context, block bool) error, timeout int, block bool) ( - error, chan error) { - if err := cmd.Start(); err != nil { - fmt.Println("failed to start sub command:", err) - return err, nil - } - var ctx context.Context - var cancel context.CancelFunc - if !block && timeout > 0 { - ctx, cancel = context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - defer cancel() - - // Create a channel to receive the result of stopCond - resultChan := make(chan error, 1) - go func() { - resultChan <- stopCond(ctx, block) - }() - - cmdExitChan := make(chan error, 1) - go func() { - cmdExitChan <- cmd.Wait() - }() - - select { - case err := <-resultChan: - return err, cmdExitChan - case err := <-cmdExitChan: - return err, cmdExitChan - case <-ctx.Done(): - return fmt.Errorf("timeout(%ds) reached", timeout), cmdExitChan - } -} diff --git a/functionsystem/apps/cli/utils/colorprint/ac.go b/functionsystem/apps/cli/utils/colorprint/ac.go deleted file mode 100644 index db7847c..0000000 --- a/functionsystem/apps/cli/utils/colorprint/ac.go +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package colorprint Print with color -// ac Aho-Corasick automation Multi-string matching single-string algorithm -package colorprint - -import ( - "strings" -) - -// two results are added for each match during AC automaton cutting, -// and two results need to be rolled back during rollback -const acEachAppend = 2 - -// acNode ac Automaton Node -type acNode struct { - fail *acNode - next map[rune]*acNode - isPattern bool -} - -func newAcNode() *acNode { - return &acNode{ - fail: nil, - isPattern: false, - next: map[rune]*acNode{}, - } -} - -// AcAutoMachine trie tree of ac automaton -type AcAutoMachine struct { - root *acNode -} - -func newAcAutoMachine() *AcAutoMachine { - return &AcAutoMachine{ - root: newAcNode(), - } -} - -func (ac *AcAutoMachine) addPattern(pattern string) { - chars := []rune(pattern) - iter := ac.root - for _, c := range chars { - if _, ok := iter.next[c]; !ok { - iter.next[c] = newAcNode() - } - iter = iter.next[c] - } - iter.isPattern = true -} - -func (ac *AcAutoMachine) buildTrie() { - var trie []*acNode - trie = append(trie, ac.root) - for len(trie) != 0 { - parent := trie[0] - trie = trie[1:] - if parent == nil { - continue - } - for char, child := range parent.next { - trie = append(trie, ac.getTrieChild(parent, child, char)) - } - } -} - -func (ac *AcAutoMachine) getTrieChild(parent, child *acNode, char rune) *acNode { - if parent == ac.root { - child.fail = ac.root - } else { - failAcNode := parent.fail - for failAcNode != nil { - if _, ok := failAcNode.next[char]; ok { - child.fail = failAcNode.next[char] - break - } - failAcNode = failAcNode.fail - } - if failAcNode == nil { - child.fail = ac.root - } - } - return child -} - -func (ac *AcAutoMachine) splitWithAC(content string) []string { - chars := []rune(content) - iter := ac.root - var start, end int - var results []string - for i, c := range chars { - _, ok := iter.next[c] - for !ok && iter != ac.root { - iter = iter.fail - } - if _, ok = iter.next[c]; ok { - if iter == ac.root { // this is the first match, record the start position - start = i - } - iter = iter.next[c] - if iter.isPattern { - results, end = ac.matchString(content, start, i, end, results) - } - } - } - if len(results) == 0 { - return []string{content} - } - if end != len(chars)-1 && len(chars) != 0 { - results = append(results, string([]rune(content)[end+1:])) - } - return results -} - -func (ac *AcAutoMachine) matchString(content string, start, current, end int, results []string) ([]string, int) { - currentStr := string([]rune(content)[start : current+1]) - if len(results) > 0 && strings.Contains(currentStr, results[len(results)-1]) && - currentStr != results[len(results)-1] { - end -= len([]rune(results[len(results)-acEachAppend])) + 1 - results = results[:len(results)-acEachAppend] - } - if end == 0 { - results = append(results, string([]rune(content)[end:start])) - } else { - results = append(results, string([]rune(content)[end+1:start])) - } - results = append(results, currentStr) - return results, current -} diff --git a/functionsystem/apps/cli/utils/colorprint/colorprint.go b/functionsystem/apps/cli/utils/colorprint/colorprint.go deleted file mode 100644 index 34edf1c..0000000 --- a/functionsystem/apps/cli/utils/colorprint/colorprint.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package colorprint Print with color -package colorprint - -import ( - "io" - "strings" -) - -// PrintSuccess Print Success Statement -func PrintSuccess(w io.Writer, successWord, str string) { - PrintColorWord(w, successWord, SuccessColor) - if str != "" { - PrintlnColorWord(w, str, Colorless) - } -} - -// PrintFail Print Failure Statement -func PrintFail(w io.Writer, str, errorWord, suffix string) { - if str != "" { - PrintColorWord(w, str, Colorless) - } - PrintColorWord(w, errorWord, ErrorColor) - if suffix != "" { - PrintColorWord(w, suffix, Colorless) - } -} - -// PrintInteractive Print interactive statements -func PrintInteractive(w io.Writer, interactiveWord string) { - PrintColorWord(w, interactiveWord, InteractiveColor) -} - -// PrintKeywords Print statements with keywords -func PrintKeywords(w io.Writer, printMap []StringColorInfo) { - for _, keywordConfig := range printMap { - PrintColorWord(w, keywordConfig.Str, keywordConfig.Color) - } -} - -// PrintColorWord Print a colored string to w -func PrintColorWord(w io.Writer, str string, color ColorType) { - printColorString(w, str, color) -} - -// PrintlnColorWord Print a colored string and \n to w -func PrintlnColorWord(w io.Writer, str string, color ColorType) { - str = str + "\n" - printColorString(w, str, color) -} - -// GetPrintString Get colored string to compare -func GetPrintString(str string, color ColorType) string { - return getPrintString(str, color) -} - -// NewColorConfig create PrintStringConfig -func NewColorConfig() *PrintStringConfig { - colorConfig := PrintStringConfig{ - ColorConfig: defaultColorConfig, - } - return &colorConfig -} - -// AddColorConfig Add a Config Before Using PrintStringUsingColorConfig -func (p *PrintStringConfig) AddColorConfig(str string, color ColorType) *PrintStringConfig { - if str == "" { - return p - } - p.ColorConfig[str] = color - return p -} - -// DeleteColorConfig Delete a Config Before Using PrintStringUsingColorConfig -func (p *PrintStringConfig) DeleteColorConfig(str string) *PrintStringConfig { - if str == "" { - return p - } - delete(p.ColorConfig, str) - return p -} - -// GetPrintStringUsingMap Print string to w using color config -func (p *PrintStringConfig) GetPrintStringUsingMap(printMap []StringColorInfo) { - for _, keywordConfig := range printMap { - getPrintString(keywordConfig.Str, keywordConfig.Color) - } - return -} - -// PrintStringUsingColorConfig Print string to w using color config -func (p *PrintStringConfig) PrintStringUsingColorConfig(w io.Writer, str string) { - result := p.split(str) - for _, printString := range result { - color, ok := p.ColorConfig[printString] - if ok { - printColorString(w, printString, color) - continue - } - printColorString(w, printString, Colorless) - } - return -} - -// GetPrintStringUsingColorConfig Get print string to compare -func (p *PrintStringConfig) GetPrintStringUsingColorConfig(str string) string { - result := p.split(str) - var coloredString strings.Builder - for _, printString := range result { - color, ok := p.ColorConfig[printString] - if ok { - coloredString.WriteString(getPrintString(printString, color)) - continue - } - coloredString.WriteString(getPrintString(printString, Colorless)) - } - return coloredString.String() -} - -func (p *PrintStringConfig) split(str string) []string { - ac := newAcAutoMachine() - for acStr := range p.ColorConfig { - ac.addPattern(acStr) - } - ac.buildTrie() - strArr := strings.Split(str, "\n") - var result []string - for _, content := range strArr { - result = append(result, ac.splitWithAC(content)...) - result = append(result, "\n") - } - return result[:len(result)-1] -} diff --git a/functionsystem/apps/cli/utils/colorprint/colorprint_unix.go b/functionsystem/apps/cli/utils/colorprint/colorprint_unix.go deleted file mode 100644 index 65cfaf2..0000000 --- a/functionsystem/apps/cli/utils/colorprint/colorprint_unix.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package colorprint Print with color -package colorprint - -import ( - "fmt" - "io" - "strconv" - "strings" -) - -func printColorString(w io.Writer, str string, color ColorType) { - if color == Colorless { - fmt.Fprint(w, str) - return - } - result := getPrintString(str, color) - fmt.Fprint(w, result) -} - -func getColorCode(color ColorType) string { - var fontCode int - var backgroundCode int - color = checkColor(color) - var colorCode strings.Builder - if color.FontColor == None && color.BackgroundColor == None { - return "0" - } - if color.FontColor != None { - fontCode = color.FontColor - 1 + frontMask - if color.FontLight { - fontCode += light - } - colorCode.WriteString(strconv.Itoa(fontCode)) - } - - if color.BackgroundColor != None { - backgroundCode = color.BackgroundColor - 1 + backgroundMask - if color.BackgroundLight { - backgroundCode += light - } - if colorCode.Len() != 0 { - colorCode.WriteString(";") - } - colorCode.WriteString(strconv.Itoa(backgroundCode)) - } - return colorCode.String() -} - -func checkColor(color ColorType) ColorType { - // linux does not support gray - if color.FontColor == Gray { - color.FontColor = None - } - if color.BackgroundColor == Gray { - color.BackgroundColor = None - } - return color -} - -func getPrintString(str string, color ColorType) string { - if color == Colorless { - return str - } - var printString strings.Builder - colorCode := getColorCode(color) - printString.WriteString(fmt.Sprintf(controllerMask, colorCode)) - printString.WriteString(str) - printString.WriteString(defaultConfig) - return printString.String() -} diff --git a/functionsystem/apps/cli/utils/colorprint/types.go b/functionsystem/apps/cli/utils/colorprint/types.go deleted file mode 100644 index 698fdf6..0000000 --- a/functionsystem/apps/cli/utils/colorprint/types.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package colorprint Print with color -package colorprint - -// ColorType Font color setting structure -type ColorType struct { - FontColor int - FontLight bool - BackgroundColor int - BackgroundLight bool -} - -// StringColorInfo Keyword color -type StringColorInfo struct { - Str string - Color ColorType -} - -// PrintStringConfig Keywords printing structure -type PrintStringConfig struct { - ColorConfig map[string]ColorType -} - -var defaultColorConfig = map[string]ColorType{ - "success": {FontColor: Green}, - "succeed": {FontColor: Green}, - "succeeded": {FontColor: Green}, - "Succeeded": {FontColor: Green}, - "failed": {FontColor: Red}, - "fail": {FontColor: Red}, -} - -// Colorless Default Color -var Colorless = ColorType{ - FontColor: None, - FontLight: false, - BackgroundColor: None, - BackgroundLight: false, -} - -// SuccessColor Success Statement Color -var SuccessColor = ColorType{ - FontColor: Green, - FontLight: false, - BackgroundColor: None, - BackgroundLight: false, -} - -// ErrorColor Error Statement Color -var ErrorColor = ColorType{ - FontColor: Red, - FontLight: false, - BackgroundColor: None, - BackgroundLight: false, -} - -// InteractiveColor Interactive Statement Color -var InteractiveColor = ColorType{ - FontColor: Yellow, - FontLight: true, - BackgroundColor: None, - BackgroundLight: false, -} - -// KeywordColor Keyword Statement Color -var KeywordColor = ColorType{ - FontColor: Cyan, - FontLight: false, - BackgroundColor: None, - BackgroundLight: false, -} diff --git a/functionsystem/apps/cli/utils/colorprint/types_unix.go b/functionsystem/apps/cli/utils/colorprint/types_unix.go deleted file mode 100644 index 8f68182..0000000 --- a/functionsystem/apps/cli/utils/colorprint/types_unix.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package colorprint Print with color -package colorprint - -const ( - // None colorless - None int = iota - // Black color - Black - // Red color - Red - // Green color - Green - // Yellow color - Yellow - // Blue color - Blue - // Purple color - Purple - // Cyan color - Cyan - // White color - White - // Gray color - Gray -) - -const ( - defaultConfig = "\x1b[0m" - frontMask = 30 - backgroundMask = 40 - light = 60 - controllerMask = "\x1b[%sm" -) diff --git a/functionsystem/apps/cli/utils/file.go b/functionsystem/apps/cli/utils/file.go deleted file mode 100644 index 454964a..0000000 --- a/functionsystem/apps/cli/utils/file.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "os" - "path/filepath" -) - -// Exists exists Whether the path exists -func Exists(path string) bool { - if _, err := filepath.Abs(path); err != nil { - return false - } - - if _, err := ReadFileInfoWithTimeout(path); err != nil { - if os.IsExist(err) { - return true - } - return false - } - - return true -} diff --git a/functionsystem/apps/cli/utils/file_test.go b/functionsystem/apps/cli/utils/file_test.go deleted file mode 100644 index d43bca2..0000000 --- a/functionsystem/apps/cli/utils/file_test.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "testing" -) - -func TestExists(t *testing.T) { - var cases = []struct { - in string // input - expected bool // expected result - }{ - {"", false}, - {"./file.go", true}, - {"./notexists", false}, - {"/%$&*", false}, - } - for _, c := range cases { - actual := Exists(c.in) - if actual != c.expected { - t.Errorf("Exists(%s) = %v; expected %v", c.in, actual, c.expected) - } - } -} diff --git a/functionsystem/apps/cli/utils/localip.go b/functionsystem/apps/cli/utils/localip.go deleted file mode 100644 index 582d831..0000000 --- a/functionsystem/apps/cli/utils/localip.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import "net" - -// GetLocalIP - -// try to use interfaces addresses -// if fail, fall back to 127.0.0.1 -func GetLocalIP() string { - fallbackChoice := "127.0.0.1" - - // try to use interfaces addresses - interfaces, err := net.Interfaces() - if err != nil { - return fallbackChoice - } - for _, iface := range interfaces { - addrs, err := iface.Addrs() - if err != nil { - continue - } - for _, addr := range addrs { - // Check if the address is an IP address - ipnet, ok := addr.(*net.IPNet) - if !ok || ipnet.IP.IsLoopback() { - continue - } - if ipnet.IP.To4() == nil { - continue - } - if "127.0.0.1" == ipnet.IP.String() || "172.17.0.1" == ipnet.IP.String() { - continue - } - return ipnet.IP.String() - } - } - return fallbackChoice -} diff --git a/functionsystem/apps/cli/utils/masterinfo.go b/functionsystem/apps/cli/utils/masterinfo.go deleted file mode 100644 index d58d33a..0000000 --- a/functionsystem/apps/cli/utils/masterinfo.go +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "io/ioutil" - "reflect" - "sort" - "strings" -) - -const pairLen = 2 - -// MasterInfo is the master info string in a struct -type MasterInfo struct { - MasterIP string `key:"master_ip"` - EtcdIP string `key:"etcd_ip"` - EtcdPort string `key:"etcd_port"` - GlobalSchedulerPort string `key:"global_scheduler_port"` - DsMasterPort string `key:"ds_master_port"` - EtcdPeerPort string `key:"etcd_peer_port"` - BusProxyPort string `key:"bus-proxy"` - BusPort string `key:"bus"` - DsWorkerPort string `key:"ds-worker"` - LocalIP string `key:"local_ip"` - EtcdAddrList []string `key:"etcd_addr_list"` -} - -// GetMasterInfoString returns the master info string -func GetMasterInfoString(filename string) (string, error) { - content, err := ioutil.ReadFile(filename) - if err != nil { - return "", err - } - return strings.TrimSpace(string(content)), nil -} - -// ExtractMasterInfo parses the input content string into a MasterInfo struct. -func ExtractMasterInfo(content string) *MasterInfo { - masterInfo := &MasterInfo{} - keyValues := strings.Split(content, ",") - - for _, kv := range keyValues { - pair := strings.Split(kv, ":") - if len(pair) != pairLen { - continue - } - key := strings.TrimSpace(pair[0]) - value := strings.TrimSpace(pair[1]) - - setFieldByKey(masterInfo, key, value) - } - return masterInfo -} - -// setFieldByKey sets the field in MasterInfo based on the key tag. -func setFieldByKey(masterInfo *MasterInfo, key, value string) { - masterInfoValue := reflect.ValueOf(masterInfo).Elem() - masterInfoType := masterInfoValue.Type() - - for i := 0; i < masterInfoType.NumField(); i++ { - field := masterInfoType.Field(i) - tag := field.Tag.Get("key") - if tag == key { - fieldValue := masterInfoValue.Field(i) - if fieldValue.Kind() == reflect.Slice { - fieldValue.Set(reflect.Append(fieldValue, reflect.ValueOf(value))) - } else { - fieldValue.SetString(value) - } - break - } - } -} - -// GetMasterInfoFromFile returns a master info in a struct -func GetMasterInfoFromFile(filename string) (*MasterInfo, error) { - content, err := GetMasterInfoString(filename) - if err != nil { - return nil, err - } - return ExtractMasterInfo(content), nil -} - -// IsMasterNode check if the master info belongs to a master node, which is started by "yr start --master ..." command -func (mi *MasterInfo) IsMasterNode() bool { - if len(mi.EtcdAddrList) > 0 { - for _, e := range mi.EtcdAddrList { - if mi.LocalIP == e { - return true - } - } - return false - } - if mi.MasterIP == mi.LocalIP { - return true - } - return false -} - -// IsSameCluster check if the incoming master info belongs to the same cluster of self -func (mi *MasterInfo) IsSameCluster(incomingMi *MasterInfo) bool { - if len(mi.EtcdAddrList) != len(incomingMi.EtcdAddrList) { - return false - } - if len(mi.EtcdAddrList) == 0 { - if mi.MasterIP == incomingMi.MasterIP && mi.EtcdIP == incomingMi.EtcdIP && mi.EtcdPort == incomingMi.EtcdPort { - return true - } - return false - } - sort.Sort(sort.StringSlice(mi.EtcdAddrList)) - sort.Sort(sort.StringSlice(incomingMi.EtcdAddrList)) - for i, v := range mi.EtcdAddrList { - if incomingMi.EtcdAddrList[i] != v { - return false - } - } - return true -} diff --git a/functionsystem/apps/cli/utils/memory.go b/functionsystem/apps/cli/utils/memory.go deleted file mode 100644 index 91e0740..0000000 --- a/functionsystem/apps/cli/utils/memory.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "bufio" - "fmt" - "os" - "strings" -) - -// Memory - -type Memory struct { - MemTotal uint64 - MemFree uint64 -} - -// ReadMemoryStats - returns memory info from /proc/meminfo -func ReadMemoryStats() (*Memory, error) { - file, err := os.Open("/proc/meminfo") - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - var totalMem, freeMem uint64 - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "MemTotal") { - _, err := fmt.Sscanf(line, "MemTotal: %d kB", &totalMem) - if err != nil { - return nil, err - } - } - if strings.HasPrefix(line, "MemFree") { - _, err := fmt.Sscanf(line, "MemFree: %d kB", &freeMem) - if err != nil { - return nil, err - } - } - } - - if totalMem+freeMem == 0 { - return nil, fmt.Errorf("failed to read any memory info at %s", file.Name()) - } - - return &Memory{ - MemTotal: totalMem, - MemFree: freeMem, - }, nil -} diff --git a/functionsystem/apps/cli/utils/memory_test.go b/functionsystem/apps/cli/utils/memory_test.go deleted file mode 100644 index 8bbecf5..0000000 --- a/functionsystem/apps/cli/utils/memory_test.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "io" - "os" - "testing" - - "github.com/agiledragon/gomonkey" - "github.com/smartystreets/goconvey/convey" -) - -func TestReadMemoryStats(t *testing.T) { - memInfoContent := `MemTotal: 16384 kB -MemFree: 8192 kB -MemAvailable: 8192 kB -` - - tmpFile, err := os.CreateTemp("", "meminfo_test") - if err != nil { - t.Fatal(err) - } - defer tmpFile.Close() - _, err = tmpFile.WriteString(memInfoContent) - if err != nil { - t.Fatal(err) - } - - patches := gomonkey.ApplyFunc(os.Open, func(name string) (*os.File, error) { - if name == "/proc/meminfo" { - tmpFile.Seek(0, io.SeekStart) - return tmpFile, nil - } - return os.Open(name) - }) - defer patches.Reset() - - convey.Convey("Test ReadMemoryStats", t, func() { - memStats, err := ReadMemoryStats() - - convey.So(err, convey.ShouldBeNil) - convey.So(memStats.MemTotal, convey.ShouldEqual, 16384) // 16384 KB = 16 MB - convey.So(memStats.MemFree, convey.ShouldEqual, 8192) // 8192 KB = 8 MB - }) -} diff --git a/functionsystem/apps/cli/utils/raw.go b/functionsystem/apps/cli/utils/raw.go deleted file mode 100644 index 6e73763..0000000 --- a/functionsystem/apps/cli/utils/raw.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "strings" -) - -// RawFormat format raw string -// raw := ` -// line -// Bar -// ` -// Is equivalent to "\n\tFoo\n\tBar\n" or "\n Foo\n Bar\n" -// I want to convert it to " Foo\n Bar\n" -func RawFormat(raw string) string { - if len(raw) == 0 { - return "" - } - - // Remove the first line \n - if raw[0] == '\n' { - raw = raw[1:] - } - // split raw by \n - lines := strings.Split(raw, "\n") - - // convert \t at the beginning of each line to two space indentation - for i, line := range lines { - if strings.HasPrefix(line, "\t") { - lines[i] = strings.Replace(line, "\t", " ", 1) - } - if strings.HasPrefix(line, " ") { - lines[i] = strings.Replace(line, " ", " ", 1) - } - } - - return strings.Join(lines, "\n") -} diff --git a/functionsystem/apps/cli/utils/raw_test.go b/functionsystem/apps/cli/utils/raw_test.go deleted file mode 100644 index 46afdf9..0000000 --- a/functionsystem/apps/cli/utils/raw_test.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import "testing" - -func TestRawFormat(t *testing.T) { - var cases = []struct { - in string // input - expected string // expected result - }{ - {"\n\tFoo\n\tBar\n", " Foo\n Bar\n"}, - {"\n Foo\n Bar\n", " Foo\n Bar\n"}, - } - for _, c := range cases { - actual := RawFormat(c.in) - if actual != c.expected { - t.Errorf("Exists(%s) = %v; expected %v", c.in, actual, c.expected) - } - } -} diff --git a/functionsystem/apps/cli/utils/reader.go b/functionsystem/apps/cli/utils/reader.go deleted file mode 100644 index 8514fd8..0000000 --- a/functionsystem/apps/cli/utils/reader.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package reader provides ReadFile with timeConsumption - -package utils - -import ( - "fmt" - "io/ioutil" - "os" - "time" -) - -// MaxReadFileTime elapsed time allowed to read config file from disk -const MaxReadFileTime = 10 - -// ReadFileWithTimeout is to ReadFile and count timeConsumption at same time -func ReadFileWithTimeout(configFile string) ([]byte, error) { - stopCh := make(chan struct{}) - go printTimeOut(stopCh) - data, err := ioutil.ReadFile(configFile) - close(stopCh) - return data, err -} - -// ReadFileInfoWithTimeout is to Read FileInfo and count timeConsumption at same time -func ReadFileInfoWithTimeout(filePath string) (os.FileInfo, error) { - stopCh := make(chan struct{}) - go printTimeOut(stopCh) - fileInfo, err := os.Stat(filePath) - close(stopCh) - return fileInfo, err -} - -// printTimeOut print error info every 10s after timeout -func printTimeOut(stopCh <-chan struct{}) { - if stopCh == nil { - os.Exit(0) - return - } - timer := time.NewTicker(time.Second * MaxReadFileTime) - count := 0 - for { - <-timer.C - select { - case _, ok := <-stopCh: - if !ok { - timer.Stop() - return - } - default: - count += MaxReadFileTime - fmt.Printf("ReadFile Timeout: elapsed time %ds\n", count) - } - } -} diff --git a/functionsystem/apps/cli/utils/reader_test.go b/functionsystem/apps/cli/utils/reader_test.go deleted file mode 100644 index bb9cb4c..0000000 --- a/functionsystem/apps/cli/utils/reader_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package reader provides ReadFile with timeConsumption -package utils - -import ( - "io/ioutil" - "os" - "testing" - "time" - - "github.com/agiledragon/gomonkey" - "github.com/stretchr/testify/assert" -) - -func TestReadFileWithTimeout(t *testing.T) { - patch := gomonkey.ApplyFunc(ioutil.ReadFile, func(string) ([]byte, error) { - return nil, nil - }) - data, _ := ReadFileWithTimeout("/sn/home") - assert.Nil(t, data) - patch.Reset() -} - -func TestReadFileInfoWithTimeout(t *testing.T) { - patch := gomonkey.ApplyFunc(os.Stat, func(string) (os.FileInfo, error) { - return nil, nil - }) - fileInfo, _ := ReadFileInfoWithTimeout("/sn/home") - assert.Nil(t, fileInfo) - patch.Reset() -} - -func TestPrintTimeout(t *testing.T) { - stopCh := make(chan struct{}) - go printTimeOut(stopCh) - time.Sleep(time.Second * 15) - close(stopCh) -} - -func TestPrintTimeoutErr(t *testing.T) { - test := 0 - patch := gomonkey.ApplyFunc(os.Exit, func(code int) { - test++ - }) - printTimeOut(nil) - assert.EqualValues(t, test, 1) - patch.Reset() -} diff --git a/functionsystem/apps/cli/utils/symbollink.go b/functionsystem/apps/cli/utils/symbollink.go deleted file mode 100644 index 6189374..0000000 --- a/functionsystem/apps/cli/utils/symbollink.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "fmt" - "os" -) - -// CreateSymbolicLinkWithForce creates a symbolic link from target to link. -// If the target path already exists (whether it's a file, directory, or symlink), it will be removed first. -func CreateSymbolicLinkWithForce(target, link string) error { - linkInfo, err := os.Lstat(link) - if err == nil { - if linkInfo.Mode()&os.ModeSymlink != 0 { - err = os.Remove(link) - if err != nil { - return fmt.Errorf("failed to remove existing symlink: %v", err) - } - } else if linkInfo.IsDir() { - err = os.RemoveAll(link) - if err != nil { - return fmt.Errorf("failed to remove existing directory: %v", err) - } - } else { - err = os.Remove(link) - if err != nil { - return fmt.Errorf("failed to remove existing file: %v", err) - } - } - } else if !os.IsNotExist(err) { - return fmt.Errorf("failed to check existing symlink: %v", err) - } - - err = os.Symlink(target, link) - if err != nil { - return fmt.Errorf("failed to create symlink: %v", err) - } - - return nil -} diff --git a/functionsystem/apps/cli/utils/util.go b/functionsystem/apps/cli/utils/util.go deleted file mode 100644 index 2af6b0e..0000000 --- a/functionsystem/apps/cli/utils/util.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "encoding/json" - "errors" - "regexp" - "runtime" - "strings" -) - -var invalidRe *regexp.Regexp = regexp.MustCompile("[$&!?*;<>{}|`\n\\[\\]\\\\]") - -// SupportSystem 当前只支持Linux -func SupportSystem() bool { - return runtime.GOOS == "linux" -} - -// ParamCheck command parameter invalid character check -func ParamCheck(param string) error { - if len(param) == 0 || invalidRe == nil { - return nil - } - if !strings.HasPrefix(param, "-") { - // value check - if !invalidRe.MatchString(param) { - return nil - } - if json.Valid([]byte(param)) { - // json pattern - return nil - } - return errors.New("invalid parameter value: " + param) - } - - index := strings.Index(param, "=") - if index == -1 { - if invalidRe.MatchString(param) { - return errors.New("invalid parameter field: " + param) - } - return nil - } - - field := param[:index] - // field check - if invalidRe.MatchString(field) { - return errors.New("invalid parameter: " + param) - } - - if index != len(param)-1 { - value := param[index+1:] - // value check - if !invalidRe.MatchString(value) { - return nil - } - if json.Valid([]byte(value)) { - // json pattern - return nil - } - return errors.New("invalid parameter: " + param) - } - return nil -} diff --git a/functionsystem/apps/cli/utils/util_test.go b/functionsystem/apps/cli/utils/util_test.go deleted file mode 100644 index c0138e9..0000000 --- a/functionsystem/apps/cli/utils/util_test.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package utils - -import ( - "runtime" - "testing" - - "github.com/smartystreets/goconvey/convey" -) - -func TestSupportSystem(t *testing.T) { - convey.Convey("TestSupportSystem", t, func() { - if runtime.GOOS == "linux" { - res := SupportSystem() - convey.So(res, convey.ShouldEqual, true) - } else { - res := SupportSystem() - convey.So(res, convey.ShouldEqual, false) - } - }) -} - -func TestParamCheck(t *testing.T) { - convey.Convey("TestParamCheck", t, func() { - err := ParamCheck("--master-info") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("-a") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("1234") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("{\"a\": \"|;*{}[]<>\"}") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("[\"a\"]") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("-a*") - convey.So(err, convey.ShouldNotBeNil) - convey.So(err.Error(), convey.ShouldEqual, "invalid parameter field: -a*") - - err = ParamCheck("{;}") - convey.So(err, convey.ShouldNotBeNil) - convey.So(err.Error(), convey.ShouldEqual, "invalid parameter value: {;}") - - err = ParamCheck("<>") - convey.So(err, convey.ShouldNotBeNil) - convey.So(err.Error(), convey.ShouldEqual, "invalid parameter value: <>") - - err = ParamCheck("-a=") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("--a=b") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("--a={}") - convey.So(err, convey.ShouldBeNil) - - err = ParamCheck("--a*=b") - convey.So(err, convey.ShouldNotBeNil) - convey.So(err.Error(), convey.ShouldEqual, "invalid parameter: --a*=b") - - err = ParamCheck("--a=*") - convey.So(err, convey.ShouldNotBeNil) - convey.So(err.Error(), convey.ShouldEqual, "invalid parameter: --a=*") - }) -} diff --git a/functionsystem/apps/meta_service/function_repo/storage/kv.go b/functionsystem/apps/meta_service/function_repo/storage/kv.go new file mode 100644 index 0000000..33526b8 --- /dev/null +++ b/functionsystem/apps/meta_service/function_repo/storage/kv.go @@ -0,0 +1,4594 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package storage + +import ( + "context" + "io" + + "meta_service/common/logger/log" + "meta_service/common/codec" + "meta_service/common/snerror" + "meta_service/function_repo/errmsg" + "meta_service/common/engine" +) + +const ( + defaultPageIndex = 1 + defaultPageSize = 1000 +) + +type generatedKV struct { + keyCodecs [13]codec.Codec + valCodecs [13]codec.Codec + eng engine.Engine +} + +func newKV(eng engine.Engine, prefix string) *generatedKV { + kv := &generatedKV{eng: eng} + + + kv.keyCodecs[0] = codec.NewReflectCodec(prefix + "/AliasRoutingIndex", true) + + kv.valCodecs[0] = codec.NewJSONCodec() + + kv.keyCodecs[1] = codec.NewReflectCodec(prefix + "/Trigger", true) + + kv.valCodecs[1] = codec.NewJSONCodec() + + kv.keyCodecs[2] = codec.NewReflectCodec(prefix + "/TriggerFunctionIndex", true) + + kv.valCodecs[2] = codec.NewJSONCodec() + + kv.keyCodecs[3] = codec.NewReflectCodec(prefix + "/FunctionResourceIDIndex", true) + + kv.valCodecs[3] = codec.NewJSONCodec() + + kv.keyCodecs[4] = codec.NewReflectCodec(prefix + "/ObjectRefIndex", true) + + kv.valCodecs[4] = codec.NewJSONCodec() + + kv.keyCodecs[5] = codec.NewReflectCodec(prefix + "/Alias", true) + + kv.valCodecs[5] = codec.NewJSONCodec() + + kv.keyCodecs[6] = codec.NewReflectCodec(prefix + "/LayerFunctionIndex", true) + + kv.valCodecs[6] = codec.NewJSONCodec() + + kv.keyCodecs[7] = codec.NewReflectCodec(prefix + "/FunctionVersion", true) + + kv.valCodecs[7] = codec.NewJSONCodec() + + kv.keyCodecs[8] = codec.NewReflectCodec(prefix + "/LayerCountIndex", true) + + kv.valCodecs[8] = codec.NewJSONCodec() + + kv.keyCodecs[9] = codec.NewReflectCodec(prefix + "/Layer", true) + + kv.valCodecs[9] = codec.NewJSONCodec() + + kv.keyCodecs[10] = codec.NewReflectCodec(prefix + "/UncontrolledObject", true) + + kv.valCodecs[10] = codec.NewJSONCodec() + + kv.keyCodecs[11] = codec.NewReflectCodec(prefix + "/FunctionStatus", true) + + kv.valCodecs[11] = codec.NewJSONCodec() + + kv.keyCodecs[12] = codec.NewReflectCodec("/yr/podpools/info", false) + + kv.valCodecs[12] = codec.NewJSONCodec() + + return kv +} + +func (kv *generatedKV) Put(ctx context.Context, key, value string) error { + return kv.eng.Put(ctx, key, value) +} + +func (kv *generatedKV) BeginTx(ctx context.Context) *generatedTx { + return &generatedTx{ + keyCodecs: kv.keyCodecs, + valCodecs: kv.valCodecs, + t: kv.eng.BeginTx(ctx), + } +} + +type pageInfo struct { + Begin, Size int +} + +func newPageInfo(pageIndex, pageSize int) (pageInfo, error) { + if pageIndex != 0 { + if pageSize != 0 { + return pageInfo{ + Begin: pageSize * (pageIndex - 1), + Size: pageSize, + }, nil + } + return pageInfo{}, errmsg.PageError + } + return pageInfo{ + Begin: defaultPageSize * (defaultPageIndex - 1), + Size: defaultPageSize, + }, nil +} + +type generatedTx struct { + keyCodecs [13]codec.Codec + valCodecs [13]codec.Codec + t engine.Transaction + ops []func() error +} + +func (t *generatedTx) Put(key, value string) { + t.t.Put(key, value) +} + +func (t *generatedTx) Delete(key string) { + t.t.Del(key) +} + +func (t *generatedTx) DeleteRange(prefix string) { + t.t.DelPrefix(prefix) +} + +func (t *generatedTx) Commit() error { + if err := t.t.Commit(); err != nil { + log.GetLogger().Errorf("failed to commit transaction: %s", err.Error()) + return snerror.New(errmsg.TransactionFailed, err.Error()) + } + return nil +} + +func (t *generatedTx) Cancel() { + t.t.Cancel() +} + +func (kv *generatedKV) AliasRoutingIndexGet(ctx context.Context, key AliasRoutingIndexKey) (AliasRoutingIndexValue, error) { + k, err := kv.keyCodecs[0].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasRoutingIndexGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasRoutingIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexValue{}, errmsg.EtcdInternalError + } + + var res AliasRoutingIndexValue + err = kv.valCodecs[0].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) AliasRoutingIndexCount(ctx context.Context, prefix AliasRoutingIndexKey) (int64, error) { + p, err := kv.keyCodecs[0].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasRoutingIndexCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [AliasRoutingIndex]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) AliasRoutingIndexFirstInRange(ctx context.Context, prefix AliasRoutingIndexKey) (AliasRoutingIndexKey, AliasRoutingIndexValue, error) { + p, err := kv.keyCodecs[0].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasRoutingIndexFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.EtcdInternalError + } + + var key AliasRoutingIndexKey + var val AliasRoutingIndexValue + if err := kv.keyCodecs[0].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[0].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) AliasRoutingIndexLastInRange(ctx context.Context, prefix AliasRoutingIndexKey) (AliasRoutingIndexKey, AliasRoutingIndexValue, error) { + p, err := kv.keyCodecs[0].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasRoutingIndexLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.EtcdInternalError + } + + var key AliasRoutingIndexKey + var val AliasRoutingIndexValue + if err := kv.keyCodecs[0].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[0].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexKey{}, AliasRoutingIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) AliasRoutingIndexStream(ctx context.Context, prefix AliasRoutingIndexKey, by engine.SortBy) *AliasRoutingIndexPrepareStmt { + return &AliasRoutingIndexPrepareStmt{ + keyCodecs: kv.keyCodecs[0], + valCodecs: kv.valCodecs[0], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type AliasRoutingIndexPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix AliasRoutingIndexKey + by engine.SortBy + filters []AliasRoutingIndexFilterFunc +} + +type AliasRoutingIndexFilterFunc func(key AliasRoutingIndexKey, val AliasRoutingIndexValue) bool + +func (ps *AliasRoutingIndexPrepareStmt) Filter(filter AliasRoutingIndexFilterFunc) *AliasRoutingIndexPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *AliasRoutingIndexPrepareStmt) Execute() ([]AliasRoutingIndexTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key AliasRoutingIndexKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val AliasRoutingIndexValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return AliasRoutingIndexTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("AliasRoutingIndexStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(AliasRoutingIndexTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []AliasRoutingIndexTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(AliasRoutingIndexTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *AliasRoutingIndexPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]AliasRoutingIndexTuple, int, error) { + log.GetLogger().Debugf("AliasRoutingIndexExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) AliasRoutingIndexPut(ctx context.Context, key AliasRoutingIndexKey, val AliasRoutingIndexValue) error { + k, err := kv.keyCodecs[0].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[0].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [AliasRoutingIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("AliasRoutingIndexPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [AliasRoutingIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) AliasRoutingIndexDelete(ctx context.Context, key AliasRoutingIndexKey) error { + k, err := kv.keyCodecs[0].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("AliasRoutingIndexDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [AliasRoutingIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) AliasRoutingIndexGet(key AliasRoutingIndexKey) (AliasRoutingIndexValue, error) { + k, err := t.keyCodecs[0].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasRoutingIndexGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasRoutingIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexValue{}, errmsg.EtcdInternalError + } + + var res AliasRoutingIndexValue + err = t.valCodecs[0].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [AliasRoutingIndex]: %s", err.Error()) + return AliasRoutingIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) AliasRoutingIndexGetRange(prefix AliasRoutingIndexKey) ([]AliasRoutingIndexTuple, error) { + p, err := t.keyCodecs[0].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasRoutingIndexGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]AliasRoutingIndexTuple, len(keys)) + for i := range tuples { + var key AliasRoutingIndexKey + if err := t.keyCodecs[0].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val AliasRoutingIndexValue + if err := t.valCodecs[0].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [AliasRoutingIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = AliasRoutingIndexTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) AliasRoutingIndexPut(key AliasRoutingIndexKey, val AliasRoutingIndexValue) error { + k, err := t.keyCodecs[0].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[0].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [AliasRoutingIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasRoutingIndexPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) AliasRoutingIndexDelete(key AliasRoutingIndexKey) error { + k, err := t.keyCodecs[0].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasRoutingIndexDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) AliasRoutingIndexDeleteRange(prefix AliasRoutingIndexKey) error { + p, err := t.keyCodecs[0].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [AliasRoutingIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasRoutingIndexDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) TriggerGet(ctx context.Context, key TriggerKey) (TriggerValue, error) { + k, err := kv.keyCodecs[1].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Trigger]: %s", err.Error()) + return TriggerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [Trigger]: %s", err.Error()) + return TriggerValue{}, errmsg.EtcdInternalError + } + + var res TriggerValue + err = kv.valCodecs[1].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [Trigger]: %s", err.Error()) + return TriggerValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) TriggerCount(ctx context.Context, prefix TriggerKey) (int64, error) { + p, err := kv.keyCodecs[1].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Trigger]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [Trigger]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) TriggerFirstInRange(ctx context.Context, prefix TriggerKey) (TriggerKey, TriggerValue, error) { + p, err := kv.keyCodecs[1].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerKey{}, TriggerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.EtcdInternalError + } + + var key TriggerKey + var val TriggerValue + if err := kv.keyCodecs[1].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[1].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) TriggerLastInRange(ctx context.Context, prefix TriggerKey) (TriggerKey, TriggerValue, error) { + p, err := kv.keyCodecs[1].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerKey{}, TriggerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.EtcdInternalError + } + + var key TriggerKey + var val TriggerValue + if err := kv.keyCodecs[1].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[1].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Trigger]: %s", err.Error()) + return TriggerKey{}, TriggerValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) TriggerStream(ctx context.Context, prefix TriggerKey, by engine.SortBy) *TriggerPrepareStmt { + return &TriggerPrepareStmt{ + keyCodecs: kv.keyCodecs[1], + valCodecs: kv.valCodecs[1], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type TriggerPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix TriggerKey + by engine.SortBy + filters []TriggerFilterFunc +} + +type TriggerFilterFunc func(key TriggerKey, val TriggerValue) bool + +func (ps *TriggerPrepareStmt) Filter(filter TriggerFilterFunc) *TriggerPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *TriggerPrepareStmt) Execute() ([]TriggerTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Trigger]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key TriggerKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Trigger]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val TriggerValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Trigger]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return TriggerTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("TriggerStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(TriggerTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [Trigger]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []TriggerTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [Trigger]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(TriggerTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *TriggerPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]TriggerTuple, int, error) { + log.GetLogger().Debugf("TriggerExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) TriggerPut(ctx context.Context, key TriggerKey, val TriggerValue) error { + k, err := kv.keyCodecs[1].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Trigger]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[1].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [Trigger]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [Trigger]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) TriggerDelete(ctx context.Context, key TriggerKey) error { + k, err := kv.keyCodecs[1].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Trigger]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [Trigger]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) TriggerGet(key TriggerKey) (TriggerValue, error) { + k, err := t.keyCodecs[1].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Trigger]: %s", err.Error()) + return TriggerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [Trigger]: %s", err.Error()) + return TriggerValue{}, errmsg.EtcdInternalError + } + + var res TriggerValue + err = t.valCodecs[1].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Trigger]: %s", err.Error()) + return TriggerValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) TriggerGetRange(prefix TriggerKey) ([]TriggerTuple, error) { + p, err := t.keyCodecs[1].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Trigger]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [Trigger]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]TriggerTuple, len(keys)) + for i := range tuples { + var key TriggerKey + if err := t.keyCodecs[1].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [Trigger]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val TriggerValue + if err := t.valCodecs[1].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Trigger]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = TriggerTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) TriggerPut(key TriggerKey, val TriggerValue) error { + k, err := t.keyCodecs[1].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Trigger]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[1].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [Trigger]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) TriggerDelete(key TriggerKey) error { + k, err := t.keyCodecs[1].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Trigger]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) TriggerDeleteRange(prefix TriggerKey) error { + p, err := t.keyCodecs[1].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Trigger]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) TriggerFunctionIndexGet(ctx context.Context, key TriggerFunctionIndexKey) (TriggerFunctionIndexValue, error) { + k, err := kv.keyCodecs[2].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerFunctionIndexGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var res TriggerFunctionIndexValue + err = kv.valCodecs[2].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) TriggerFunctionIndexCount(ctx context.Context, prefix TriggerFunctionIndexKey) (int64, error) { + p, err := kv.keyCodecs[2].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerFunctionIndexCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [TriggerFunctionIndex]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) TriggerFunctionIndexFirstInRange(ctx context.Context, prefix TriggerFunctionIndexKey) (TriggerFunctionIndexKey, TriggerFunctionIndexValue, error) { + p, err := kv.keyCodecs[2].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerFunctionIndexFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var key TriggerFunctionIndexKey + var val TriggerFunctionIndexValue + if err := kv.keyCodecs[2].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[2].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) TriggerFunctionIndexLastInRange(ctx context.Context, prefix TriggerFunctionIndexKey) (TriggerFunctionIndexKey, TriggerFunctionIndexValue, error) { + p, err := kv.keyCodecs[2].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerFunctionIndexLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var key TriggerFunctionIndexKey + var val TriggerFunctionIndexValue + if err := kv.keyCodecs[2].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[2].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexKey{}, TriggerFunctionIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) TriggerFunctionIndexStream(ctx context.Context, prefix TriggerFunctionIndexKey, by engine.SortBy) *TriggerFunctionIndexPrepareStmt { + return &TriggerFunctionIndexPrepareStmt{ + keyCodecs: kv.keyCodecs[2], + valCodecs: kv.valCodecs[2], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type TriggerFunctionIndexPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix TriggerFunctionIndexKey + by engine.SortBy + filters []TriggerFunctionIndexFilterFunc +} + +type TriggerFunctionIndexFilterFunc func(key TriggerFunctionIndexKey, val TriggerFunctionIndexValue) bool + +func (ps *TriggerFunctionIndexPrepareStmt) Filter(filter TriggerFunctionIndexFilterFunc) *TriggerFunctionIndexPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *TriggerFunctionIndexPrepareStmt) Execute() ([]TriggerFunctionIndexTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key TriggerFunctionIndexKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val TriggerFunctionIndexValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return TriggerFunctionIndexTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("TriggerFunctionIndexStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(TriggerFunctionIndexTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []TriggerFunctionIndexTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(TriggerFunctionIndexTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *TriggerFunctionIndexPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]TriggerFunctionIndexTuple, int, error) { + log.GetLogger().Debugf("TriggerFunctionIndexExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) TriggerFunctionIndexPut(ctx context.Context, key TriggerFunctionIndexKey, val TriggerFunctionIndexValue) error { + k, err := kv.keyCodecs[2].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[2].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerFunctionIndexPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) TriggerFunctionIndexDelete(ctx context.Context, key TriggerFunctionIndexKey) error { + k, err := kv.keyCodecs[2].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("TriggerFunctionIndexDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) TriggerFunctionIndexGet(key TriggerFunctionIndexKey) (TriggerFunctionIndexValue, error) { + k, err := t.keyCodecs[2].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerFunctionIndexGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return TriggerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var res TriggerFunctionIndexValue + err = t.valCodecs[2].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [TriggerFunctionIndex]: %s", err.Error()) + return TriggerFunctionIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) TriggerFunctionIndexGetRange(prefix TriggerFunctionIndexKey) ([]TriggerFunctionIndexTuple, error) { + p, err := t.keyCodecs[2].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerFunctionIndexGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]TriggerFunctionIndexTuple, len(keys)) + for i := range tuples { + var key TriggerFunctionIndexKey + if err := t.keyCodecs[2].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val TriggerFunctionIndexValue + if err := t.valCodecs[2].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [TriggerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = TriggerFunctionIndexTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) TriggerFunctionIndexPut(key TriggerFunctionIndexKey, val TriggerFunctionIndexValue) error { + k, err := t.keyCodecs[2].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[2].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerFunctionIndexPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) TriggerFunctionIndexDelete(key TriggerFunctionIndexKey) error { + k, err := t.keyCodecs[2].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerFunctionIndexDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) TriggerFunctionIndexDeleteRange(prefix TriggerFunctionIndexKey) error { + p, err := t.keyCodecs[2].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [TriggerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: TriggerFunctionIndexDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) FunctionResourceIDIndexGet(ctx context.Context, key FunctionResourceIDIndexKey) (FunctionResourceIDIndexValue, error) { + k, err := kv.keyCodecs[3].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionResourceIDIndexGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionResourceIDIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexValue{}, errmsg.EtcdInternalError + } + + var res FunctionResourceIDIndexValue + err = kv.valCodecs[3].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) FunctionResourceIDIndexCount(ctx context.Context, prefix FunctionResourceIDIndexKey) (int64, error) { + p, err := kv.keyCodecs[3].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionResourceIDIndexCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [FunctionResourceIDIndex]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) FunctionResourceIDIndexFirstInRange(ctx context.Context, prefix FunctionResourceIDIndexKey) (FunctionResourceIDIndexKey, FunctionResourceIDIndexValue, error) { + p, err := kv.keyCodecs[3].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionResourceIDIndexFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.EtcdInternalError + } + + var key FunctionResourceIDIndexKey + var val FunctionResourceIDIndexValue + if err := kv.keyCodecs[3].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[3].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) FunctionResourceIDIndexLastInRange(ctx context.Context, prefix FunctionResourceIDIndexKey) (FunctionResourceIDIndexKey, FunctionResourceIDIndexValue, error) { + p, err := kv.keyCodecs[3].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionResourceIDIndexLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.EtcdInternalError + } + + var key FunctionResourceIDIndexKey + var val FunctionResourceIDIndexValue + if err := kv.keyCodecs[3].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[3].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexKey{}, FunctionResourceIDIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) FunctionResourceIDIndexStream(ctx context.Context, prefix FunctionResourceIDIndexKey, by engine.SortBy) *FunctionResourceIDIndexPrepareStmt { + return &FunctionResourceIDIndexPrepareStmt{ + keyCodecs: kv.keyCodecs[3], + valCodecs: kv.valCodecs[3], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type FunctionResourceIDIndexPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix FunctionResourceIDIndexKey + by engine.SortBy + filters []FunctionResourceIDIndexFilterFunc +} + +type FunctionResourceIDIndexFilterFunc func(key FunctionResourceIDIndexKey, val FunctionResourceIDIndexValue) bool + +func (ps *FunctionResourceIDIndexPrepareStmt) Filter(filter FunctionResourceIDIndexFilterFunc) *FunctionResourceIDIndexPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *FunctionResourceIDIndexPrepareStmt) Execute() ([]FunctionResourceIDIndexTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key FunctionResourceIDIndexKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val FunctionResourceIDIndexValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return FunctionResourceIDIndexTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("FunctionResourceIDIndexStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(FunctionResourceIDIndexTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []FunctionResourceIDIndexTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(FunctionResourceIDIndexTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *FunctionResourceIDIndexPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]FunctionResourceIDIndexTuple, int, error) { + log.GetLogger().Debugf("FunctionResourceIDIndexExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) FunctionResourceIDIndexPut(ctx context.Context, key FunctionResourceIDIndexKey, val FunctionResourceIDIndexValue) error { + k, err := kv.keyCodecs[3].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[3].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionResourceIDIndexPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) FunctionResourceIDIndexDelete(ctx context.Context, key FunctionResourceIDIndexKey) error { + k, err := kv.keyCodecs[3].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionResourceIDIndexDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) FunctionResourceIDIndexGet(key FunctionResourceIDIndexKey) (FunctionResourceIDIndexValue, error) { + k, err := t.keyCodecs[3].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionResourceIDIndexGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionResourceIDIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexValue{}, errmsg.EtcdInternalError + } + + var res FunctionResourceIDIndexValue + err = t.valCodecs[3].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [FunctionResourceIDIndex]: %s", err.Error()) + return FunctionResourceIDIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) FunctionResourceIDIndexGetRange(prefix FunctionResourceIDIndexKey) ([]FunctionResourceIDIndexTuple, error) { + p, err := t.keyCodecs[3].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionResourceIDIndexGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]FunctionResourceIDIndexTuple, len(keys)) + for i := range tuples { + var key FunctionResourceIDIndexKey + if err := t.keyCodecs[3].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val FunctionResourceIDIndexValue + if err := t.valCodecs[3].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [FunctionResourceIDIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = FunctionResourceIDIndexTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) FunctionResourceIDIndexPut(key FunctionResourceIDIndexKey, val FunctionResourceIDIndexValue) error { + k, err := t.keyCodecs[3].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[3].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionResourceIDIndexPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) FunctionResourceIDIndexDelete(key FunctionResourceIDIndexKey) error { + k, err := t.keyCodecs[3].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionResourceIDIndexDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) FunctionResourceIDIndexDeleteRange(prefix FunctionResourceIDIndexKey) error { + p, err := t.keyCodecs[3].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionResourceIDIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionResourceIDIndexDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) ObjectRefIndexGet(ctx context.Context, key ObjectRefIndexKey) (ObjectRefIndexValue, error) { + k, err := kv.keyCodecs[4].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("ObjectRefIndexGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return ObjectRefIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexValue{}, errmsg.EtcdInternalError + } + + var res ObjectRefIndexValue + err = kv.valCodecs[4].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) ObjectRefIndexCount(ctx context.Context, prefix ObjectRefIndexKey) (int64, error) { + p, err := kv.keyCodecs[4].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("ObjectRefIndexCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [ObjectRefIndex]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) ObjectRefIndexFirstInRange(ctx context.Context, prefix ObjectRefIndexKey) (ObjectRefIndexKey, ObjectRefIndexValue, error) { + p, err := kv.keyCodecs[4].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("ObjectRefIndexFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.EtcdInternalError + } + + var key ObjectRefIndexKey + var val ObjectRefIndexValue + if err := kv.keyCodecs[4].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[4].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) ObjectRefIndexLastInRange(ctx context.Context, prefix ObjectRefIndexKey) (ObjectRefIndexKey, ObjectRefIndexValue, error) { + p, err := kv.keyCodecs[4].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("ObjectRefIndexLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.EtcdInternalError + } + + var key ObjectRefIndexKey + var val ObjectRefIndexValue + if err := kv.keyCodecs[4].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[4].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexKey{}, ObjectRefIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) ObjectRefIndexStream(ctx context.Context, prefix ObjectRefIndexKey, by engine.SortBy) *ObjectRefIndexPrepareStmt { + return &ObjectRefIndexPrepareStmt{ + keyCodecs: kv.keyCodecs[4], + valCodecs: kv.valCodecs[4], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type ObjectRefIndexPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix ObjectRefIndexKey + by engine.SortBy + filters []ObjectRefIndexFilterFunc +} + +type ObjectRefIndexFilterFunc func(key ObjectRefIndexKey, val ObjectRefIndexValue) bool + +func (ps *ObjectRefIndexPrepareStmt) Filter(filter ObjectRefIndexFilterFunc) *ObjectRefIndexPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *ObjectRefIndexPrepareStmt) Execute() ([]ObjectRefIndexTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key ObjectRefIndexKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val ObjectRefIndexValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return ObjectRefIndexTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("ObjectRefIndexStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(ObjectRefIndexTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []ObjectRefIndexTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(ObjectRefIndexTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *ObjectRefIndexPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]ObjectRefIndexTuple, int, error) { + log.GetLogger().Debugf("ObjectRefIndexExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) ObjectRefIndexPut(ctx context.Context, key ObjectRefIndexKey, val ObjectRefIndexValue) error { + k, err := kv.keyCodecs[4].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[4].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [ObjectRefIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("ObjectRefIndexPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [ObjectRefIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) ObjectRefIndexDelete(ctx context.Context, key ObjectRefIndexKey) error { + k, err := kv.keyCodecs[4].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("ObjectRefIndexDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [ObjectRefIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) ObjectRefIndexGet(key ObjectRefIndexKey) (ObjectRefIndexValue, error) { + k, err := t.keyCodecs[4].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: ObjectRefIndexGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return ObjectRefIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexValue{}, errmsg.EtcdInternalError + } + + var res ObjectRefIndexValue + err = t.valCodecs[4].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [ObjectRefIndex]: %s", err.Error()) + return ObjectRefIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) ObjectRefIndexGetRange(prefix ObjectRefIndexKey) ([]ObjectRefIndexTuple, error) { + p, err := t.keyCodecs[4].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: ObjectRefIndexGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]ObjectRefIndexTuple, len(keys)) + for i := range tuples { + var key ObjectRefIndexKey + if err := t.keyCodecs[4].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val ObjectRefIndexValue + if err := t.valCodecs[4].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [ObjectRefIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = ObjectRefIndexTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) ObjectRefIndexPut(key ObjectRefIndexKey, val ObjectRefIndexValue) error { + k, err := t.keyCodecs[4].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[4].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [ObjectRefIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: ObjectRefIndexPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) ObjectRefIndexDelete(key ObjectRefIndexKey) error { + k, err := t.keyCodecs[4].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: ObjectRefIndexDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) ObjectRefIndexDeleteRange(prefix ObjectRefIndexKey) error { + p, err := t.keyCodecs[4].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [ObjectRefIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: ObjectRefIndexDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) AliasGet(ctx context.Context, key AliasKey) (AliasValue, error) { + k, err := kv.keyCodecs[5].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Alias]: %s", err.Error()) + return AliasValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [Alias]: %s", err.Error()) + return AliasValue{}, errmsg.EtcdInternalError + } + + var res AliasValue + err = kv.valCodecs[5].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [Alias]: %s", err.Error()) + return AliasValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) AliasCount(ctx context.Context, prefix AliasKey) (int64, error) { + p, err := kv.keyCodecs[5].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Alias]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [Alias]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) AliasFirstInRange(ctx context.Context, prefix AliasKey) (AliasKey, AliasValue, error) { + p, err := kv.keyCodecs[5].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasKey{}, AliasValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.EtcdInternalError + } + + var key AliasKey + var val AliasValue + if err := kv.keyCodecs[5].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[5].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) AliasLastInRange(ctx context.Context, prefix AliasKey) (AliasKey, AliasValue, error) { + p, err := kv.keyCodecs[5].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("AliasLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasKey{}, AliasValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.EtcdInternalError + } + + var key AliasKey + var val AliasValue + if err := kv.keyCodecs[5].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[5].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Alias]: %s", err.Error()) + return AliasKey{}, AliasValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) AliasStream(ctx context.Context, prefix AliasKey, by engine.SortBy) *AliasPrepareStmt { + return &AliasPrepareStmt{ + keyCodecs: kv.keyCodecs[5], + valCodecs: kv.valCodecs[5], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type AliasPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix AliasKey + by engine.SortBy + filters []AliasFilterFunc +} + +type AliasFilterFunc func(key AliasKey, val AliasValue) bool + +func (ps *AliasPrepareStmt) Filter(filter AliasFilterFunc) *AliasPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *AliasPrepareStmt) Execute() ([]AliasTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Alias]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key AliasKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Alias]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val AliasValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Alias]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return AliasTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("AliasStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(AliasTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [Alias]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []AliasTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [Alias]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(AliasTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *AliasPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]AliasTuple, int, error) { + log.GetLogger().Debugf("AliasExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) AliasPut(ctx context.Context, key AliasKey, val AliasValue) error { + k, err := kv.keyCodecs[5].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Alias]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[5].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [Alias]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("AliasPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [Alias]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) AliasDelete(ctx context.Context, key AliasKey) error { + k, err := kv.keyCodecs[5].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Alias]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("AliasDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [Alias]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) AliasGet(key AliasKey) (AliasValue, error) { + k, err := t.keyCodecs[5].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Alias]: %s", err.Error()) + return AliasValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return AliasValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [Alias]: %s", err.Error()) + return AliasValue{}, errmsg.EtcdInternalError + } + + var res AliasValue + err = t.valCodecs[5].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Alias]: %s", err.Error()) + return AliasValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) AliasGetRange(prefix AliasKey) ([]AliasTuple, error) { + p, err := t.keyCodecs[5].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Alias]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [Alias]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]AliasTuple, len(keys)) + for i := range tuples { + var key AliasKey + if err := t.keyCodecs[5].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [Alias]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val AliasValue + if err := t.valCodecs[5].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Alias]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = AliasTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) AliasPut(key AliasKey, val AliasValue) error { + k, err := t.keyCodecs[5].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Alias]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[5].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [Alias]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) AliasDelete(key AliasKey) error { + k, err := t.keyCodecs[5].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Alias]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) AliasDeleteRange(prefix AliasKey) error { + p, err := t.keyCodecs[5].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Alias]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: AliasDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) LayerFunctionIndexGet(ctx context.Context, key LayerFunctionIndexKey) (LayerFunctionIndexValue, error) { + k, err := kv.keyCodecs[6].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerFunctionIndexGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var res LayerFunctionIndexValue + err = kv.valCodecs[6].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) LayerFunctionIndexCount(ctx context.Context, prefix LayerFunctionIndexKey) (int64, error) { + p, err := kv.keyCodecs[6].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerFunctionIndexCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [LayerFunctionIndex]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) LayerFunctionIndexFirstInRange(ctx context.Context, prefix LayerFunctionIndexKey) (LayerFunctionIndexKey, LayerFunctionIndexValue, error) { + p, err := kv.keyCodecs[6].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerFunctionIndexFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var key LayerFunctionIndexKey + var val LayerFunctionIndexValue + if err := kv.keyCodecs[6].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[6].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) LayerFunctionIndexLastInRange(ctx context.Context, prefix LayerFunctionIndexKey) (LayerFunctionIndexKey, LayerFunctionIndexValue, error) { + p, err := kv.keyCodecs[6].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerFunctionIndexLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var key LayerFunctionIndexKey + var val LayerFunctionIndexValue + if err := kv.keyCodecs[6].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[6].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexKey{}, LayerFunctionIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) LayerFunctionIndexStream(ctx context.Context, prefix LayerFunctionIndexKey, by engine.SortBy) *LayerFunctionIndexPrepareStmt { + return &LayerFunctionIndexPrepareStmt{ + keyCodecs: kv.keyCodecs[6], + valCodecs: kv.valCodecs[6], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type LayerFunctionIndexPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix LayerFunctionIndexKey + by engine.SortBy + filters []LayerFunctionIndexFilterFunc +} + +type LayerFunctionIndexFilterFunc func(key LayerFunctionIndexKey, val LayerFunctionIndexValue) bool + +func (ps *LayerFunctionIndexPrepareStmt) Filter(filter LayerFunctionIndexFilterFunc) *LayerFunctionIndexPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *LayerFunctionIndexPrepareStmt) Execute() ([]LayerFunctionIndexTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key LayerFunctionIndexKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val LayerFunctionIndexValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return LayerFunctionIndexTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("LayerFunctionIndexStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(LayerFunctionIndexTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []LayerFunctionIndexTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(LayerFunctionIndexTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *LayerFunctionIndexPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]LayerFunctionIndexTuple, int, error) { + log.GetLogger().Debugf("LayerFunctionIndexExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) LayerFunctionIndexPut(ctx context.Context, key LayerFunctionIndexKey, val LayerFunctionIndexValue) error { + k, err := kv.keyCodecs[6].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[6].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [LayerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("LayerFunctionIndexPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [LayerFunctionIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) LayerFunctionIndexDelete(ctx context.Context, key LayerFunctionIndexKey) error { + k, err := kv.keyCodecs[6].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("LayerFunctionIndexDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [LayerFunctionIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) LayerFunctionIndexGet(key LayerFunctionIndexKey) (LayerFunctionIndexValue, error) { + k, err := t.keyCodecs[6].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerFunctionIndexGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerFunctionIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexValue{}, errmsg.EtcdInternalError + } + + var res LayerFunctionIndexValue + err = t.valCodecs[6].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [LayerFunctionIndex]: %s", err.Error()) + return LayerFunctionIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) LayerFunctionIndexGetRange(prefix LayerFunctionIndexKey) ([]LayerFunctionIndexTuple, error) { + p, err := t.keyCodecs[6].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerFunctionIndexGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]LayerFunctionIndexTuple, len(keys)) + for i := range tuples { + var key LayerFunctionIndexKey + if err := t.keyCodecs[6].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val LayerFunctionIndexValue + if err := t.valCodecs[6].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [LayerFunctionIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = LayerFunctionIndexTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) LayerFunctionIndexPut(key LayerFunctionIndexKey, val LayerFunctionIndexValue) error { + k, err := t.keyCodecs[6].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[6].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [LayerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerFunctionIndexPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) LayerFunctionIndexDelete(key LayerFunctionIndexKey) error { + k, err := t.keyCodecs[6].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerFunctionIndexDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) LayerFunctionIndexDeleteRange(prefix LayerFunctionIndexKey) error { + p, err := t.keyCodecs[6].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerFunctionIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerFunctionIndexDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) FunctionVersionGet(ctx context.Context, key FunctionVersionKey) (FunctionVersionValue, error) { + k, err := kv.keyCodecs[7].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionVersion]: %s", err.Error()) + return FunctionVersionValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionVersionGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionVersionValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [FunctionVersion]: %s", err.Error()) + return FunctionVersionValue{}, errmsg.EtcdInternalError + } + + var res FunctionVersionValue + err = kv.valCodecs[7].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionVersion]: %s", err.Error()) + return FunctionVersionValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) FunctionVersionCount(ctx context.Context, prefix FunctionVersionKey) (int64, error) { + p, err := kv.keyCodecs[7].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionVersion]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionVersionCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [FunctionVersion]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) FunctionVersionFirstInRange(ctx context.Context, prefix FunctionVersionKey) (FunctionVersionKey, FunctionVersionValue, error) { + p, err := kv.keyCodecs[7].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionVersionFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.EtcdInternalError + } + + var key FunctionVersionKey + var val FunctionVersionValue + if err := kv.keyCodecs[7].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[7].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) FunctionVersionLastInRange(ctx context.Context, prefix FunctionVersionKey) (FunctionVersionKey, FunctionVersionValue, error) { + p, err := kv.keyCodecs[7].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionVersionLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.EtcdInternalError + } + + var key FunctionVersionKey + var val FunctionVersionValue + if err := kv.keyCodecs[7].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[7].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionVersion]: %s", err.Error()) + return FunctionVersionKey{}, FunctionVersionValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) FunctionVersionStream(ctx context.Context, prefix FunctionVersionKey, by engine.SortBy) *FunctionVersionPrepareStmt { + return &FunctionVersionPrepareStmt{ + keyCodecs: kv.keyCodecs[7], + valCodecs: kv.valCodecs[7], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type FunctionVersionPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix FunctionVersionKey + by engine.SortBy + filters []FunctionVersionFilterFunc +} + +type FunctionVersionFilterFunc func(key FunctionVersionKey, val FunctionVersionValue) bool + +func (ps *FunctionVersionPrepareStmt) Filter(filter FunctionVersionFilterFunc) *FunctionVersionPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *FunctionVersionPrepareStmt) Execute() ([]FunctionVersionTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionVersion]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key FunctionVersionKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionVersion]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val FunctionVersionValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionVersion]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return FunctionVersionTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("FunctionVersionStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(FunctionVersionTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [FunctionVersion]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []FunctionVersionTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [FunctionVersion]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(FunctionVersionTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *FunctionVersionPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]FunctionVersionTuple, int, error) { + log.GetLogger().Debugf("FunctionVersionExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) FunctionVersionPut(ctx context.Context, key FunctionVersionKey, val FunctionVersionValue) error { + k, err := kv.keyCodecs[7].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionVersion]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[7].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [FunctionVersion]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionVersionPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [FunctionVersion]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) FunctionVersionDelete(ctx context.Context, key FunctionVersionKey) error { + k, err := kv.keyCodecs[7].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionVersion]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionVersionDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [FunctionVersion]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) FunctionVersionGet(key FunctionVersionKey) (FunctionVersionValue, error) { + k, err := t.keyCodecs[7].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionVersion]: %s", err.Error()) + return FunctionVersionValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionVersionGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionVersionValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [FunctionVersion]: %s", err.Error()) + return FunctionVersionValue{}, errmsg.EtcdInternalError + } + + var res FunctionVersionValue + err = t.valCodecs[7].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [FunctionVersion]: %s", err.Error()) + return FunctionVersionValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) FunctionVersionGetRange(prefix FunctionVersionKey) ([]FunctionVersionTuple, error) { + p, err := t.keyCodecs[7].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionVersion]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionVersionGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [FunctionVersion]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]FunctionVersionTuple, len(keys)) + for i := range tuples { + var key FunctionVersionKey + if err := t.keyCodecs[7].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [FunctionVersion]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val FunctionVersionValue + if err := t.valCodecs[7].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [FunctionVersion]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = FunctionVersionTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) FunctionVersionPut(key FunctionVersionKey, val FunctionVersionValue) error { + k, err := t.keyCodecs[7].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionVersion]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[7].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [FunctionVersion]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionVersionPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) FunctionVersionDelete(key FunctionVersionKey) error { + k, err := t.keyCodecs[7].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionVersion]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionVersionDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) FunctionVersionDeleteRange(prefix FunctionVersionKey) error { + p, err := t.keyCodecs[7].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionVersion]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionVersionDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) LayerCountIndexGet(ctx context.Context, key LayerCountIndexKey) (LayerCountIndexValue, error) { + k, err := kv.keyCodecs[8].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerCountIndexGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerCountIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexValue{}, errmsg.EtcdInternalError + } + + var res LayerCountIndexValue + err = kv.valCodecs[8].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) LayerCountIndexCount(ctx context.Context, prefix LayerCountIndexKey) (int64, error) { + p, err := kv.keyCodecs[8].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerCountIndex]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerCountIndexCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [LayerCountIndex]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) LayerCountIndexFirstInRange(ctx context.Context, prefix LayerCountIndexKey) (LayerCountIndexKey, LayerCountIndexValue, error) { + p, err := kv.keyCodecs[8].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerCountIndexFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.EtcdInternalError + } + + var key LayerCountIndexKey + var val LayerCountIndexValue + if err := kv.keyCodecs[8].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[8].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) LayerCountIndexLastInRange(ctx context.Context, prefix LayerCountIndexKey) (LayerCountIndexKey, LayerCountIndexValue, error) { + p, err := kv.keyCodecs[8].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerCountIndexLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.EtcdInternalError + } + + var key LayerCountIndexKey + var val LayerCountIndexValue + if err := kv.keyCodecs[8].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[8].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexKey{}, LayerCountIndexValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) LayerCountIndexStream(ctx context.Context, prefix LayerCountIndexKey, by engine.SortBy) *LayerCountIndexPrepareStmt { + return &LayerCountIndexPrepareStmt{ + keyCodecs: kv.keyCodecs[8], + valCodecs: kv.valCodecs[8], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type LayerCountIndexPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix LayerCountIndexKey + by engine.SortBy + filters []LayerCountIndexFilterFunc +} + +type LayerCountIndexFilterFunc func(key LayerCountIndexKey, val LayerCountIndexValue) bool + +func (ps *LayerCountIndexPrepareStmt) Filter(filter LayerCountIndexFilterFunc) *LayerCountIndexPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *LayerCountIndexPrepareStmt) Execute() ([]LayerCountIndexTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key LayerCountIndexKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val LayerCountIndexValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return LayerCountIndexTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("LayerCountIndexStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(LayerCountIndexTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []LayerCountIndexTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(LayerCountIndexTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *LayerCountIndexPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]LayerCountIndexTuple, int, error) { + log.GetLogger().Debugf("LayerCountIndexExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) LayerCountIndexPut(ctx context.Context, key LayerCountIndexKey, val LayerCountIndexValue) error { + k, err := kv.keyCodecs[8].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerCountIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[8].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [LayerCountIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("LayerCountIndexPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [LayerCountIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) LayerCountIndexDelete(ctx context.Context, key LayerCountIndexKey) error { + k, err := kv.keyCodecs[8].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [LayerCountIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("LayerCountIndexDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [LayerCountIndex]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) LayerCountIndexGet(key LayerCountIndexKey) (LayerCountIndexValue, error) { + k, err := t.keyCodecs[8].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerCountIndexGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerCountIndexValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexValue{}, errmsg.EtcdInternalError + } + + var res LayerCountIndexValue + err = t.valCodecs[8].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [LayerCountIndex]: %s", err.Error()) + return LayerCountIndexValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) LayerCountIndexGetRange(prefix LayerCountIndexKey) ([]LayerCountIndexTuple, error) { + p, err := t.keyCodecs[8].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerCountIndexGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]LayerCountIndexTuple, len(keys)) + for i := range tuples { + var key LayerCountIndexKey + if err := t.keyCodecs[8].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val LayerCountIndexValue + if err := t.valCodecs[8].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [LayerCountIndex]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = LayerCountIndexTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) LayerCountIndexPut(key LayerCountIndexKey, val LayerCountIndexValue) error { + k, err := t.keyCodecs[8].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerCountIndex]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[8].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [LayerCountIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerCountIndexPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) LayerCountIndexDelete(key LayerCountIndexKey) error { + k, err := t.keyCodecs[8].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerCountIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerCountIndexDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) LayerCountIndexDeleteRange(prefix LayerCountIndexKey) error { + p, err := t.keyCodecs[8].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [LayerCountIndex]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerCountIndexDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) LayerGet(ctx context.Context, key LayerKey) (LayerValue, error) { + k, err := kv.keyCodecs[9].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Layer]: %s", err.Error()) + return LayerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [Layer]: %s", err.Error()) + return LayerValue{}, errmsg.EtcdInternalError + } + + var res LayerValue + err = kv.valCodecs[9].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [Layer]: %s", err.Error()) + return LayerValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) LayerCount(ctx context.Context, prefix LayerKey) (int64, error) { + p, err := kv.keyCodecs[9].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Layer]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [Layer]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) LayerFirstInRange(ctx context.Context, prefix LayerKey) (LayerKey, LayerValue, error) { + p, err := kv.keyCodecs[9].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerKey{}, LayerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.EtcdInternalError + } + + var key LayerKey + var val LayerValue + if err := kv.keyCodecs[9].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[9].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) LayerLastInRange(ctx context.Context, prefix LayerKey) (LayerKey, LayerValue, error) { + p, err := kv.keyCodecs[9].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("LayerLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerKey{}, LayerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.EtcdInternalError + } + + var key LayerKey + var val LayerValue + if err := kv.keyCodecs[9].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[9].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Layer]: %s", err.Error()) + return LayerKey{}, LayerValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) LayerStream(ctx context.Context, prefix LayerKey, by engine.SortBy) *LayerPrepareStmt { + return &LayerPrepareStmt{ + keyCodecs: kv.keyCodecs[9], + valCodecs: kv.valCodecs[9], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type LayerPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix LayerKey + by engine.SortBy + filters []LayerFilterFunc +} + +type LayerFilterFunc func(key LayerKey, val LayerValue) bool + +func (ps *LayerPrepareStmt) Filter(filter LayerFilterFunc) *LayerPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *LayerPrepareStmt) Execute() ([]LayerTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Layer]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key LayerKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Layer]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val LayerValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Layer]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return LayerTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("LayerStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(LayerTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [Layer]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []LayerTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [Layer]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(LayerTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *LayerPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]LayerTuple, int, error) { + log.GetLogger().Debugf("LayerExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) LayerPut(ctx context.Context, key LayerKey, val LayerValue) error { + k, err := kv.keyCodecs[9].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Layer]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[9].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [Layer]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("LayerPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [Layer]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) LayerDelete(ctx context.Context, key LayerKey) error { + k, err := kv.keyCodecs[9].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Layer]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("LayerDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [Layer]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) LayerGet(key LayerKey) (LayerValue, error) { + k, err := t.keyCodecs[9].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Layer]: %s", err.Error()) + return LayerValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return LayerValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [Layer]: %s", err.Error()) + return LayerValue{}, errmsg.EtcdInternalError + } + + var res LayerValue + err = t.valCodecs[9].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Layer]: %s", err.Error()) + return LayerValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) LayerGetRange(prefix LayerKey) ([]LayerTuple, error) { + p, err := t.keyCodecs[9].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Layer]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [Layer]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]LayerTuple, len(keys)) + for i := range tuples { + var key LayerKey + if err := t.keyCodecs[9].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [Layer]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val LayerValue + if err := t.valCodecs[9].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Layer]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = LayerTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) LayerPut(key LayerKey, val LayerValue) error { + k, err := t.keyCodecs[9].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Layer]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[9].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [Layer]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) LayerDelete(key LayerKey) error { + k, err := t.keyCodecs[9].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Layer]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) LayerDeleteRange(prefix LayerKey) error { + p, err := t.keyCodecs[9].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Layer]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: LayerDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) UncontrolledObjectGet(ctx context.Context, key UncontrolledObjectKey) (UncontrolledObjectValue, error) { + k, err := kv.keyCodecs[10].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("UncontrolledObjectGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return UncontrolledObjectValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectValue{}, errmsg.EtcdInternalError + } + + var res UncontrolledObjectValue + err = kv.valCodecs[10].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) UncontrolledObjectCount(ctx context.Context, prefix UncontrolledObjectKey) (int64, error) { + p, err := kv.keyCodecs[10].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [UncontrolledObject]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("UncontrolledObjectCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [UncontrolledObject]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) UncontrolledObjectFirstInRange(ctx context.Context, prefix UncontrolledObjectKey) (UncontrolledObjectKey, UncontrolledObjectValue, error) { + p, err := kv.keyCodecs[10].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("UncontrolledObjectFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.EtcdInternalError + } + + var key UncontrolledObjectKey + var val UncontrolledObjectValue + if err := kv.keyCodecs[10].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[10].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) UncontrolledObjectLastInRange(ctx context.Context, prefix UncontrolledObjectKey) (UncontrolledObjectKey, UncontrolledObjectValue, error) { + p, err := kv.keyCodecs[10].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("UncontrolledObjectLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.EtcdInternalError + } + + var key UncontrolledObjectKey + var val UncontrolledObjectValue + if err := kv.keyCodecs[10].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[10].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectKey{}, UncontrolledObjectValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) UncontrolledObjectStream(ctx context.Context, prefix UncontrolledObjectKey, by engine.SortBy) *UncontrolledObjectPrepareStmt { + return &UncontrolledObjectPrepareStmt{ + keyCodecs: kv.keyCodecs[10], + valCodecs: kv.valCodecs[10], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type UncontrolledObjectPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix UncontrolledObjectKey + by engine.SortBy + filters []UncontrolledObjectFilterFunc +} + +type UncontrolledObjectFilterFunc func(key UncontrolledObjectKey, val UncontrolledObjectValue) bool + +func (ps *UncontrolledObjectPrepareStmt) Filter(filter UncontrolledObjectFilterFunc) *UncontrolledObjectPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *UncontrolledObjectPrepareStmt) Execute() ([]UncontrolledObjectTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key UncontrolledObjectKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val UncontrolledObjectValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return UncontrolledObjectTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("UncontrolledObjectStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(UncontrolledObjectTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []UncontrolledObjectTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(UncontrolledObjectTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *UncontrolledObjectPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]UncontrolledObjectTuple, int, error) { + log.GetLogger().Debugf("UncontrolledObjectExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) UncontrolledObjectPut(ctx context.Context, key UncontrolledObjectKey, val UncontrolledObjectValue) error { + k, err := kv.keyCodecs[10].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [UncontrolledObject]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[10].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [UncontrolledObject]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("UncontrolledObjectPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [UncontrolledObject]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) UncontrolledObjectDelete(ctx context.Context, key UncontrolledObjectKey) error { + k, err := kv.keyCodecs[10].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [UncontrolledObject]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("UncontrolledObjectDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [UncontrolledObject]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) UncontrolledObjectGet(key UncontrolledObjectKey) (UncontrolledObjectValue, error) { + k, err := t.keyCodecs[10].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: UncontrolledObjectGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return UncontrolledObjectValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectValue{}, errmsg.EtcdInternalError + } + + var res UncontrolledObjectValue + err = t.valCodecs[10].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [UncontrolledObject]: %s", err.Error()) + return UncontrolledObjectValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) UncontrolledObjectGetRange(prefix UncontrolledObjectKey) ([]UncontrolledObjectTuple, error) { + p, err := t.keyCodecs[10].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: UncontrolledObjectGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]UncontrolledObjectTuple, len(keys)) + for i := range tuples { + var key UncontrolledObjectKey + if err := t.keyCodecs[10].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val UncontrolledObjectValue + if err := t.valCodecs[10].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [UncontrolledObject]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = UncontrolledObjectTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) UncontrolledObjectPut(key UncontrolledObjectKey, val UncontrolledObjectValue) error { + k, err := t.keyCodecs[10].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [UncontrolledObject]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[10].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [UncontrolledObject]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: UncontrolledObjectPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) UncontrolledObjectDelete(key UncontrolledObjectKey) error { + k, err := t.keyCodecs[10].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [UncontrolledObject]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: UncontrolledObjectDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) UncontrolledObjectDeleteRange(prefix UncontrolledObjectKey) error { + p, err := t.keyCodecs[10].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [UncontrolledObject]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: UncontrolledObjectDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) FunctionStatusGet(ctx context.Context, key FunctionStatusKey) (FunctionStatusValue, error) { + k, err := kv.keyCodecs[11].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionStatus]: %s", err.Error()) + return FunctionStatusValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionStatusGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionStatusValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [FunctionStatus]: %s", err.Error()) + return FunctionStatusValue{}, errmsg.EtcdInternalError + } + + var res FunctionStatusValue + err = kv.valCodecs[11].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionStatus]: %s", err.Error()) + return FunctionStatusValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) FunctionStatusCount(ctx context.Context, prefix FunctionStatusKey) (int64, error) { + p, err := kv.keyCodecs[11].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionStatus]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionStatusCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [FunctionStatus]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) FunctionStatusFirstInRange(ctx context.Context, prefix FunctionStatusKey) (FunctionStatusKey, FunctionStatusValue, error) { + p, err := kv.keyCodecs[11].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionStatusFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.EtcdInternalError + } + + var key FunctionStatusKey + var val FunctionStatusValue + if err := kv.keyCodecs[11].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[11].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) FunctionStatusLastInRange(ctx context.Context, prefix FunctionStatusKey) (FunctionStatusKey, FunctionStatusValue, error) { + p, err := kv.keyCodecs[11].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionStatusLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.EtcdInternalError + } + + var key FunctionStatusKey + var val FunctionStatusValue + if err := kv.keyCodecs[11].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[11].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionStatus]: %s", err.Error()) + return FunctionStatusKey{}, FunctionStatusValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) FunctionStatusStream(ctx context.Context, prefix FunctionStatusKey, by engine.SortBy) *FunctionStatusPrepareStmt { + return &FunctionStatusPrepareStmt{ + keyCodecs: kv.keyCodecs[11], + valCodecs: kv.valCodecs[11], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type FunctionStatusPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix FunctionStatusKey + by engine.SortBy + filters []FunctionStatusFilterFunc +} + +type FunctionStatusFilterFunc func(key FunctionStatusKey, val FunctionStatusValue) bool + +func (ps *FunctionStatusPrepareStmt) Filter(filter FunctionStatusFilterFunc) *FunctionStatusPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *FunctionStatusPrepareStmt) Execute() ([]FunctionStatusTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionStatus]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key FunctionStatusKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [FunctionStatus]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val FunctionStatusValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [FunctionStatus]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return FunctionStatusTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("FunctionStatusStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(FunctionStatusTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [FunctionStatus]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []FunctionStatusTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [FunctionStatus]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(FunctionStatusTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *FunctionStatusPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]FunctionStatusTuple, int, error) { + log.GetLogger().Debugf("FunctionStatusExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) FunctionStatusPut(ctx context.Context, key FunctionStatusKey, val FunctionStatusValue) error { + k, err := kv.keyCodecs[11].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionStatus]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[11].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [FunctionStatus]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionStatusPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [FunctionStatus]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) FunctionStatusDelete(ctx context.Context, key FunctionStatusKey) error { + k, err := kv.keyCodecs[11].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [FunctionStatus]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("FunctionStatusDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [FunctionStatus]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) FunctionStatusGet(key FunctionStatusKey) (FunctionStatusValue, error) { + k, err := t.keyCodecs[11].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionStatus]: %s", err.Error()) + return FunctionStatusValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionStatusGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return FunctionStatusValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [FunctionStatus]: %s", err.Error()) + return FunctionStatusValue{}, errmsg.EtcdInternalError + } + + var res FunctionStatusValue + err = t.valCodecs[11].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [FunctionStatus]: %s", err.Error()) + return FunctionStatusValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) FunctionStatusGetRange(prefix FunctionStatusKey) ([]FunctionStatusTuple, error) { + p, err := t.keyCodecs[11].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionStatus]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionStatusGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [FunctionStatus]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]FunctionStatusTuple, len(keys)) + for i := range tuples { + var key FunctionStatusKey + if err := t.keyCodecs[11].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [FunctionStatus]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val FunctionStatusValue + if err := t.valCodecs[11].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [FunctionStatus]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = FunctionStatusTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) FunctionStatusPut(key FunctionStatusKey, val FunctionStatusValue) error { + k, err := t.keyCodecs[11].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionStatus]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[11].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [FunctionStatus]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionStatusPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) FunctionStatusDelete(key FunctionStatusKey) error { + k, err := t.keyCodecs[11].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionStatus]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionStatusDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) FunctionStatusDeleteRange(prefix FunctionStatusKey) error { + p, err := t.keyCodecs[11].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [FunctionStatus]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: FunctionStatusDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} + +func (kv *generatedKV) PoolGet(ctx context.Context, key PoolKey) (PoolValue, error) { + k, err := kv.keyCodecs[12].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Pool]: %s", err.Error()) + return PoolValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("PoolGet key: %+v", key) + v, err := kv.eng.Get(ctx, k) + if err != nil { + if err == engine.ErrKeyNotFound { + return PoolValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get from [Pool]: %s", err.Error()) + return PoolValue{}, errmsg.EtcdInternalError + } + + var res PoolValue + err = kv.valCodecs[12].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("failed to decode value of [Pool]: %s", err.Error()) + return PoolValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (kv *generatedKV) PoolCount(ctx context.Context, prefix PoolKey) (int64, error) { + p, err := kv.keyCodecs[12].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Pool]: %s", err.Error()) + return 0, errmsg.MarshalError + } + log.GetLogger().Debugf("PoolCount prefix: %+v", prefix) + n, err := kv.eng.Count(ctx, p) + if err != nil { + log.GetLogger().Errorf("failed to count from [Pool]: %s", err.Error()) + return 0, errmsg.EtcdInternalError + } + return n, nil +} + +func (kv *generatedKV) PoolFirstInRange(ctx context.Context, prefix PoolKey) (PoolKey, PoolValue, error) { + p, err := kv.keyCodecs[12].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("PoolFirstInRange prefix: %+v", prefix) + k, v, err := kv.eng.FirstInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return PoolKey{}, PoolValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get first in range from [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.EtcdInternalError + } + + var key PoolKey + var val PoolValue + if err := kv.keyCodecs[12].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[12].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) PoolLastInRange(ctx context.Context, prefix PoolKey) (PoolKey, PoolValue, error) { + p, err := kv.keyCodecs[12].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("PoolLastInRange prefix: %+v", prefix) + k, v, err := kv.eng.LastInRange(ctx, p) + if err != nil { + if err == engine.ErrKeyNotFound { + return PoolKey{}, PoolValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("failed to get last in range from [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.EtcdInternalError + } + + var key PoolKey + var val PoolValue + if err := kv.keyCodecs[12].Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.UnmarshalError + } + if err := kv.valCodecs[12].Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Pool]: %s", err.Error()) + return PoolKey{}, PoolValue{}, errmsg.UnmarshalError + } + return key, val, nil +} + +func (kv *generatedKV) PoolStream(ctx context.Context, prefix PoolKey, by engine.SortBy) *PoolPrepareStmt { + return &PoolPrepareStmt{ + keyCodecs: kv.keyCodecs[12], + valCodecs: kv.valCodecs[12], + eng: kv.eng, + ctx: ctx, + prefix: prefix, + by: by, + } +} + +type PoolPrepareStmt struct { + keyCodecs codec.Codec + valCodecs codec.Codec + eng engine.Engine + ctx context.Context + prefix PoolKey + by engine.SortBy + filters []PoolFilterFunc +} + +type PoolFilterFunc func(key PoolKey, val PoolValue) bool + +func (ps *PoolPrepareStmt) Filter(filter PoolFilterFunc) *PoolPrepareStmt { + ps.filters = append(ps.filters, filter) + return ps +} + + +func (ps *PoolPrepareStmt) Execute() ([]PoolTuple, error) { + p, err := ps.keyCodecs.Encode(ps.prefix) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Pool]: %s", err.Error()) + return nil, errmsg.MarshalError + } + decode := func(k, v string) (interface{}, error) { + var key PoolKey + if err := ps.keyCodecs.Decode(k, &key); err != nil { + log.GetLogger().Errorf("failed to decode key of [Pool]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + var val PoolValue + if err := ps.valCodecs.Decode(v, &val); err != nil { + log.GetLogger().Errorf("failed to decode value of [Pool]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + return PoolTuple{ + Key: key, + Value: val, + }, nil + } + log.GetLogger().Debugf("PoolStream prefix: %+v, by: %+v", ps.prefix, ps.by) + prepStmt := ps.eng.PrepareStream(ps.ctx, p, decode, ps.by) + for _, filter := range ps.filters { + prepStmt = prepStmt.Filter(func(i interface{}) bool { + tuple := i.(PoolTuple) + return filter(tuple.Key, tuple.Value) + }) + } + stream, err := prepStmt.Execute() + if err != nil { + log.GetLogger().Errorf("failed to execute stream of [Pool]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + + var res []PoolTuple + for { + i, err := stream.Next() + if err != nil { + if err == io.EOF { + break + } + log.GetLogger().Errorf("failed to get next from stream [Pool]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + tuple := i.(PoolTuple) + res = append(res, tuple) + } + if len(res) == 0 { + return nil, errmsg.KeyNotFoundError + } + return res, nil +} + +func (ps *PoolPrepareStmt) ExecuteWithPage(pageIndex, pageSize int) ([]PoolTuple, int, error) { + log.GetLogger().Debugf("PoolExecuteWithPage pageIndex %v, pageSize %v", pageIndex, pageSize) + pageInfo, err := newPageInfo(pageIndex, pageSize) + if err != nil { + return nil, 0, err + } + + res, err := ps.Execute() + if err != nil { + return nil, 0, err + } + + if pageInfo.Begin > len(res) { + return nil, len(res), nil + } + + if pageInfo.Begin+pageInfo.Size > len(res) { + return res[pageInfo.Begin:], len(res), nil + } + + return res[pageInfo.Begin : pageInfo.Begin+pageInfo.Size], len(res), nil +} + +func (kv *generatedKV) PoolPut(ctx context.Context, key PoolKey, val PoolValue) error { + k, err := kv.keyCodecs[12].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Pool]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := kv.valCodecs[12].Encode(val) + if err != nil { + log.GetLogger().Errorf("failed to encode value of [Pool]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("PoolPut key: %+v, val: %+v", key, val) + err = kv.eng.Put(ctx, k, v) + if err != nil { + log.GetLogger().Errorf("failed to put to [Pool]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (kv *generatedKV) PoolDelete(ctx context.Context, key PoolKey) error { + k, err := kv.keyCodecs[12].Encode(key) + if err != nil { + log.GetLogger().Errorf("failed to encode key of [Pool]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("PoolDelete key: %+v", key) + err = kv.eng.Delete(ctx, k) + if err != nil { + log.GetLogger().Errorf("failed to delete from [Pool]: %s", err.Error()) + return errmsg.EtcdInternalError + } + return nil +} + +func (t *generatedTx) PoolGet(key PoolKey) (PoolValue, error) { + k, err := t.keyCodecs[12].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Pool]: %s", err.Error()) + return PoolValue{}, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: PoolGet key: %+v", key) + v, err := t.t.Get(k) + if err != nil { + if err == engine.ErrKeyNotFound { + return PoolValue{}, errmsg.KeyNotFoundError + } + log.GetLogger().Errorf("transaction: failed to get from [Pool]: %s", err.Error()) + return PoolValue{}, errmsg.EtcdInternalError + } + + var res PoolValue + err = t.valCodecs[12].Decode(v, &res) + if err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Pool]: %s", err.Error()) + return PoolValue{}, errmsg.UnmarshalError + } + return res, nil +} + +func (t *generatedTx) PoolGetRange(prefix PoolKey) ([]PoolTuple, error) { + p, err := t.keyCodecs[12].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Pool]: %s", err.Error()) + return nil, errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: PoolGetRange prefix: %+v", prefix) + keys, vals, err := t.t.GetPrefix(p) + if err != nil { + log.GetLogger().Errorf("transaction: failed to get range from [Pool]: %s", err.Error()) + return nil, errmsg.EtcdInternalError + } + if len(keys) == 0 { + return nil, nil + } + + tuples := make([]PoolTuple, len(keys)) + for i := range tuples { + var key PoolKey + if err := t.keyCodecs[12].Decode(keys[i], &key); err != nil { + log.GetLogger().Errorf("transaction: failed to decode key of [Pool]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + var val PoolValue + if err := t.valCodecs[12].Decode(vals[i], &val); err != nil { + log.GetLogger().Errorf("transaction: failed to decode value of [Pool]: %s", err.Error()) + return nil, errmsg.UnmarshalError + } + + tuples[i] = PoolTuple{ + Key: key, + Value: val, + } + } + return tuples, nil +} + +func (t *generatedTx) PoolPut(key PoolKey, val PoolValue) error { + k, err := t.keyCodecs[12].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Pool]: %s", err.Error()) + return errmsg.MarshalError + } + v, err := t.valCodecs[12].Encode(val) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode value of [Pool]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: PoolPut key: %+v, val: %+v", key, val) + t.t.Put(k, v) + return nil +} + +func (t *generatedTx) PoolDelete(key PoolKey) error { + k, err := t.keyCodecs[12].Encode(key) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Pool]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: PoolDelete key: %+v", key) + t.t.Del(k) + return nil +} + +func (t *generatedTx) PoolDeleteRange(prefix PoolKey) error { + p, err := t.keyCodecs[12].Encode(prefix) + if err != nil { + log.GetLogger().Errorf("transaction: failed to encode key of [Pool]: %s", err.Error()) + return errmsg.MarshalError + } + log.GetLogger().Debugf("transaction: PoolDeleteRange prefix: %+v", prefix) + t.t.DelPrefix(p) + return nil +} -- Gitee From f4ac7d471399d3eddb2880eca3a521e8f7cd9c60 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 18 Dec 2025 05:12:28 +0000 Subject: [PATCH 2/2] feat: refactor cli by python Signed-off-by: Yuhang Wei --- functionsystem/apps/pyproject.toml | 76 +++ functionsystem/apps/yr/__init__.py | 0 functionsystem/apps/yr/cli/__init__.py | 0 functionsystem/apps/yr/cli/build_info.py | 2 + functionsystem/apps/yr/cli/component/base.py | 273 ++++++++++ .../apps/yr/cli/component/ds_worker.py | 44 ++ functionsystem/apps/yr/cli/component/etcd.py | 93 ++++ .../apps/yr/cli/component/faas_frontend.py | 54 ++ .../apps/yr/cli/component/function_agent.py | 32 ++ .../apps/yr/cli/component/function_master.py | 32 ++ .../apps/yr/cli/component/function_proxy.py | 32 ++ .../apps/yr/cli/component/registry.py | 16 + functionsystem/apps/yr/cli/config.py | 215 ++++++++ functionsystem/apps/yr/cli/config.toml.jinja | 470 +++++++++++++++++ functionsystem/apps/yr/cli/const.py | 29 ++ functionsystem/apps/yr/cli/main.py | 193 +++++++ functionsystem/apps/yr/cli/requirements.txt | 1 + functionsystem/apps/yr/cli/services.yaml | 24 + functionsystem/apps/yr/cli/system_launcher.py | 472 ++++++++++++++++++ .../apps/yr/cli/system_launcher_test.py | 30 ++ functionsystem/apps/yr/cli/utils.py | 87 ++++ functionsystem/apps/yr/cli/values.toml | 102 ++++ 22 files changed, 2277 insertions(+) create mode 100644 functionsystem/apps/pyproject.toml create mode 100644 functionsystem/apps/yr/__init__.py create mode 100644 functionsystem/apps/yr/cli/__init__.py create mode 100644 functionsystem/apps/yr/cli/build_info.py create mode 100644 functionsystem/apps/yr/cli/component/base.py create mode 100644 functionsystem/apps/yr/cli/component/ds_worker.py create mode 100644 functionsystem/apps/yr/cli/component/etcd.py create mode 100644 functionsystem/apps/yr/cli/component/faas_frontend.py create mode 100644 functionsystem/apps/yr/cli/component/function_agent.py create mode 100644 functionsystem/apps/yr/cli/component/function_master.py create mode 100644 functionsystem/apps/yr/cli/component/function_proxy.py create mode 100644 functionsystem/apps/yr/cli/component/registry.py create mode 100644 functionsystem/apps/yr/cli/config.py create mode 100644 functionsystem/apps/yr/cli/config.toml.jinja create mode 100644 functionsystem/apps/yr/cli/const.py create mode 100644 functionsystem/apps/yr/cli/main.py create mode 100644 functionsystem/apps/yr/cli/requirements.txt create mode 100644 functionsystem/apps/yr/cli/services.yaml create mode 100644 functionsystem/apps/yr/cli/system_launcher.py create mode 100644 functionsystem/apps/yr/cli/system_launcher_test.py create mode 100644 functionsystem/apps/yr/cli/utils.py create mode 100644 functionsystem/apps/yr/cli/values.toml diff --git a/functionsystem/apps/pyproject.toml b/functionsystem/apps/pyproject.toml new file mode 100644 index 0000000..6ba4bc4 --- /dev/null +++ b/functionsystem/apps/pyproject.toml @@ -0,0 +1,76 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools>=61.0", "wheel"] + +[project] +authors = [{ name = "openyuanrong team" }] +description = "Open YuanRong" +license = { text = "Apache-2.0" } +name = "openyuanrong" +readme = "README.md" +requires-python = ">=3.9" +version = "0.0.1" +maintainers = [ + { name = "Yi Liang", email = "liangyi20@huawei.com" }, + { name = "ZhanCheng Luo", email = "luozhancheng@huawei.com" }, +] +keywords = ["openyuanrong", "serverless", "openeuler", "yr"] +classifiers = [ + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.9", +] +dependencies = [ + "tomli_w", + "jinja2", +] + +[project.urls] +Homepage = "https://www.openeuler.org/zh/projects/yuanrong/" +Documentation = "https://pages.openeuler.openatom.cn/openyuanrong/docs/zh-cn/latest/index.html" +Repository = "https://www.openeuler.org/zh/sig/sig-YuanRong#repositories" + +[project.optional-dependencies] +functionsystem = ["openyuanrong-functionsystem>=0.0.1"] +runtime = ["openyuanrong-runtime>=0.0.1"] +datasystem = ["openyuanrong-datasystem>=0.0.1"] +all = [ + "openyuanrong-functionsystem>=0.0.1", + "openyuanrong-runtime>=0.0.1", + "openyuanrong-datasystem>=0.0.1", +] + + +# console_scripts +[project.scripts] +yr = "yr.cli.main:main" + +# [tool.setuptools] +# 等价于 include_package_data=True（PEP 621 下建议用声明式 package-data） +# already default to true +# include-package-data = true + +# 自动发现包，相当于 find_packages() +[tool.setuptools.packages.find] +include = ["yr*"] +namespaces = false +where = ["."] + +# 用声明式替代 MANIFEST.in（可二选一：要么保留 MANIFEST.in，要么写 package-data） +[tool.setuptools.package-data] +yr = ["**/*"] +# python -m wheel tags --platform-tag cp39-cp39-manylinux_2_41_aarch64 ./openyuanrong-0.0.1-py3-none-any.whl + +[tool.ruff] +exclude = [".venv", "*.pyi", "myvenv"] +src = ["yr"] +line-length = 120 + +# [tool.ruff.lint] +# select = ["ALL"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" \ No newline at end of file diff --git a/functionsystem/apps/yr/__init__.py b/functionsystem/apps/yr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/functionsystem/apps/yr/cli/__init__.py b/functionsystem/apps/yr/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/functionsystem/apps/yr/cli/build_info.py b/functionsystem/apps/yr/cli/build_info.py new file mode 100644 index 0000000..7aea6bb --- /dev/null +++ b/functionsystem/apps/yr/cli/build_info.py @@ -0,0 +1,2 @@ +VERSION = "0.0.0" +COMMIT = "unknown" diff --git a/functionsystem/apps/yr/cli/component/base.py b/functionsystem/apps/yr/cli/component/base.py new file mode 100644 index 0000000..8e1e4d8 --- /dev/null +++ b/functionsystem/apps/yr/cli/component/base.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import logging +import os +import socket +import ssl +import subprocess +import time +import urllib.error +import urllib.request +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum + +from yr.cli.config import ConfigResolver +from yr.cli.const import DEFAULT_LOG_DIR + +logger = logging.getLogger(__name__) + + +class ComponentStatus(Enum): + STOPPED = "stopped" + STARTING = "starting" + RUNNING = "running" + HEALTHY = "healthy" + ERROR = "error" + + +@dataclass +class ComponentConfig: + name: str + enabled: bool = True + args: list[str] = field(default_factory=list) + env_vars: dict[str, str] = field(default_factory=dict) + working_dir: str | None = None + depends_on: list[str] = field(default_factory=list) + + # 运行时状态 + status: ComponentStatus = ComponentStatus.STOPPED + process: subprocess.Popen | None = None + pid: int | None = None + start_time: float | None = None + exit_code: int | None = None + + +class ComponentLauncher(ABC): + """组件启动器基类""" + + def __init__(self, name: str, resolver: ConfigResolver, config: ComponentConfig | None = None): + self.name = name + self.resolver = resolver + self.component_config = config or ComponentConfig(name=name) + + @abstractmethod + def prepare_command(self) -> list[str]: + """准备启动命令""" + + @abstractmethod + def health_check(self) -> bool: + """准备启动命令""" + + def prestart_hook(self) -> None: + pass + + def poststart_hook(self) -> None: + pass + + def prepare_environment( + self, + env: dict[str, str] | None = None, + ) -> dict[str, str]: + config_env = self.resolver.rendered_config[self.component_config.name]["env"] + # if config_env has LD_LIBRARY_PATH or LD_PRELOAD, merge them + for k, v in env.items(): + if k in ["LD_LIBRARY_PATH", "LD_PRELOAD"] and k in config_env: + config_env[k] = str(v).lower() if isinstance(v, bool) else str(v) + ":" + config_env[k] + else: + config_env[k] = str(v).lower() if isinstance(v, bool) else str(v) + return config_env + + def wait_until_healthy( + self, + overall_deadline: float | None = None, + ) -> bool: + """ + 阻塞直到当前组件健康或确定失败。 + overall_deadline 用于系统级总超时控制。 + """ + timeout = self.resolver.rendered_config[self.component_config.name]["health_check"].get("timeout", 30) + + start = time.time() + while True: + if self.component_config.status != ComponentStatus.RUNNING: + logger.info(f"{self.name}: process not running") + return False + if overall_deadline is not None and time.time() >= overall_deadline: + logger.info(f"{self.name}: overall health check deadline reached") + return False + + if self.health_check(): + logger.info(f"{self.name}: health check passed") + self.component_config.status = ComponentStatus.HEALTHY + self.poststart_hook() + return True + + if time.time() - start >= timeout: + logger.info(f"{self.name}: health check timeout ({timeout}s)") + return False + + time.sleep(1) + + def _check_port_health(self) -> bool: + """端口健康检查""" + if not self.component_config.health_check_port: + return True + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + result = sock.connect_ex( + ("127.0.0.1", self.component_config.health_check_port), + ) + sock.close() + return result == 0 + except Exception: + return False + + def _check_http_health( + self, + timeout: int = 1, + node_id: str | None = None, + pid: str | None = None, + tls_cacert: str | None = None, + tls_cert: str | None = None, + tls_key: str | None = None, + ) -> bool: + url = self.resolver.rendered_config[self.component_config.name]["health_check"]["endpoint"] + headers = {} + if node_id: + headers["Node-ID"] = node_id + if pid: + headers["PID"] = pid + + request = urllib.request.Request(url, headers=headers) + if url.startswith("https://") and (tls_cacert and tls_cert and tls_key): + ssl_context = ssl.create_default_context(cafile=tls_cacert) + ssl_context.load_cert_chain(certfile=tls_cert, keyfile=tls_key) + else: + ssl_context = None + + try: + response = urllib.request.urlopen(request, timeout=timeout, context=ssl_context) + status_code = response.getcode() + return status_code == 200 + except urllib.error.URLError as e: + if isinstance(e, urllib.error.HTTPError): + logger.error(f"HTTP error during health check: {e.code} - {e.reason}") + else: + logger.error(f"URL error during health check: {str(e)}") + return False + except Exception as e: + logger.error(f"Exception during HTTP health check: {str(e)}") + return False + + def _check_process_health(self) -> bool: + if not self.component_config.process: + logger.error(f"{self.name}: process not started") + self.component_config.status = ComponentStatus.ERROR + return False + if self.component_config.process.poll() is not None: + logger.error(f"{self.name}: process exited with code {self.component_config.process.returncode}") + self.component_config.status = ComponentStatus.STOPPED + return False + return True + + def _check_file_health(self) -> bool: + endpoint = self.resolver.rendered_config[self.component_config.name]["health_check"]["endpoint"] + if os.path.exists(endpoint): + return True + logger.error(f"{self.name}: health check file not found at {endpoint}") + return False + + def _check_custom_health(self) -> bool: + return True + + def _get_args_from_config(self, component: str, args: list[str]) -> list[str]: + config_args = self.resolver.rendered_config[component]["args"] + for k, v in config_args.items(): + if v is None or v == "": + # args.append(f'--{k}=""') + continue + else: + value_str = str(v).lower() if isinstance(v, bool) else str(v) + args.append(f"--{k}={value_str}") + + def _update_args(self, new_config: dict[str, any]) -> None: + args_config = self.resolver.rendered_config[self.name]["args"] + for key, value in new_config.items(): + args_config[key] = value + logger.debug(f"Set {self.name} arg {key} to {value}") + + def launch(self) -> subprocess.Popen: + self.prestart_hook() + + cmd = self.prepare_command() + full_env = os.environ.copy() + env = self.prepare_environment(full_env) + self.component_config.args = cmd + full_env.update(env) + cwd = self.resolver.runtime_context["deploy_path"] + + logger.info(f"Starting {self.name}: {' '.join(cmd)}") + logger.debug(f"Environment: {full_env}") + + log_file = os.path.join(DEFAULT_LOG_DIR, f"{self.name}_stdout.log") + with open(log_file, "a") as log: + log.write(f"\n=== Started at {time.ctime()} ===\n") + log.write(f"Command: {' '.join(cmd)}\n\n") + stdout_file = open(log_file, "a") + stderr_file = open(log_file, "a") + + process = subprocess.Popen( + cmd, + env=full_env, + cwd=cwd, + stdout=stdout_file, + stderr=stderr_file, + ) + + self.component_config.process = process + self.component_config.pid = process.pid + self.component_config.start_time = time.time() + self.component_config.status = ComponentStatus.RUNNING + + return process + + def terminate(self, force: bool = False): + if not self.component_config.process: + return + + process = self.component_config.process + if process.poll() is None: # process is still running + if force: + process.kill() + logger.info(f"Killed {self.name} (PID: {process.pid})") + else: + process.terminate() + logger.info(f"Terminated {self.name} (PID: {process.pid})") + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + logger.info(f"Force killed {self.name} after timeout") + + self.component_config.status = ComponentStatus.STOPPED + self.component_config.process = None + + # exec component in current process for entrypoint use case + # any other specific preparation should be done before call this method + # component start parameters should be set in config(default /etc/yuanrong/config.toml) + # such as mkdir -p -m 750 /home/sn/logs && yr exec function-agent + def exec(self) -> None: + cmd = self.prepare_command() + full_env = os.environ.copy() + env = self.prepare_environment(full_env) + full_env.update(env) + + logger.debug(f"Executing {self.name}: {' '.join(cmd)}") + os.execve( + cmd[0], + cmd, + full_env, + ) diff --git a/functionsystem/apps/yr/cli/component/ds_worker.py b/functionsystem/apps/yr/cli/component/ds_worker.py new file mode 100644 index 0000000..1930514 --- /dev/null +++ b/functionsystem/apps/yr/cli/component/ds_worker.py @@ -0,0 +1,44 @@ +import logging +from pathlib import Path + +from yr.cli.component.base import ComponentLauncher + +logger = logging.getLogger(__name__) + + +class DsWorkerLauncher(ComponentLauncher): + def prepare_command(self) -> list[str]: + executable = self.resolver.rendered_config[self.name]["bin_path"] + args = [executable] + + def get_args(args: list[str], config: dict[str, any]) -> list[str]: + for k, v in config.items(): + if v is None or v == "": + continue + else: + value_str = str(v).lower() if isinstance(v, bool) else str(v) + args.append(f"-{k}={value_str}") + return args + + return get_args(args, self.resolver.rendered_config[self.name]["args"]) + + def health_check(self) -> bool: + logger.info(f"{self.name}: performing custom health check") + return self._check_process_health() and self._check_file_health() + + # create rocksdb, socket and log dir before start + def prestart_hook(self): + logger.info(f"{self.name}: prestart hook executing") + # these should be already set in prepare_config + rocksdb_dir = self.resolver.rendered_config[self.name]["args"]["rocksdb_store_dir"] + socket_dir = self.resolver.rendered_config[self.name]["args"]["unix_domain_socket_dir"] + logs_dir = self.resolver.rendered_config[self.name]["args"]["log_dir"] + dirs = [rocksdb_dir, socket_dir, logs_dir] + if self.resolver.rendered_config[self.name].get("spill", False): + spill_dir = self.resolver.rendered_config[self.name]["args"].get("spill_directory") + if spill_dir: + dirs.append(spill_dir) + for dir_path in dirs: + Path(dir_path).mkdir(parents=True, exist_ok=True) + + logger.debug(f"Created directories for {self.name}: {dirs}") diff --git a/functionsystem/apps/yr/cli/component/etcd.py b/functionsystem/apps/yr/cli/component/etcd.py new file mode 100644 index 0000000..a2324b0 --- /dev/null +++ b/functionsystem/apps/yr/cli/component/etcd.py @@ -0,0 +1,93 @@ +import logging +import subprocess +from pathlib import Path + +from yr.cli.component.base import ComponentLauncher + +logger = logging.getLogger(__name__) + + +class ETCDLauncher(ComponentLauncher): + def prepare_command(self) -> list[str]: + executable = self.resolver.rendered_config[self.component_config.name]["bin_path"] + args = [executable] + self._get_args_from_config(self.name, args) + return args + + def prestart_hook(self): + logger.info(f"{self.name}: prestart hook executing") + data_dir = self.resolver.rendered_config[self.component_config.name]["args"]["data-dir"] + Path(data_dir).mkdir(parents=True, exist_ok=True) + + def health_check(self) -> bool: + etcdctl_path = self.resolver.rendered_config["etcd"]["health_check"]["etcdctl_path"] + endpoint = self.resolver.rendered_config["etcd"]["health_check"]["endpoint"] + if etcdctl_path == "": + logger.error("etcd: etcdctl_path is not set for health check.") + return False + try: + args = [str(etcdctl_path)] + if ( + self.resolver.rendered_config["etcd"]["health_check"]["client_cert_file"] != "" + and self.resolver.rendered_config["etcd"]["health_check"]["client_key_file"] != "" + and self.resolver.rendered_config["etcd"]["args"]["trusted-ca-file"] != "" + ): + args.extend( + [ + "--cacert", + self.resolver.rendered_config["etcd"]["args"]["trusted-ca-file"], + "--cert", + self.resolver.rendered_config["etcd"]["health_check"]["client_cert_file"], + "--key", + self.resolver.rendered_config["etcd"]["health_check"]["client_key_file"], + ] + ) + args.extend(["--endpoints", endpoint, "endpoint", "health"]) + + logger.debug(f"{self.name}: health check command: {' '.join(args)}") + result = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=2, + ) + if result.returncode != 0: + logger.warning( + f"{self.name}: health check failed: {result.stderr.decode().strip()}", + ) + self._check_process_health() + return False + return True + except Exception as e: + logger.error(f"Exception during etcd health check: {e}") + return False + + def poststart_hook(self): + logger.info(f"{self.name}: poststart hook executing") + endpoint = self.resolver.rendered_config["etcd"]["args"]["listen-client-urls"] + if ( + self.resolver.rendered_config["etcd"]["health_check"]["client_cert_file"] == "" + and self.resolver.rendered_config["etcd"]["health_check"]["client_key_file"] == "" + ): + etcdctl_path = self.resolver.rendered_config["etcd"]["health_check"]["etcdctl_path"] + if etcdctl_path == "": + logger.error("etcd: etcdctl_path is not set for disabling auth.") + return + args = [str(etcdctl_path), "--endpoints", endpoint, "auth", "disable"] + logger.debug(f"etcd: disabling etcd auth with command: {' '.join(args)}") + + try: + result = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=5, + ) + if result.returncode != 0: + logger.warning( + f"Failed to disable etcd auth: {result.stderr.decode().strip()}", + ) + else: + logger.info("etcd: auth disabled successfully.") + except Exception as e: + logger.error(f"etcd: exception while disabling etcd auth: {e}") diff --git a/functionsystem/apps/yr/cli/component/faas_frontend.py b/functionsystem/apps/yr/cli/component/faas_frontend.py new file mode 100644 index 0000000..3969c99 --- /dev/null +++ b/functionsystem/apps/yr/cli/component/faas_frontend.py @@ -0,0 +1,54 @@ +import logging + +from yr.cli.component.base import ComponentLauncher + +logger = logging.getLogger(__name__) + + +class FaaSFrontendLauncher(ComponentLauncher): + def prepare_command(self) -> list[str]: + executable = self.resolver.rendered_config[self.name]["bin_path"] + args = [executable] + + def get_args(args: list[str], config: dict[str, any]) -> list[str]: + for k, v in config.items(): + if v is None or v == "": + continue + else: + value_str = str(v).lower() if isinstance(v, bool) else str(v) + args.append(f"-{k}={value_str}") + return args + + return get_args(args, self.resolver.rendered_config[self.name]["args"]) + + def health_check(self) -> bool: + logger.info(f"{self.name}: performing custom health check") + return self._check_process_health() + + def prestart_hook(self): + logger.info(f"{self.name}: prestart hook executing") + # modify init_frontend_args.json and copy it to path + + # def patch_init_frontend_args(self, path) -> None: + # path = Path(path).resolve() + # text = path.read_text() + + # config = self.resolver.rendered_config + # faas_args = config[self.component_config.name]["args"] + # faas_values = config["values"]["faas_frontend"] + # # attention: etcd address getting from function-proxy component + # etcd_addr = config["function-proxy"]["args"]["etcd-address"] + # ip_address = faa[] + + # # Apply the same placeholder replacements as the shell script + # text = text.replace("{etcdAddr}", etcd_addr_replacement) + # text = text.replace("{faas_frontend_http_ip}", os.environ.get("IP_ADDRESS", "")) + # text = text.replace("{faas_frontend_http_port}", os.environ.get("FAAS_FRONTEND_HTTP_PORT", "")) + # text = text.replace("{sslEnable}", os.environ.get("SSL_ENABLE", "")) + # text = text.replace("{sccEnable}", os.environ.get("SCC_ENABLE", "")) + # text = text.replace("{etcdAuthType}", os.environ.get("ETCD_AUTH_TYPE", "")) + # text = text.replace("{azPrefix}", os.environ.get("ETCD_TABLE_PREFIX", "")) + # text = text.replace("{sslBasePath}", os.environ.get("SSL_BASE_PATH", "")) + # text = text.replace("{sccBasePath}", os.environ.get("SCC_BASE_PATH", "")) + + # path.write_text(text) diff --git a/functionsystem/apps/yr/cli/component/function_agent.py b/functionsystem/apps/yr/cli/component/function_agent.py new file mode 100644 index 0000000..19739ca --- /dev/null +++ b/functionsystem/apps/yr/cli/component/function_agent.py @@ -0,0 +1,32 @@ +import json +import logging +from pathlib import Path + +from yr.cli.component.base import ComponentLauncher + +logger = logging.getLogger(__name__) + + +class FunctionAgentLauncher(ComponentLauncher): + def prepare_command(self) -> list[str]: + executable = self.resolver.rendered_config[self.name]["bin_path"] + args = [executable] + self._get_args_from_config(self.name, args) + return args + + def health_check(self) -> bool: + logger.info(f"{self.name}: performing custom health check") + node_id = self.resolver.rendered_config[self.name]["args"][ + "node_id" + ] # this should be already set in prepare_config + pid = self.component_config.pid + logger.debug( + f"{self.name}: health check endpoint: {self.resolver.rendered_config[self.component_config.name]['health_check']['endpoint']}, node_id: {node_id}, pid: {pid}" + ) + return self._check_process_health() and self._check_http_health(node_id=node_id, pid=str(pid)) + + def prestart_hook(self): + logger.info(f"{self.name}: prestart hook executing") + log_config = self.resolver.rendered_config[self.component_config.name]["args"]["log_config"] + fs_log_path = json.loads(log_config).get("filepath", "/tmp/yr/sessions_latest/logs/function_system") + Path(fs_log_path).mkdir(parents=True, exist_ok=True) diff --git a/functionsystem/apps/yr/cli/component/function_master.py b/functionsystem/apps/yr/cli/component/function_master.py new file mode 100644 index 0000000..0af68d2 --- /dev/null +++ b/functionsystem/apps/yr/cli/component/function_master.py @@ -0,0 +1,32 @@ +import json +import logging +from pathlib import Path + +from yr.cli.component.base import ComponentLauncher + +logger = logging.getLogger(__name__) + + +class FunctionMasterLauncher(ComponentLauncher): + def prepare_command(self) -> list[str]: + executable = self.resolver.rendered_config[self.name]["bin_path"] + args = [executable] + self._get_args_from_config(self.name, args) + return args + + def health_check(self) -> bool: + logger.info(f"{self.name}: performing custom health check") + node_id = self.resolver.rendered_config[self.name]["args"][ + "node_id" + ] # this should be already set in prepare_config + pid = self.component_config.pid + logger.debug( + f"{self.name}: health check endpoint: {self.resolver.rendered_config[self.component_config.name]['health_check']['endpoint']}, node_id: {node_id}, pid: {pid}" + ) + return self._check_process_health() and self._check_http_health(node_id=node_id, pid=str(pid)) + + def prestart_hook(self): + logger.info(f"{self.name}: prestart hook executing") + log_config = self.resolver.rendered_config[self.component_config.name]["args"]["log_config"] + fs_log_path = json.loads(log_config).get("filepath", "/tmp/yr/sessions_latest/logs/function_system") + Path(fs_log_path).mkdir(parents=True, exist_ok=True) diff --git a/functionsystem/apps/yr/cli/component/function_proxy.py b/functionsystem/apps/yr/cli/component/function_proxy.py new file mode 100644 index 0000000..c803a4f --- /dev/null +++ b/functionsystem/apps/yr/cli/component/function_proxy.py @@ -0,0 +1,32 @@ +import json +import logging +from pathlib import Path + +from yr.cli.component.base import ComponentLauncher + +logger = logging.getLogger(__name__) + + +class FunctionProxyLauncher(ComponentLauncher): + def prepare_command(self) -> list[str]: + executable = self.resolver.rendered_config[self.name]["bin_path"] + args = [executable] + self._get_args_from_config(self.name, args) + return args + + def health_check(self) -> bool: + logger.info(f"{self.name}: performing custom health check") + node_id = self.resolver.rendered_config[self.name]["args"][ + "node_id" + ] # this should be already set in prepare_config + pid = self.component_config.pid + logger.debug( + f"{self.name}: health check endpoint: {self.resolver.rendered_config[self.component_config.name]['health_check']['endpoint']}, node_id: {node_id}, pid: {pid}" + ) + return self._check_process_health() and self._check_http_health(node_id=node_id, pid=str(pid)) + + def prestart_hook(self): + logger.info(f"{self.name}: prestart hook executing") + log_config = self.resolver.rendered_config[self.component_config.name]["args"]["log_config"] + fs_log_path = json.loads(log_config).get("filepath", "/tmp/yr/sessions_latest/logs/function_system") + Path(fs_log_path).mkdir(parents=True, exist_ok=True) diff --git a/functionsystem/apps/yr/cli/component/registry.py b/functionsystem/apps/yr/cli/component/registry.py new file mode 100644 index 0000000..69ac9b1 --- /dev/null +++ b/functionsystem/apps/yr/cli/component/registry.py @@ -0,0 +1,16 @@ +from yr.cli.component.ds_worker import DsWorkerLauncher +from yr.cli.component.etcd import ETCDLauncher +from yr.cli.component.faas_frontend import FaaSFrontendLauncher +from yr.cli.component.function_agent import FunctionAgentLauncher +from yr.cli.component.function_master import FunctionMasterLauncher +from yr.cli.component.function_proxy import FunctionProxyLauncher + +LAUNCHER_CLASSES = { + "etcd": ETCDLauncher, + "ds-master": DsWorkerLauncher, + "ds-worker": DsWorkerLauncher, + "function-master": FunctionMasterLauncher, + "function-proxy": FunctionProxyLauncher, + "function-agent": FunctionAgentLauncher, + "faas-frontend": FaaSFrontendLauncher, +} diff --git a/functionsystem/apps/yr/cli/config.py b/functionsystem/apps/yr/cli/config.py new file mode 100644 index 0000000..1a9464f --- /dev/null +++ b/functionsystem/apps/yr/cli/config.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import logging +import os +import socket +import time +from argparse import Namespace +from pathlib import Path +from typing import Any, Dict, List + +import tomli_w +from jinja2 import Environment, StrictUndefined + +try: + import tomllib +except ImportError: + from pip._vendor import tomli as tomllib + +from yr.cli.const import DEFAULT_CONFIG_TEMPLATE_PATH, DEFAULT_VALUES_TOML, StartMode +from yr.cli.utils import get_ip, get_total_memory_mb, port_or_free, trim_hostname + +logger = logging.getLogger(__name__) + + +class ConfigResolver: + def __init__( + self, config_path: Path, cli_dir: Path, mode: StartMode | None = None, args: Namespace | None = None + ) -> None: + self.mode = mode + self.yr_package_path = cli_dir.parent + self.jinja_env = Environment( + undefined=StrictUndefined, + autoescape=False, + trim_blocks=True, + lstrip_blocks=True, + keep_trailing_newline=True, + newline_sequence="\n", + ) + self.runtime_context = self._build_runtime_context(cli_dir) + self.rendered_config = self._load_config(config_path, args) + + def _load_config(self, config_path: Path, args: Namespace | None = None) -> dict[str, any]: + # Load default templates + values_path = self.yr_package_path / DEFAULT_VALUES_TOML + template_path = self.yr_package_path / DEFAULT_CONFIG_TEMPLATE_PATH + default_values_str = values_path.read_text() + template_str = template_path.read_text() + default_values = tomllib.loads(default_values_str) + logger.debug(f"Default values: {default_values}") + + # Load user config if exists + user_config = tomllib.loads(config_path.read_text()) if config_path.exists() else {} + logger.debug(f"User config: {user_config}" if user_config else "No user config found") + + # Merge default and user values + merged_values = merge_config(default_values.get("values", {}), user_config.get("values", {})) + logger.debug(f"Merged values: {merged_values}") + + # First pass: render root scalar values using runtime context + rendered_root = { + k: self.jinja_env.from_string(v).render() if isinstance(v, str) else v + for k, v in merged_values.items() + if not isinstance(v, dict) + } + logger.debug(f"Rendered root values: {rendered_root}") + + # Second pass: render full values including nested using rendered root + full_values = {**merged_values, **rendered_root} + values_template = tomli_w.dumps({"values": full_values}) + final_values = tomllib.loads(self.jinja_env.from_string(values_template).render(values=full_values)) + logger.debug(f"Final rendered values: {final_values}") + + # Render config template and merge remaining user config + rendered_config = tomllib.loads(self.jinja_env.from_string(template_str).render(**final_values)) + user_config.pop("values", None) + merged_config = merge_config(rendered_config, user_config) if user_config else rendered_config + return {"values": final_values.get("values", {}), **merged_config} + + def _print_default_config(self, values_path: Path, template_path: Path) -> dict[str, any]: + with open(values_path, "r") as f: + values_str = f.read() + with open(template_path, "r") as f: + template_str = f.read() + print("------------------------------ START --------------------------------") + print("---------------------------- values.toml ----------------------------") + print(values_str) + print("---------------------------- config.toml ----------------------------") + print(template_str) + print("------------------------------- END ---------------------------------") + + def _build_runtime_context(self, cli_dir: Path) -> dict[str, any]: + """build runtime context, collect environment information""" + hostname = socket.gethostname() + pid = os.getpid() + node_id = f"{trim_hostname()}-{pid}" + cpu_millicores = os.cpu_count() * 1000 + memory_num_mb = get_total_memory_mb() + ip = get_ip() + timestamp = time.time() + time_str = time.strftime("%Y%m%d_%H%M%S", time.localtime()) + context = { + "yr_package_path": self.yr_package_path, + "hostname": hostname, + "pid": pid, + "node_id": node_id, + "cpu_millicores": cpu_millicores, + "memory_num_mb": memory_num_mb, + "ip": ip, + "timestamp": timestamp, + "time": time_str, + "deploy_path": f"/tmp/yr/session_{time_str}", + } + self.jinja_env.globals.update( + { + "yr_package_path": self.yr_package_path, + "hostname": hostname, + "pid": pid, + "node_id": node_id, + "cpu_millicores": cpu_millicores, + "memory_num_mb": memory_num_mb, + "ip": ip, + "timestamp": timestamp, + "time": time_str, + "deploy_path": f"/tmp/yr/session_{time_str}", + } + ) + self.jinja_env.filters["check_port"] = port_or_free + return context + + +def type_convert(value: str) -> Any: + value = value.strip() + if value.lower() in ("true", "false"): + return value.lower() == "true" + if value.isdigit() or (value.startswith("-") and value[1:].isdigit()): + return int(value) + + try: + return float(value) + except ValueError: + pass + + # 列表（逗号分隔） + if "," in value and value.startswith("[") and value.endswith("]"): + parts = [type_convert(part.strip()) for part in value[1:-1].split(",")] + return parts + + return value + + +def merge_dot_notation( + dot_notation_list: List[str], target_dict: Dict[str, Any], convert_types: bool = True +) -> Dict[str, Any]: + """ + 点分隔键值对合并 + + Args: + dot_notation_list: 格式为 ["key.path=value"] 的列表 + target_dict: 目标字典 + convert_types: 是否自动转换类型 + + Returns: + 合并后的字典 + """ + for item in dot_notation_list: + if "=" not in item: + logger.warning(f"skip wrong format argument: {item}") + continue + + key_path, value = item.split("=", 1) + keys = key_path.split(".") + processed_value = type_convert(value) if convert_types else value + + current = target_dict + for key in keys[:-1]: + if key not in current: + current[key] = {} + current = current[key] + + final_key = keys[-1] + current[final_key] = processed_value + + return target_dict + + +def merge_config( + default_config: dict[str, any], + user_config: dict[str, any], + nested_key_path: str = "", +) -> dict[str, any]: + """Recursively merge user configuration into default configuration. + + This function merges a user-provided configuration dictionary into a default + configuration dictionary. It validates that all keys in the user config exist + in the default config and maintains the expected structure. + """ + for key, value in user_config.items(): + if key not in default_config: + raise ValueError( + f"Invalid config key: {nested_key_path + ('.' if nested_key_path else '')}{key}", + ) + if isinstance(value, dict): + if not isinstance(default_config[key], dict): + raise ValueError( + f"Invalid config key: {nested_key_path + ('.' if nested_key_path else '')}{key}, expected dict", + ) + default_config[key] = merge_config( + default_config[key], + value, + nested_key_path + ("." if nested_key_path else "") + key, + ) + else: + default_config[key] = value + + return default_config diff --git a/functionsystem/apps/yr/cli/config.toml.jinja b/functionsystem/apps/yr/cli/config.toml.jinja new file mode 100644 index 0000000..b449ed4 --- /dev/null +++ b/functionsystem/apps/yr/cli/config.toml.jinja @@ -0,0 +1,470 @@ +version = "1" + +[mode] + +[mode.master] +etcd = true +ds-master = true +ds-worker = true +function-master = true +function-proxy = true +function-agent = true + +[mode.agent] +ds-worker = true +function-proxy = true +function-agent = true + +{% set etcd_local = (values.etcd.address | selectattr('ip', 'equalto', values.host_ip) | list)[0] %} + +{% if values.etcd.enable_multi_master %} +{% set etcd_cluster %} +{% for node in values.etcd.address -%} +{{ node.ip }}:{{ node.port }}{% if not loop.last %},{% endif %} +{%- endfor %} +{% endset %} +{% else %} +{% set etcd_cluster = etcd_local.ip ~ ':' ~ etcd_local.port %} +{% endif %} + + +[etcd] +bin_path = "{{ values.etcd.bin_path }}" + +[etcd.health_check] +enable = true +etcdctl_path = "{{ values.yr_package_path }}/third_party/etcd/etcdctl" +timeout = 20 +endpoint = "http://{{ etcd_local.ip }}:{{ etcd_local.port }}" +{% if values.etcd.auth_type == "TLS" %} +client_cert_file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.client_cert_file }}" +client_key_file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.client_key_file }}" +{% endif %} + +[etcd.env] +LD_LIBRARY_PATH = "" + +[etcd.args] +name = "etcd0" +unsafe-no-fsync = true +data-dir="{{ values.deploy_path }}/third_party/etcd" +auto-compaction-retention="100000" +auto-compaction-mode="revision" +quota-backend-bytes="8589934592" +listen-client-urls = "http://{{ etcd_local.ip }}:{{ etcd_local.port }}" +advertise-client-urls = "http://{{ etcd_local.ip }}:{{ etcd_local.port }}" +listen-peer-urls = "http://{{ etcd_local.ip }}:{{ etcd_local.peer_port }}" +{% if values.etcd.enable_multi_master or values.etcd.auth_type == "TLS" %} +initial-advertise-peer-urls = "http://{{ etcd_local.ip }}:{{ etcd_local.peer_port }}" +{% endif %} +{% if values.etcd.auth_type == "TLS" %} +client-cert-auth = true +cipher-suites = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384" +trusted-ca-file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.ca_file }}" +cert-file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.cert_file }}" +key-file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.key_file }}" +peer-client-cert-auth = true +peer-trusted-ca-file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.ca_file }}" +peer-cert-file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.cert_file }}" +peer-key-file = "{{ values.etcd.auth.ssl_base_path}}/{{ values.etcd.auth.key_file }}" +{% endif %} +{% if values.etcd.enable_multi_master %} +initial-cluster = "{% for node in values.etcd.address -%} +etcd{{ loop.index0 }}=http://{{ node.ip }}:{{ node.peer_port }}{% if not loop.last %},{% endif %} +{%- endfor %}" +initial-cluster-state = "new" +initial-cluster-token = "etcd-cluster-1" +{% endif -%} + +[ds-master] +bin_path = "{{ values.ds_master.bin_path }}" + +[ds-master.health_check] +enable = true +timeout = 65 +endpoint = "{{ values.deploy_path }}/ds_master_{{ values.pid }}/master/health" + +[ds-master.env] +LD_LIBRARY_PATH = "{{ values.yr_package_path }}/datasystem/service/lib" + +[ds-master.args] +master_address = "{{ values.ds_master.ip }}:{{ values.ds_master.port }}" +log_dir = "{{ values.deploy_path }}/logs/data_system/master" +worker_address = "{{ values.ds_master.ip }}:{{ values.ds_master.port }}" +unix_domain_socket_dir = "{{ values.deploy_path }}/ds_master_{{ values.pid }}/socket" +v = "0" +minloglevel = "0" +health_check_path = "{{ values.deploy_path }}/ds_master_{{ values.pid }}/master/health" +ready_check_path = "{{ values.deploy_path }}/ds_master_{{ values.pid }}/master/ready" +enable_reconciliation = false +rpc_thread_num = 32 +heartbeat_interval_ms = 3000 # TODO: recheck +enable_multi_stubs = true +max_log_size = 40 +max_log_file_num = 32 +node_dead_timeout_s = 30 # TODO: recheck +node_timeout_s = 10 # TODO: recheck +rocksdb_store_dir = "{{ values.deploy_path }}/ds_master_{{ values.pid }}/rocksdb" +etcd_address = "{{ etcd_cluster }}" +cluster_name = "" +etcd_target_name_override = "" +enable_etcd_auth = false +arena_per_tenant = 16 +enable_fallocate = true +enable_huge_tlb = false +enable_thp = false +etcd_ca = "" +etcd_cert = "" +etcd_key = "" +enable_curve_zmq = false +curve_key_dir = "" +logfile_mode = 416 +l2_cache_type = "none" +sfs_path = "" +log_monitor = false +zmq_chunk_sz = 1048576 +enable_lossless_data_exit_mode = false +enable_distributed_master = false +stderrthreshold = 3 + +[ds-worker] +bin_path = "{{ values.ds_worker.bin_path }}" + +[ds-worker.health_check] +enable = true +timeout = 65 +endpoint = "{{ values.deploy_path }}/ds_worker_{{ values.pid }}/worker/health" + +[ds-worker.env] +LD_LIBRARY_PATH = "{{ values.yr_package_path }}/datasystem/service/lib" + +[ds-worker.args] +master_address = "{{ values.ds_master.ip }}:{{ values.ds_master.port }}" +log_dir = "{{ values.deploy_path }}/logs/data_system/worker" +worker_address = "{{ values.ds_worker.ip }}:{{ values.ds_worker.port }}" +unix_domain_socket_dir = "{{ values.deploy_path }}/ds_worker_{{ values.pid }}/socket" +v = "0" +minloglevel = "0" +health_check_path = "{{ values.deploy_path }}/ds_worker_{{ values.pid }}/worker/health" +ready_check_path = "{{ values.deploy_path }}/ds_worker_{{ values.pid }}/worker/ready" +enable_reconciliation = false +rpc_thread_num = 32 +heartbeat_interval_ms = 300000 +enable_multi_stubs = true +max_log_size = 40 +max_log_file_num = 32 +node_dead_timeout_s = 86400 +node_timeout_s = 1800 +rocksdb_store_dir = "{{ values.deploy_path }}/ds_worker_{{ values.pid }}/rocksdb" +etcd_address = "{{ etcd_cluster }}" +cluster_name = "" +etcd_target_name_override = "" +enable_etcd_auth = false +arena_per_tenant = 16 +enable_fallocate = true +enable_huge_tlb = false +enable_thp = false +etcd_ca = "" +etcd_cert = "" +etcd_key = "" +enable_curve_zmq = false +curve_key_dir = "" +logfile_mode = 416 +l2_cache_type = "none" +sfs_path = "" +log_monitor = false +zmq_chunk_sz = 1048576 +enable_lossless_data_exit_mode = false +enable_distributed_master = false +stderrthreshold = 3 +shared_memory_size_mb = {{ values.shared_memory_num }} +client_dead_timeout_s = 86400 +max_client_num = 1000 +memory_reclamation_time_second = 5 +#spill +spill_directory = "" +spill_size_limit = "{% if values.ds_worker.spill %}21474836480{% endif %}" # copy from data_system/deploy/install.sh + +[function-master] +bin_path = "{{ values.function_master.bin_path }}" + +[function-master.health_check] +enable = true +timeout = 60 +endpoint = "http://{{ values.function_master.ip }}:{{ values.function_master.global_scheduler_port }}/global-scheduler/healthy" + +[function-master.env] +LD_LIBRARY_PATH = "{{ values.yr_package_path }}/functionsystem/lib" +LD_PRELOAD = "" # for jemalloc use + +[function-master.args] +ip = "{{ values.function_master.ip }}:{{ values.function_master.global_scheduler_port }}" +{% if values.meta_store.enable and values.meta_store.mode != "passthrough" %} +meta_store_address = "{{ values.function_master.ip }}:{{ values.function_master.global_scheduler_port }}" +{% else %} +meta_store_address = "{{ etcd_cluster }}" +{% endif %} +log_config = '''{"filepath":"{{ values.fs.log.path }}","level":"{{ values.fs.log.level }}","compress":{{ values.fs.log.compress_enable|lower() }},"rolling":{"maxsize":{{ values.fs.log.rolling_max_size }},"maxfiles":{{ values.fs.log.rolling_max_files }},"retentionDays":{{ values.fs.log.rolling_retention_days }}},"async":{"logBufSecs":{{ values.fs.log.async_log_buf_secs }},"maxQueueSize":{{ values.fs.log.async_log_max_queue_size }},"threadCount":{{ values.fs.log.async_log_thread_count }}},"alsologtostderr":{{ values.fs.log.also_log_to_stderr|lower() }}}''' +etcd_address = "{{ etcd_cluster }}" +node_id = "{{ values.node_id }}" +sys_func_retry_period = 5000 +runtime_recover_enable = false +litebus_thread_num = 20 +system_timeout = 1800000 +enable_metrics = true +metrics_config = "{{ values.metrics.metrics_config }}" +metrics_config_file = "{{ values.metrics.metrics_config_file }}" +pull_resource_interval = 1000 +is_schedule_tolerate_abnormal = false +enable_print_resource_view = false +schedule_plugins = '''{{ values.fs.schedule_plugins }}''' +schedule_relaxed = -1 +max_priority = 0 +enable_preemption = false +enable_meta_store = false +enable_persistence = false +meta_store_mode = "local" +meta_store_excluded_keys = "," +election_mode = "" +services_path = "{{ values.yr_package_path }}/cli/services.yaml" +lib_path = "{{ values.yr_package_path }}/functionsystem/lib" +etcd_auth_type = "{{ values.etcd.auth_type }}" +etcd_root_ca_file = "{{ values.etcd.auth.ca_file }}" +etcd_cert_file = "{{ values.etcd.auth.client_cert_file }}" +etcd_key_file = "{{ values.etcd.auth.client_key_file }}" +etcd_ssl_base_path = "{{ values.etcd.auth.ssl_base_path }}" +etcd_table_prefix = "" +etcd_target_name_override = "" +ssl_enable = false +ssl_base_path = "" +ssl_root_file = "" +ssl_cert_file = "" +ssl_key_file = "" +meta_store_max_flush_concurrency = 100 +meta_store_max_flush_batch_size = 50 + +[function-proxy] +bin_path = "{{ values.function_proxy.bin_path }}" + +[function-proxy.health_check] +enable = true +timeout = 60 +endpoint = "http://{{ values.function_proxy.ip }}:{{ values.function_proxy.port }}/local-scheduler/healthy" + +[function-proxy.env] +LD_LIBRARY_PATH = "{{ values.yr_package_path }}/functionsystem/lib" +LD_PRELOAD = "" # for jemalloc use + +[function-proxy.args] +ip = "{{ values.function_proxy.ip }}" +{% if values.meta_store.enable and values.meta_store.mode != "passthrough" %} +meta_store_address = "{{ values.function_master.ip }}:{{ values.function_master.global_scheduler_port }}" +{% else %} +meta_store_address = "{{ etcd_cluster }}" +{% endif %} +log_config = '''{"filepath":"{{ values.fs.log.path }}","level":"{{ values.fs.log.level }}","compress":{{ values.fs.log.compress_enable|lower() }},"rolling":{"maxsize":{{ values.fs.log.rolling_max_size }},"maxfiles":{{ values.fs.log.rolling_max_files }},"retentionDays":{{ values.fs.log.rolling_retention_days }}},"async":{"logBufSecs":{{ values.fs.log.async_log_buf_secs }},"maxQueueSize":{{ values.fs.log.async_log_max_queue_size }},"threadCount":{{ values.fs.log.async_log_thread_count }}},"alsologtostderr":{{ values.fs.log.also_log_to_stderr|lower() }}}''' +etcd_address = "{{ etcd_cluster }}" +node_id = "{{ values.node_id }}" +services_path = "{{ values.yr_package_path }}/cli/services.yaml" +lib_path = "{{ values.yr_package_path }}/functionsystem/lib" +enable_metrics = true +metrics_config = "" +metrics_config_file = "{{ values.metrics.metrics_config_file }}" +litebus_thread_num = 20 +runtime_recover_enable = false +system_timeout = 1800000 +enable_print_resource_view = false +schedule_plugins = '''{{ values.fs.schedule_plugins }}''' +schedule_relaxed = -1 +max_priority = 0 +enable_preemption = false +enable_meta_store = false +meta_store_mode = "local" +meta_store_excluded_keys = "," +election_mode = "" +ssl_enable = false +ssl_base_path = "" +etcd_auth_type = "{{ values.etcd.auth_type }}" +etcd_root_ca_file = "{{ values.etcd.auth.ca_file }}" +etcd_cert_file = "{{ values.etcd.auth.client_cert_file }}" +etcd_key_file = "{{ values.etcd.auth.client_key_file }}" +etcd_ssl_base_path = "{{ values.etcd.auth.ssl_base_path }}" +etcd_table_prefix = "" +etcd_target_name_override = "" +ssl_root_file = "" +ssl_cert_file = "" +ssl_key_file = "" +address = "{{ values.function_proxy.ip }}:{{ values.function_proxy.port }}" +grpc_listen_port = {{ values.function_proxy.grpc_listen_port }} +enable_driver = false +enable_trace = false +runtime_heartbeat_enable = true +runtime_max_heartbeat_timeout_times = 18 +runtime_heartbeat_timeout_ms = 100000 +state_storage_type = "disable" +runtime_init_call_timeout_seconds = 600 +global_scheduler_address = "{{ values.function_master.ip }}:{{ values.function_master.global_scheduler_port }}" +update_resource_cycle = 1000 +runtime_conn_timeout_s = 30 +pseudo_data_plane = false # TODO: if cpu_num <= 100, set to True +cache_storage_host = "{{ values.ds_worker.ip }}" +cache_storage_port = {{ values.ds_worker.port }} +enable_server_mode = true +max_instance_cpu_size = 16000 +max_instance_memory_size = 1073741824 +min_instance_cpu_size = 0 +min_instance_memory_size = 0 +unregister_while_stop = true +runtime_ds_auth_enable = false +runtime_ds_encrypt_enable = false +curve_key_path = "" +cache_storage_auth_enable = false +cache_storage_auth_type = "Noauth" +ssl_downgrade_enable = true +enable_print_perf = false +is_partial_watch_instances = false +runtime_instance_debug_enable = false + +[function-agent] +bin_path = "{{ values.function_agent.bin_path }}" + +[function-agent.env] +LD_LIBRARY_PATH = "{{ values.yr_package_path }}/functionsystem/lib" +RUNTIME_METRICS_CONFIG = "false" +INIT_LABELS = "" + +[function-agent.health_check] +enable = true +timeout = 60 +endpoint = "http://{{ values.function_agent.ip }}:{{ values.function_agent.port }}/function-agent/healthy" + +[function-agent.args] +enable_merge_process = true +ip = "{{ values.function_agent.ip }}" +node_id = "{{ values.node_id }}" +agent_uid = "{{ values.node_id }}" +alias = "" +log_config = '''{"filepath":"{{ values.fs.log.path }}","level":"{{ values.fs.log.level }}","compress":{{ values.fs.log.compress_enable|lower() }},"rolling":{"maxsize":{{ values.fs.log.rolling_max_size }},"maxfiles":{{ values.fs.log.rolling_max_files }},"retentionDays":{{ values.fs.log.rolling_retention_days }}},"async":{"logBufSecs":{{ values.fs.log.async_log_buf_secs }},"maxQueueSize":{{ values.fs.log.async_log_max_queue_size }},"threadCount":{{ values.fs.log.async_log_thread_count }}},"alsologtostderr":{{ values.fs.log.also_log_to_stderr|lower() }}}''' +litebus_thread_num = 20 +local_scheduler_address = "{{ values.function_proxy.ip }}:{{ values.function_proxy.port }}" +agent_listen_port = {{ values.function_agent.port }} +runtime_dir = "{{ values.yr_package_path }}/runtime/service" +runtime_home_dir = "" +runtime_logs_dir = "{{ values.deploy_path }}/logs" +runtime_std_log_dir = "" +runtime_ld_library_path = "{{ values.yr_package_path }}/runtime/service/cpp/snlib:{{ values.yr_package_path }}/runtime/sdk/cpp/lib" +runtime_log_level = "INFO" +runtime_max_log_size = 40 +runtime_max_log_file_num = 20 +runtime_config_dir = "{{ values.yr_package_path }}/runtime/service/cpp/config" +enable_separated_redirect_runtime_std = false +user_log_export_mode = "file" +npu_collection_mode = "all" +gpu_collection_enable = false +proxy_grpc_server_port = "{{ values.function_proxy.grpc_listen_port }}" +setCmdCred = false +python_dependency_path = "{{ values.yr_package_path }}/runtime/service/python" +python_log_config_path = "{{ values.yr_package_path }}/runtime/service/python/config/python-runtime-log.json" +java_system_property = "{{ values.yr_package_path }}/runtime/service/java/log4j2.xml" +java_system_library_path = "{{ values.yr_package_path }}/runtime/service/java/lib" +host_ip = "{{ values.function_agent.ip }}" +port = {{ values.function_agent.port }} +data_system_port = "{{ values.ds_worker.port }}" +agent_address = "{{ values.function_agent.ip }}:{{ values.function_agent.port }}" +enable_metrics = true +metrics_config = "" +metrics_config_file = "{{ values.metrics.metrics_config_file }}" +runtime_initial_port = 21006 +port_num = 65535 +system_timeout = 1800000 +metrics_collector_type = "proc" +proc_metrics_cpu = {{ values.cpu_num }} +custom_resources = "" +is_protomsg_to_runtime = true # if cli_package_path/runtime/service/cpp/snlib exist, set to false +massif_enable = false +enable_inherit_env = false +memory_detection_interval = 1000 +oom_kill_enable = false +oom_kill_control_limit = 0 +oom_consecutive_detection_count = 3 +kill_process_timeout_seconds = 0 +runtime_ds_connect_timeout = 1800 +runtime_direct_connection_enable = false +ssl_enable = false +ssl_base_path = "" +etcd_auth_type = "{{ values.etcd.auth_type }}" +etcd_root_ca_file = "{{ values.etcd.auth.ca_file }}" +etcd_cert_file = "{{ values.etcd.auth.client_cert_file }}" +etcd_key_file = "{{ values.etcd.auth.client_key_file }}" +etcd_ssl_base_path = "{{ values.etcd.auth.ssl_base_path }}" +ssl_root_file = "" +ssl_cert_file = "" +ssl_key_file = "" +runtime_default_config = "" +proc_metrics_memory = {{ values.memory_num }} +runtime_instance_debug_enable = false +local_node_id = "{{ values.node_id }}" + + +# [collector] +# bin_path = "collector" + +# [collector.env] +# LD_LIBRARY_PATH = "./runtime/go/bin;./data_system/sdk/go/lib" + +# [collector.args] +# collect_id = "" +# datasystem_port = 31501 +# etcd_config_services = "127.0.0.1:32379" +# ip = "127.0.0.1" +# log_root = "/var/log/collector" +# manager_address = "127.0.0.1:9081" +# port = 9082 + +[faas-frontend] +bin_path = "{{ values.faas_frontend.bin_path }}" + +[faas-frontend.env] +LD_LIBRARY_PATH = "{{ values.yr_package_path }}/functionsystem/lib:{{ values.yr_package_path }}/runtime/service/go/bin" +GO_RUNTIME_BIN="{{ values.yr_package_path }}/runtime/service/go/bin" +POD_NAME="frontend-process" +FUNCTION_LIB_PATH="{{ values.yr_package_path }}/pattern/pattern_faas/faasfrontend/faasfrontend.so" +INIT_ARGS_FILE_PATH="{{ values.deploy_path }}/faas_frontend_init_args.json" +ENABLE_SERVER_MODE="true" +INIT_HANDLER="faasfrontend.InitHandler" +CALL_HANDLER="faasfrontend.CallHandler" +CHECKPOINT_HANDLER="faasfrontend.CheckpointHandler" +RECOVER_HANDLER="faasfrontend.RecoverHandler" +SHUTDOWN_HANDLER="faasfrontend.ShutdownHandler" +SIGNAL_HANDLER="faasfrontend.SignalHandler" +YR_FUNCTION_LIB_PATH="{{ values.yr_package_path }}/pattern/pattern_faas/faasfrontend/" +GLOG_log_dir="{{ values.fs.log.path }}" +YR_LOG_LEVEL="{{ values.fs.log.level }}" +POD_IP="{{ values.faas_frontend.ip }}" +NODE_IP="{{ values.faas_frontend.ip }}" +DATASYSTEM_ADDR="{{ values.ds_worker.ip }}:{{ values.ds_worker.port }}" +INSTANCE_ID="driver-faas-frontend-${NODE_ID}" +FAAS_LOG_PATH="{{ values.fs.log.path }}" + +[faas-frontend.health_check] +enable = true +timeout = 60 + +[faas-frontend.args] +jobId= "{{ values.node_id }}" +runtimeId="faas_frontend_libruntime" +instanceId="driver-faas-frontend-{{ values.node_id }}" +functionName='0/0-system-faasfrontend/$latest' +logLevel="{{ values.fs.log.level }}" +logPath="{{ values.fs.log.path }}" +functionSystemAddress="${IP_ADDRESS}:${FUNCTION_PROXY_GRPC_PORT}" +driverMode = true +enableMTLS = false +# privateKeyPath=${PRIVATE_KEY_PATH} +# certificateFilePath=${CERTIFICATE_FILE_PATH} +# verifyFilePath=${VERIFY_FILE_PATH} +# encryptPrivateKeyPasswd=${ENCRYPT_PRIVATE_KEY_PASSWD} +# primaryKeyStoreFile=${PRIMARY_KEY_STORE_FILE} +# standbyKeyStoreFile=${STANDBY_KEY_STORE_FILE} +enableDsEncrypt = false +# >> "${FS_LOG_PATH}/${NODE_ID}-faas_frontend${STD_LOG_SUFFIX}" 2>&1 & \ No newline at end of file diff --git a/functionsystem/apps/yr/cli/const.py b/functionsystem/apps/yr/cli/const.py new file mode 100644 index 0000000..7405574 --- /dev/null +++ b/functionsystem/apps/yr/cli/const.py @@ -0,0 +1,29 @@ +from enum import Enum + +DEFAULT_CONFIG_PATH = "/etc/yuanrong/config.toml" +HTTP_PROTOCOL = "http://" +SESSION_LATEST_PATH = "/tmp/yr/session_latest" +DEFAULT_LOG_DIR = f"{SESSION_LATEST_PATH}/logs" +DEFAULT_DEPLOY_DIR = f"{SESSION_LATEST_PATH}/deploy" +SESSION_JSON_PATH = f"{SESSION_LATEST_PATH}/session.json" +DEFAULT_DEPLOY_DIR = f"{SESSION_LATEST_PATH}/deploy" +DEFAULT_VALUES_TOML = "cli/values.toml" +DEFAULT_CONFIG_TEMPLATE_PATH = "cli/config.toml.jinja" + +FS_COMPONENTS = [ + "domain-scheduler", + "function-agent", + "function-master", + "function-proxy", + "function-accessor", + "iam-server", + "runtime-manager", +] + +ETCD_CIPHER_SUITES = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384" + + +class StartMode(Enum): + MASTER = "master" + AGENT = "agent" + ALL = "all" diff --git a/functionsystem/apps/yr/cli/main.py b/functionsystem/apps/yr/cli/main.py new file mode 100644 index 0000000..8c9381b --- /dev/null +++ b/functionsystem/apps/yr/cli/main.py @@ -0,0 +1,193 @@ +import argparse +import logging +import sys +from pathlib import Path + +from yr.cli.config import ConfigResolver +from yr.cli.const import DEFAULT_CONFIG_PATH, DEFAULT_CONFIG_TEMPLATE_PATH, DEFAULT_VALUES_TOML, StartMode +from yr.cli.system_launcher import SystemLauncher + +# logging.basicConfig( +# level=logging.DEBUG, +# format="%(asctime)s.%(msecs)03d | %(levelname)-7s | %(name)s:%(funcName)s:%(lineno)d - %(message)s", +# datefmt="%Y-%m-%d %H:%M:%S", +# ) +# logger = logging.getLogger(__name__) +RESET = "\033[0m" +LEVEL_COLORS = { + logging.DEBUG: "\033[37m", # white/grey + logging.INFO: "\033[36m", # cyan + logging.WARNING: "\033[33m", # yellow + logging.ERROR: "\033[31m", # red + logging.CRITICAL: "\033[35m", # magenta +} + + +class ColorFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + color = LEVEL_COLORS.get(record.levelno, "") + # original_levelname = record.levelname + # record.levelname = f"{color}{original_levelname}{RESET}" + # try: + # return super().format(record) + # finally: + # record.levelname = original_levelname + msg = super().format(record) + return f"{color}{msg}{RESET}" + + +fmt = "%(asctime)s.%(msecs)03d | %(levelname)-7s | %(name)s:%(funcName)s:%(lineno)d - %(message)s" +datefmt = "%Y-%m-%d %H:%M:%S" +handler = logging.StreamHandler() +handler.setFormatter(ColorFormatter(fmt, datefmt=datefmt)) +logging.basicConfig(level=logging.DEBUG, handlers=[handler]) +logger = logging.getLogger(__name__) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="yr", + description="openYuanRong CLI", + ) + + parser.add_argument( + "-c", + "--config", + dest="config", + help="Path to config.toml (default: /etc/yuanrong/config.toml)", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + # start master|agent + start_parser = subparsers.add_parser("start", help="Start yr master or agent") + start_parser.add_argument( + "--master-info", + type=str, + help='Master info in format "ip:port"', + ) + start_parser.add_argument( + "-s", + "--set", + action="append", + type=str, + nargs="*", + metavar="KEY=VALUE", + help="override config values from command line, such as -s etcd.bin_path=/custom/path/to/etcd etcd.health_check.enable=false", + ) + start_parser.add_argument( + "target", + choices=["master", "agent"], + help="Target to start: master or agent", + ) + + # exec + exec_parser = subparsers.add_parser( + "exec", help="Exec a single component based on config.toml. Intended for use as a container entrypoint" + ) + exec_parser.add_argument( + "component", + help="Component name, e.g. function-master, runtime-manager, function-agent", + ) + + # status + status_parser = subparsers.add_parser("status", help="Show system status") + status_parser.add_argument( + "-f", + "--file", + help="Path to session file (default: /tmp/yr/session_latest/session.json)", + ) + + # stop + stop_parser = subparsers.add_parser("stop", help="Stop system components") + stop_parser.add_argument( + "--force", + action="store_true", + help="Force stop components (SIGKILL instead of SIGTERM)", + ) + stop_parser.add_argument( + "-f", + "--file", + help="Path to session file (default: /tmp/yr/session_latest/session.json)", + ) + + # version + subparsers.add_parser("version", help="Show version") + + # config dump|template + cfg_parser = subparsers.add_parser("config", help="Config related commands") + cfg_sub = cfg_parser.add_subparsers(dest="config_cmd", required=True) + cfg_sub.add_parser("dump", help="Dump merged config") + cfg_sub.add_parser("template", help="Print config template") + + return parser + + +def main(cmdargs=None) -> None: + parser = build_parser() + args = parser.parse_args(cmdargs) + + config_path = Path(args.config or DEFAULT_CONFIG_PATH).resolve() + if args.config and not config_path.exists(): + logger.critical(f"Config file {config_path} does not exist.") + sys.exit(1) + + cli_dir = Path(__file__).resolve().parent + + if args.command == "start": + mode = StartMode.MASTER if args.target == "master" else StartMode.AGENT + launcher = SystemLauncher(config_path, cli_dir, mode, args) + launcher.load_components() + success = launcher.start_all() + sys.exit(0 if success else 1) + + elif args.command == "exec": + from yr.cli.component.registry import LAUNCHER_CLASSES + + cfg = ConfigResolver(config_path, cli_dir) + component = LAUNCHER_CLASSES.get(args.component)(args.component, cfg) + component.exec() + + elif args.command == "status": + session_file = None + if args.file: + session_file = args.file + launcher = SystemLauncher(config_path, cli_dir, session_file=session_file) + launcher.status() + sys.exit(0) + + elif args.command == "stop": + session_file = None + if args.file: + session_file = args.file + launcher = SystemLauncher(config_path, cli_dir, session_file=session_file) + ok = launcher.stop_from_session(force=args.force) + sys.exit(0 if ok else 1) + + elif args.command == "version": + from yr.cli.build_info import COMMIT, VERSION + + print(f"yr version: {VERSION}") + print(f"commit: {COMMIT}") + sys.exit(0) + + elif args.command == "config": + import tomli_w + + if args.config_cmd == "dump": + cfg = ConfigResolver(config_path, cli_dir) + print(tomli_w.dumps(cfg.rendered_config)) + elif args.config_cmd == "template": + cfg = ConfigResolver(config_path, cli_dir) + values_path = cli_dir.parent / DEFAULT_VALUES_TOML + template_path = cli_dir.parent / DEFAULT_CONFIG_TEMPLATE_PATH + cfg._print_default_config(values_path, template_path) + sys.exit(0) + + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/functionsystem/apps/yr/cli/requirements.txt b/functionsystem/apps/yr/cli/requirements.txt new file mode 100644 index 0000000..373ce17 --- /dev/null +++ b/functionsystem/apps/yr/cli/requirements.txt @@ -0,0 +1 @@ +tomli_w \ No newline at end of file diff --git a/functionsystem/apps/yr/cli/services.yaml b/functionsystem/apps/yr/cli/services.yaml new file mode 100644 index 0000000..2419b18 --- /dev/null +++ b/functionsystem/apps/yr/cli/services.yaml @@ -0,0 +1,24 @@ +- service: defaultservice # 服务名（必填） + kind: yrlib # 函数类型（必填） 进程部署只支持 yrlib + description: this is the default service # 函数描述（非必填） + functions: # 函数（必填） + default: # 函数名（必填） + cpu: 500 # 函数 CPU 大小，单位：1/1000 核（必填） + memory: 500 # 函数 MEM 大小，单位：MB（必填） + runtime: python3.9 # 函数 runtime 类型（必填） + # environment: # 函数环境变量，内置环境变量定义 + # "key1" : "value1" + # storageType: "local" # 代码包存储类型，默认值为 local, 进程部署只支持 local: 代码包存储在磁盘中 + # codePath: "/home/sn/function-packages" # 代码包本地路径，默认值为空（除了 python，此项必填） + py: + cpu: 500 + memory: 500 + runtime: python3.9 + java: + cpu: 500 + memory: 500 + runtime: java1.8 + cpp: + cpu: 500 + memory: 500 + runtime: cpp11 \ No newline at end of file diff --git a/functionsystem/apps/yr/cli/system_launcher.py b/functionsystem/apps/yr/cli/system_launcher.py new file mode 100644 index 0000000..68fefb6 --- /dev/null +++ b/functionsystem/apps/yr/cli/system_launcher.py @@ -0,0 +1,472 @@ +from __future__ import annotations + +import errno +import json +import logging +import os +import signal +import subprocess +import sys +import threading +import time +from argparse import Namespace +from pathlib import Path +from typing import Optional + +from yr.cli.component.base import ComponentConfig, ComponentLauncher +from yr.cli.component.registry import LAUNCHER_CLASSES +from yr.cli.config import ConfigResolver +from yr.cli.const import DEFAULT_DEPLOY_DIR, DEFAULT_LOG_DIR, SESSION_JSON_PATH, SESSION_LATEST_PATH, StartMode + +logger = logging.getLogger(__name__) + + +class SessionManager: + def __init__(self, session_id: str, session_file: str | None = None) -> None: + self.session_file = session_file or SESSION_JSON_PATH + self.session_data = { + "components": {}, + "start_time": None, + "session_id": session_id, + "cluster_info": {}, + } + + def save_session(self, components: dict[str, ComponentConfig], config: dict[str, any]): + self.session_data["start_time"] = time.ctime() + + for name, comp in components.items(): + if comp.pid: + self.session_data["components"][name] = { + "pid": comp.pid, + "status": comp.status.value, + "start_time": comp.start_time, + "cmd": comp.args, + } + + def parse_addr(addr: str) -> tuple[str, str]: + return tuple(addr.split("//")[-1].rsplit(":", 1)) + + etcd_ip, etcd_port = parse_addr(config["etcd"]["args"]["listen-client-urls"]) + _, etcd_peer_port = parse_addr(config["etcd"]["args"]["listen-peer-urls"]) + ds_master_ip, ds_master_port = parse_addr(config["ds-master"]["args"]["master_address"]) + fm_ip, fm_port = parse_addr(config["function-master"]["args"]["ip"]) + _, fp_port = parse_addr(config["function-proxy"]["args"]["address"]) + _, ds_worker_port = parse_addr(config["ds-worker"]["args"]["worker_address"]) + + self.session_data["cluster_info"] = { + "for-join": { + "function_master.ip": fm_ip, + "function_master.port": fm_port, + "etcd.ip": etcd_ip, + "etcd.port": etcd_port, + "etcd.peer_port": etcd_peer_port, + "ds_master.ip": ds_master_ip, + "ds_master.port": ds_master_port, + "function_proxy.port": fp_port, + "function_proxy.grpc_port": config["function-proxy"]["args"]["grpc_listen_port"], + "ds_worker.port": ds_worker_port, + "agent.ip": config["function-agent"]["args"]["ip"], + }, + "for-sdk": { + "YR_DS_ADDRESS": config["ds-worker"]["args"]["worker_address"], + "YR_SERVER_ADDRESS": f"{config['function-proxy']['args']['ip']}:{config['function-proxy']['args']['grpc_listen_port']}", + }, + } + + with open(self.session_file, "w") as f: + json.dump(self.session_data, f, indent=2) + f.write("\n") + + def load_session(self) -> dict[str, any]: + if os.path.exists(self.session_file): + with open(self.session_file) as f: + return json.load(f) + return {} + + def clear_session(self): + if os.path.exists(self.session_file): + os.remove(self.session_file) + + def is_session_active(self) -> bool: + if not os.path.exists(self.session_file): + logger.info(f"No active session found in {self.session_file}.") + return False + session = self.load_session() + if not session or "session_id" not in session: + return False + for comp_info in session.get("components", {}).values(): + pid = comp_info.get("pid") + if pid: + try: + os.kill(pid, 0) + return True + except OSError: + logger.warning(f"Process {comp_info.get('name', 'unknown')} with PID {pid} not running.") + continue + + +class SystemLauncher: + def __init__( + self, + config_path: Path, + cli_dir: Path, + mode: StartMode | None = None, + args: Namespace | None = None, + session_file: str | None = None, + ): + self.mode = mode + self.resolver = ConfigResolver(config_path, cli_dir, mode, args) + # TODO: self.resolver.validate() + self.session_manager = SessionManager(self.resolver.runtime_context["time"], session_file) + self.components: dict[str, ComponentLauncher] = {} + self.processes: dict[str, subprocess.Popen] = {} + + self._register_component_launchers() + + # TODO: at exit, stop all components + + # def get_final_config(self) -> dict[str, any]: + # for _, launcher in self.components.items(): + # launcher.prepare_config() + # return self.resolver.raw_config + + def _register_component_launchers(self): + self.launcher_classes = LAUNCHER_CLASSES + + def _prepare_environment(self): + deploy_path = self.resolver.runtime_context["deploy_path"] + + os.makedirs(deploy_path) + symlink_path = SESSION_LATEST_PATH + if os.path.islink(symlink_path) or os.path.exists(symlink_path): + os.remove(symlink_path) + logger.debug(f"Creating symlink {symlink_path} -> {deploy_path}") + # /tmp/yr/session_latest -> real deploy path. we only need to focus on dealing with /tmp/yr/session_latest + os.symlink(deploy_path, symlink_path) + os.makedirs(DEFAULT_LOG_DIR, exist_ok=True) + os.makedirs(DEFAULT_DEPLOY_DIR, exist_ok=True) + os.chdir(DEFAULT_DEPLOY_DIR) + self.resolver.runtime_context["deploy_path"] = deploy_path + + def load_components(self): + if self.mode == StartMode.ALL: + for comp_name, launcher in LAUNCHER_CLASSES.items(): + self.components[comp_name] = launcher(comp_name, self.resolver) + return + + components_config = self.resolver.rendered_config["mode"].get(self.mode.value, {}) + + for comp_name, enable in components_config.items(): + if enable: + launcher_class: ComponentLauncher | None = self.launcher_classes.get(comp_name) + if launcher_class is None: + logger.critical(f"Unknown component: {comp_name}") + sys.exit(1) + + launcher = launcher_class(comp_name, self.resolver) + self.components[comp_name] = launcher + + def _start_component(self, component_name: str) -> Optional[subprocess.Popen]: + launcher = self.components[component_name] + process = launcher.launch() + self.processes[component_name] = process + + return process + + def start_all(self) -> bool: + """启动所有组件并等待健康检查通过""" + logger.info("Starting system components...") + self._prepare_environment() + components_order = self._get_start_order() + for comp_name in components_order: + logger.info(f"Starting {comp_name}...") + process = self._start_component(comp_name) + if not process: + logger.info(f"Failed to start {comp_name}") + return False + + time.sleep(1) + + logger.info("✅ All components started. Beginning health checks...") + + all_healthy = self._perform_health_checks() + if all_healthy: + self.session_manager.save_session( + {name: launcher.component_config for name, launcher in self.components.items()}, + self.resolver.rendered_config, + ) + logger.info("✅ All components are healthy!") + logger.debug(f"Session saved to: {self.session_manager.session_file}") + self._print_status() + return True + logger.info("❌ Some components failed health checks.") + self.stop_all(force=True) + return False + + def _get_start_order(self) -> list[str]: + """ + 根据 depends_on 依赖关系确定启动顺序（拓扑排序）。 + 假设每个 launcher.component_config.depends_on 是一个组件名列表。 + """ + graph: dict[str, set[str]] = {name: set() for name in self.components.keys()} + in_degree: dict[str, int] = {name: 0 for name in self.components.keys()} + + for name, launcher in self.components.items(): + depends_on = launcher.component_config.depends_on or [] + for dep in depends_on: + if dep not in self.components: + logger.critical(f"Component '{name}' depends on unknown component '{dep}'") + sys.exit(1) + if name not in graph[dep]: + graph[dep].add(name) + in_degree[name] += 1 + + from collections import deque + + queue: deque[str] = deque([name for name, deg in in_degree.items() if deg == 0]) + order: list[str] = [] + + while queue: + cur = queue.popleft() + order.append(cur) + for nxt in graph[cur]: + in_degree[nxt] -= 1 + if in_degree[nxt] == 0: + queue.append(nxt) + + if len(order) != len(self.components): + cycle_nodes = [name for name, deg in in_degree.items() if deg > 0] + logger.critical(f"Detected cyclic or unresolved dependencies among components: {cycle_nodes}") + sys.exit(1) + + logger.debug(f"Component start order (topological): {order}") + return order + + def _perform_health_checks(self) -> bool: + if not self.components: + return True + + overall_timeout = max( + launcher.resolver.rendered_config[launcher.component_config.name]["health_check"].get("timeout", 30) + for launcher in self.components.values() + ) + overall_deadline = time.time() + overall_timeout + + results: dict[str, bool] = {} + threads: list[threading.Thread] = [] + + def run_check(name: str, launcher: ComponentLauncher): + ok = launcher.wait_until_healthy(overall_deadline=overall_deadline) + results[name] = ok + + for name, launcher in self.components.items(): + t = threading.Thread(target=run_check, args=(name, launcher), daemon=True) + threads.append(t) + t.start() + + for t in threads: + remaining = overall_deadline - time.time() + if remaining <= 0: + break + t.join(timeout=remaining) + + unhealthy_components = [name for name, ok in results.items() if not ok] + + if not unhealthy_components and len(results) == len(self.components): + logger.info("All components are healthy") + return True + + logger.info("Health check failed. Unhealthy components:") + for name in unhealthy_components: + launcher = self.components[name] + logger.info(f" ❌ {name} (status: {launcher.component_config.status.value})") + if launcher.component_config.process: + exit_code = launcher.component_config.process.poll() + if exit_code is not None: + logger.info(f" Process exited with code: {exit_code}") + + missing = set(self.components.keys()) - set(results.keys()) + for name in missing: + logger.info(f" ❌ {name} (health check did not complete in time)") + + return False + + def stop_all(self, force: bool = False): + """停止所有组件""" + logger.info("Stopping system components...") + + for _, launcher in self.components.items(): + launcher.terminate(force=force) + self.session_manager.clear_session() + logger.info("✅ All components stopped.") + + def stop_from_session(self, force: bool = False): + """从会话文件停止系统""" + + def wait_pid_exit(pid: int, deadline: float, interval: int = 2) -> bool: + """Poll until PID exits or timeout. Returns True if exited.""" + logger.info(f"Waiting for PID {pid} to exit...") + while time.time() < deadline: + try: + os.kill(pid, 0) + except OSError as e: + if e.errno == errno.ESRCH: + return True + else: + logger.error(f"Error checking PID {pid}: {e}") + return True + time.sleep(interval) + return False + + if not os.path.exists(self.session_manager.session_file): + logger.info("Error: No active session found.") + return False + session = self.session_manager.load_session() + if not session or "components" not in session: + logger.error("Error: Invalid session file.") + return False + + logger.info(f"Stopping components from session file {self.session_manager.session_file}...") + + for comp_name, comp_info in session["components"].items(): + pid = comp_info.get("pid") + if pid: + try: + if force: + os.kill(pid, signal.SIGKILL) + logger.info(f"Killed {comp_name} (PID: {pid})") + else: + os.kill(pid, signal.SIGTERM) + logger.info(f"Terminated {comp_name} (PID: {pid})") + except ProcessLookupError: + logger.info(f"Process {comp_name} (PID: {pid}) not found") + except Exception as e: + logger.info(f"Error stopping {comp_name}: {e}") + + deadline = time.time() + 60 # wait up to 60 seconds for graceful exit + logger.info("Waiting for components to exit gracefully with timeout 60 seconds...") + for comp_name, comp_info in session["components"].items(): + pid = comp_info.get("pid") + if pid: + exited = wait_pid_exit(pid, deadline) + if exited: + logger.info(f"{comp_name} (PID: {pid}) has exited.") + else: + logger.warning(f"{comp_name} (PID: {pid}) did not gracefully exit in time.") + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + logger.info(f"Process {comp_name} (PID: {pid}) already exited.") + logger.info(f"Killed {comp_name} (PID: {pid}) after timeout.") + + self.session_manager.clear_session() + logger.info("✅ All components stopped.") + return True + + def _print_status(self): + logger.info("Component Status:") + logger.info("-" * 60) + logger.info(f"{'Component':<15} {'PID':<8} {'Status':<12} {'Uptime':<10}") + logger.info("-" * 60) + + for name, launcher in self.components.items(): + config = launcher.component_config + pid = config.pid or "N/A" + status = config.status.value + if config.start_time: + uptime = int(time.time() - config.start_time) + uptime_str = f"{uptime}s" + else: + uptime_str = "N/A" + logger.info(f"{name:<15} {pid:<8} {status:<12} {uptime_str:<10}") + logger.info("-" * 60) + if self.mode == StartMode.MASTER: + self._print_join_info() + + # where should we put sessions info? ray writes to /tmp/ray/session_latest + def _print_join_info(self): + logger.info("Export the following environment variables to enable cluster auto-detection when using YR SDK:") + cluster_env = """ +export YR_LOG_LEVEL=DEBUG +export DEPLOY_PATH=/tmp/yr/session_latest +export YR_SERVER_ADDRESS={0} +export YR_DS_ADDRESS={1} +export YR_IS_CLUSTER=true +export YRFUNCID='sn:cn:yrk:12345678901234561234567890123456:function:0-test-test:$latest' +export YR_CPP_FUNCID='sn:cn:yrk:12345678901234561234567890123456:function:0-test-test-cpp:$latest' +""".format( + self.session_manager.session_data["cluster_info"]["for-sdk"]["YR_SERVER_ADDRESS"], + self.session_manager.session_data["cluster_info"]["for-sdk"]["YR_DS_ADDRESS"], + ) + print(cluster_env) + join_help = """To join an existing cluster, execute the following commands in your shell on worker nodes: + +yr start agent --master-info "etcd.ip={0};etcd.port={1};etcd.peer_port={2};function-master.args.ip={3};ds-master.ip={4};ds-master.port={5}"(not working yet) + +OR + +mkdir -p /etc/yuanrong/ && cat << EOF > /etc/yuanrong/config.toml && yr start agent +[values.etcd] +ip = "{0}" +port = {1} +peer_port = {2} + +[values.ds_master] +ip = "{3}" +port = {4} + +[values.function_master] +ip = "{5}" +global_scheduler_port = {6} +EOF +""".format( + self.session_manager.session_data["cluster_info"]["for-join"]["etcd.ip"], + self.session_manager.session_data["cluster_info"]["for-join"]["etcd.port"], + self.session_manager.session_data["cluster_info"]["for-join"]["etcd.peer_port"], + self.session_manager.session_data["cluster_info"]["for-join"]["ds_master.ip"], + self.session_manager.session_data["cluster_info"]["for-join"]["ds_master.port"], + self.session_manager.session_data["cluster_info"]["for-join"]["function_master.ip"], + self.session_manager.session_data["cluster_info"]["for-join"]["function_master.port"], + ) + print(join_help) + + # TODO: keep this for old runtime sdk yr.init() + old_deploy_path = "/tmp/yr_sessions" + os.makedirs(old_deploy_path, exist_ok=True) + old_session_file = os.path.join(old_deploy_path, "yr_current_master_info") + with open(old_session_file, "w") as f: + master_info = "local_ip:{9},master_ip:{0},etcd_ip:{1},etcd_port:{2},etcd_peer_port:{3},global_scheduler_port:{4},ds_master_port:{5},bus-proxy:{6},bus:{7},ds-worker:{8},\n".format( + self.session_manager.session_data["cluster_info"]["for-join"]["function_master.ip"], + self.session_manager.session_data["cluster_info"]["for-join"]["etcd.ip"], + self.session_manager.session_data["cluster_info"]["for-join"]["etcd.port"], + self.session_manager.session_data["cluster_info"]["for-join"]["etcd.peer_port"], + self.session_manager.session_data["cluster_info"]["for-join"]["function_master.port"], + self.session_manager.session_data["cluster_info"]["for-join"]["ds_master.port"], + self.session_manager.session_data["cluster_info"]["for-join"]["function_proxy.port"], + self.session_manager.session_data["cluster_info"]["for-join"]["function_proxy.grpc_port"], + self.session_manager.session_data["cluster_info"]["for-join"]["ds_worker.port"], + self.session_manager.session_data["cluster_info"]["for-join"]["agent.ip"], + ) + f.write(master_info) + + def status(self): + """显示系统状态""" + if self.session_manager.is_session_active(): + session = self.session_manager.load_session() + + logger.info("System Status: RUNNING") + logger.info(f"Started at: {session.get('start_time')}") + logger.info(f"Session ID: {session.get('session_id')}") + logger.info("Components:") + + for comp_name, comp_info in session.get("components", {}).items(): + pid = comp_info.get("pid") + status = "RUNNING" + if pid: + try: + os.kill(pid, 0) + except (OSError, ProcessLookupError): + status = "STOPPED" + logger.info(f" {comp_name}: {status} (PID: {pid})") + else: + logger.info("System Status: STOPPED") diff --git a/functionsystem/apps/yr/cli/system_launcher_test.py b/functionsystem/apps/yr/cli/system_launcher_test.py new file mode 100644 index 0000000..494b66a --- /dev/null +++ b/functionsystem/apps/yr/cli/system_launcher_test.py @@ -0,0 +1,30 @@ +import unittest + +from yr.cli.component.base import ComponentConfig +from yr.cli.component.etcd import ETCDLauncher +from yr.cli.system_launcher import SystemLauncher + + +class TestSystemLauncher(unittest.TestCase): + def test_topological_sort_no_cycle(self): + launcher = SystemLauncher.__new__(SystemLauncher) + resolver = None + launcher.components = { + "etcd": ETCDLauncher("etcd", resolver, config=ComponentConfig(name="etcd")), + "ds-worker": ETCDLauncher( + "ds-worker", resolver, config=ComponentConfig(name="ds-worker", depends_on=["etcd"]) + ), + "function-master": ETCDLauncher( + "function-master", + resolver, + config=ComponentConfig(name="function-master", depends_on=["ds-worker", "etcd"]), + ), + } + + order = launcher._get_start_order() + print(order) + self.assertEqual(order, ["etcd", "ds-worker", "function-master"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/functionsystem/apps/yr/cli/utils.py b/functionsystem/apps/yr/cli/utils.py new file mode 100644 index 0000000..5697a59 --- /dev/null +++ b/functionsystem/apps/yr/cli/utils.py @@ -0,0 +1,87 @@ +import logging +import os +import random +import socket +import string +import sys +from contextlib import closing +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def get_ip() -> str: + hostname = socket.gethostname() + ip_list = socket.getaddrinfo(hostname, None) + + for ip_info in ip_list: + ip = ip_info[4][0] + if ip != "127.0.0.1" and not ip.startswith("::") and "." in ip: + return ip + + logger.critical("Failed to get IP address") + sys.exit(1) + + +def trim_hostname(): + hostname = get_hostname() + length = len(hostname) + if length == 0: + random_hostname = "".join(random.choices(string.ascii_letters, k=6)) + return f"node-{random_hostname}" + else: + return hostname + + +def get_hostname() -> str: + try: + return os.uname().nodename + except AttributeError: + return socket.gethostname() + except Exception as e: + logger.error(f"get host name failed {e}") + return "" + + +def is_valid_port(port: int) -> bool: + return 1 <= port <= 65535 + + +def is_port_available(port: int, host: str) -> bool: + if not is_valid_port(port): + return False + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(0.5) + s.bind((host, port)) + return True + except (socket.timeout, OSError): + return False + + +# TODO: what if two requests get free port at the same time? +def port_or_free(port) -> int: + if is_port_available(int(port), ""): + return int(port) + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + new_port = s.getsockname()[1] + logger.debug(f"port {port} is occupied, use {new_port} instead") + return new_port + + +def is_valid_file_path(path: str, allow_empty: bool = True) -> bool: + if not path: + return allow_empty + return Path(path).exists() + + +def get_total_memory_mb() -> int: + with open("/proc/meminfo", "r") as f: + meminfo = f.read() + total_kb = 0 + for line in meminfo.split("\n"): + if line.startswith("MemTotal:"): + total_kb = int(line.split(":")[1].strip().replace(" kB", "")) + total_mb = total_kb // 1024 + return total_mb diff --git a/functionsystem/apps/yr/cli/values.toml b/functionsystem/apps/yr/cli/values.toml new file mode 100644 index 0000000..183d414 --- /dev/null +++ b/functionsystem/apps/yr/cli/values.toml @@ -0,0 +1,102 @@ +[values] +# host_ip = "10.88.0.9" +# deploy_path = "/tmp/yr/session_20251216_011824" +# yr_package_path = "/yuanrong/functionsystem/yr" +# cpu_num = "4000" # in millicores +# memory_num = "7358" # in MB +# shared_memory_num = "2452" # in MB +# node_id = "da721039f899-41520" +# log_level = "INFO" +# pid = "41520" +# host_ip = "x.x.x.x" +# deploy_path = "/tmp/yr/session_demo" +# yr_package_path = "/path/to/your/yr/package" +# cpu_num = "4000" # in millicores +# memory_num = "8192" # in MB +# shared_memory_num = "2730" # in MB +# node_id = "demo-node-001" +# log_level = "INFO" +# pid = "12345" +host_ip = "{{ ip }}" +deploy_path = "{{ deploy_path }}" +yr_package_path = "{{ yr_package_path }}" +cpu_num = "{{ cpu_millicores }}" # in millicores +memory_num = "{{ memory_num_mb }}" # in MB +shared_memory_num = "{{ memory_num_mb // 3 }}" # in MB +node_id = "{{ node_id }}" +log_level = "INFO" +pid = "{{ pid }}" + +[values.metrics] +metrics_config = "" +metrics_config_file = "{{ values.yr_package_path }}/functionsystem/config/metrics/metrics_config.json" + +[values.meta_store] +enable = false +address = "" +mode = "local" + +[values.fs] +schedule_plugins = '''["Label", "ResourceSelector", "Default", "Heterogeneous"]''' + +[values.fs.log] +also_log_to_stderr = false +async_log_buf_secs = 30 +async_log_max_queue_size = 51200 +async_log_thread_count = 1 +level = "{{ values.log_level }}" +path = "{{ values.deploy_path }}/logs/function_system" +rolling_max_files = 10 +rolling_max_size = 40 +rolling_retention_days = 30 +compress_enable = false + +[values.etcd] +bin_path = "{{ values.yr_package_path }}/third_party/etcd/etcd" +enable_multi_master = false +auth_type = "Noauth" + +[values.etcd.auth] +ssl_base_path = "" +ca_file = "" +cert_file = "" +key_file = "" +client_cert_file = "" +client_key_file = "" + +[[values.etcd.address]] +ip = "{{ values.host_ip }}" +peer_port = "{{ 32380|check_port() }}" +port = "{{ 32379|check_port() }}" + +[values.ds_master] +bin_path = "{{ values.yr_package_path }}/datasystem/service/datasystem_worker" +ip = "{{ values.host_ip }}" +port = "{{ 12123|check_port() }}" + +[values.ds_worker] +bin_path = "{{ values.yr_package_path }}/datasystem/service/datasystem_worker" +ip = "{{ values.host_ip }}" +port = "{{ 31501|check_port() }}" +spill = false + +[values.function_master] +bin_path = "{{ values.yr_package_path }}/functionsystem/bin/function_master" +ip = "{{ values.host_ip }}" +global_scheduler_port = "{{ 22770|check_port() }}" + +[values.function_proxy] +bin_path = "{{ values.yr_package_path }}/functionsystem/bin/function_proxy" +ip = "{{ values.host_ip }}" +port = "{{ 22772|check_port() }}" +grpc_listen_port = "{{ 22773|check_port() }}" + +[values.function_agent] +bin_path = "{{ values.yr_package_path }}/functionsystem/bin/function_agent" +ip = "{{ values.host_ip }}" +port = "{{ 58866|check_port() }}" + +[values.faas_frontend] +bin_path = "{{ values.yr_package_path }}/runtime/service/go/bin/goruntime" +ip = "{{ values.host_ip }}" +port = "{{ 8888|check_port() }}" -- Gitee