From b95f1982732f4bca2cfefc20fcf477a2c05491fe Mon Sep 17 00:00:00 2001 From: Chen22 Date: Tue, 19 Oct 2021 00:26:26 -0700 Subject: [PATCH 1/4] feat: install hybridse package and sdk locally (#536) --- .github/workflows/integration-test-src.yml | 2 +- .github/workflows/java.yml | 12 +++++++++++- src/base/ddl_parser.cc | 3 +-- steps/init_env.sh | 9 +++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-test-src.yml b/.github/workflows/integration-test-src.yml index 9970188f3..5d7a7119d 100644 --- a/.github/workflows/integration-test-src.yml +++ b/.github/workflows/integration-test-src.yml @@ -9,7 +9,7 @@ on: default: 'all' env: GIT_SUBMODULE_STRATEGY: recursive - HYBRIDSE_SOURCE: + HYBRIDSE_SOURCE: local jobs: java-sdk-test-standalone-0: diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index a417e63d5..33759e14c 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -20,19 +20,25 @@ on: pull_request: branches: - main +env: + HYBRIDSE_SOURCE: local jobs: linux-unit-test: name: Run unit tests in Linux runs-on: ubuntu-latest container: - image: maven:3.6-jdk-8 + image: ghcr.io/4paradigm/hybridsql:0.4.0 steps: - uses: actions/checkout@v2 + - name: Build and install hybridse-sdk + run: | + bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }} - name: Run unit tests for Linux working-directory: java/openmldb-batch run: | mvn clean compile test - name: Run unit tests for all-in-one + if: ${{ github.event_name == 'push' }} working-directory: java/openmldb-batch run: | mvn clean compile test -Pallinone @@ -46,11 +52,15 @@ jobs: with: java-version: '8' distribution: 'adopt' + - name: Build and install hybridse-sdk + run: | + bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }} - name: Run unit tests for MacOS working-directory: java/openmldb-batch run: | mvn clean compile test -Pmacos - name: Run unit tests for all-in-one + if: ${{ github.event_name == 'push' }} working-directory: java/openmldb-batch run: | mvn clean compile test -Pallinone diff --git a/src/base/ddl_parser.cc b/src/base/ddl_parser.cc index 76ba731e4..fd6b84640 100644 --- a/src/base/ddl_parser.cc +++ b/src/base/ddl_parser.cc @@ -412,8 +412,7 @@ bool ResolveColumnToSourceColumnName(const hybridse::node::ColumnRefNode* col, c // try loose the relation if (!status.isOK() && !col->GetRelationName().empty()) { - status = schemas_ctx->ResolveColumnID("", "", col->GetColumnName(), &column_id, &path_idx, - &child_column_id, + status = schemas_ctx->ResolveColumnID("", "", col->GetColumnName(), &column_id, &path_idx, &child_column_id, &source_column_id, &source); } diff --git a/steps/init_env.sh b/steps/init_env.sh index 9fe33e4c9..f0d3fab21 100755 --- a/steps/init_env.sh +++ b/steps/init_env.sh @@ -63,6 +63,15 @@ if [[ ${HYBRIDSE_SOURCE} = "local" ]]; then cmake --build build --target install -- -j"$(nproc)" mv hybridse "$THIRDPARTY_PATH/hybridse" popd + + pushd "${ROOT}/hybridse/java" + if [[ "$OSTYPE" = "darwin"* ]]; then + mvn install -Dmaven.test.skip=true -Pmacos -Dgpg.skip + elif [[ "$OSTYPE" = "linux-gnu"* ]]; then + mvn install -Dmaven.test.skip=true -Dgpg.skip + fi + popd + else echo "Download hybridse package" pushd "${THIRDSRC_PATH}" -- Gitee From b0f286a9bb135c8070f81071abfc5fad4d06a245 Mon Sep 17 00:00:00 2001 From: frankszn Date: Fri, 15 Oct 2021 17:14:08 +0800 Subject: [PATCH 2/4] change the api of EngineOptions and JitOptions to follow google cpp style just leave the corresponding part in openmldb-batch unchanged --- .../examples/toydb/src/bm/engine_bm_case.cc | 12 +++--- .../toydb/src/cmd/toydb_run_engine.cc | 18 ++++----- .../toydb/src/testing/toydb_engine_test.cc | 8 ++-- .../src/testing/toydb_engine_test_base.cc | 2 +- hybridse/include/vm/engine.h | 40 +++++++++---------- hybridse/include/vm/engine_context.h | 16 ++++---- .../_4paradigm/hybridse/sdk/JitManager.java | 8 ++-- .../hybridse/sdk/RequestEngine.java | 6 +-- .../hybridse/sdk/SqlEngineTest.java | 6 +-- .../physical/batch_request_optimize_test.cc | 2 +- hybridse/src/testing/engine_test_base.h | 2 +- hybridse/src/vm/engine.cc | 28 ++++++------- hybridse/src/vm/engine_compile_test.cc | 26 ++++++------ hybridse/src/vm/jit.cc | 6 +-- hybridse/src/vm/jit_wrapper.cc | 6 +-- hybridse/src/vm/jit_wrapper_test.cc | 16 ++++---- hybridse/src/vm/schemas_context_test.cc | 2 +- src/base/ddl_parser.h | 6 +-- src/catalog/sdk_catalog_test.cc | 6 +-- src/sdk/db_sdk.cc | 8 ++-- src/tablet/tablet_impl.cc | 2 +- 21 files changed, 113 insertions(+), 113 deletions(-) diff --git a/hybridse/examples/toydb/src/bm/engine_bm_case.cc b/hybridse/examples/toydb/src/bm/engine_bm_case.cc index 153f4db11..fcccbc3fa 100644 --- a/hybridse/examples/toydb/src/bm/engine_bm_case.cc +++ b/hybridse/examples/toydb/src/bm/engine_bm_case.cc @@ -109,7 +109,7 @@ static void EngineRequestMode(const std::string sql, MODE mode, auto catalog = vm::BuildOnePkTableStorage(size); vm::EngineOptions options; if (hybridse::sqlcase::SqlCase::IsCluster()) { - options.set_cluster_optimized(true); + options.SetClusterOptimized(true); } Engine engine(catalog, options); RequestRunSession session; @@ -827,18 +827,18 @@ void EngineBenchmarkOnCase(hybridse::sqlcase::SqlCase& sql_case, // NOLINT vm::EngineOptions engine_options; if (engine_mode == vm::kBatchRequestMode) { - engine_options.set_batch_request_optimized( + engine_options.SetBatchRequestOptimized( sql_case.batch_request_optimized_); } if (hybridse::sqlcase::SqlCase::IsCluster()) { - engine_options.set_cluster_optimized(true); + engine_options.SetClusterOptimized(true); } else { - engine_options.set_cluster_optimized(false); + engine_options.SetClusterOptimized(false); } if (hybridse::sqlcase::SqlCase::IsDisableExprOpt()) { - engine_options.set_enable_expr_optimize(false); + engine_options.SetEnableExprOptimize(false); } else { - engine_options.set_enable_expr_optimize(true); + engine_options.SetEnableExprOptimize(true); } std::unique_ptr engine_runner; if (engine_mode == vm::kBatchMode) { diff --git a/hybridse/examples/toydb/src/cmd/toydb_run_engine.cc b/hybridse/examples/toydb/src/cmd/toydb_run_engine.cc index 91876c324..efbd0c7b0 100644 --- a/hybridse/examples/toydb/src/cmd/toydb_run_engine.cc +++ b/hybridse/examples/toydb/src/cmd/toydb_run_engine.cc @@ -71,18 +71,18 @@ int RunSingle(const std::string& yaml_path) { return ENGINE_TEST_RET_INVALID_CASE; } EngineOptions options; - options.set_cluster_optimized(FLAGS_cluster_mode == "cluster"); - options.set_batch_request_optimized(FLAGS_enable_batch_request_opt); - options.set_performance_sensitive(FLAGS_performance_sensitive); - options.set_enable_expr_optimize(FLAGS_enable_expr_opt); - options.set_enable_batch_window_parallelization( + options.SetClusterOptimized(FLAGS_cluster_mode == "cluster"); + options.SetBatchRequestOptimized(FLAGS_enable_batch_request_opt); + options.SetPerformanceSensitive(FLAGS_performance_sensitive); + options.SetEnableExprOptimize(FLAGS_enable_expr_opt); + options.SetEnableBatchWindowParallelization( FLAGS_enable_batch_window_parallelization); JitOptions& jit_options = options.jit_options(); - jit_options.set_enable_mcjit(FLAGS_enable_mcjit); - jit_options.set_enable_vtune(FLAGS_enable_vtune); - jit_options.set_enable_gdb(FLAGS_enable_gdb); - jit_options.set_enable_perf(FLAGS_enable_perf); + jit_options.SetEnableMCJIT(FLAGS_enable_mcjit); + jit_options.SetEnableVTune(FLAGS_enable_vtune); + jit_options.SetEnableGDB(FLAGS_enable_gdb); + jit_options.SetEnablePerf(FLAGS_enable_perf); for (auto& sql_case : cases) { if (FLAGS_case_id >= 0 && diff --git a/hybridse/examples/toydb/src/testing/toydb_engine_test.cc b/hybridse/examples/toydb/src/testing/toydb_engine_test.cc index 33b44f6fc..2743ec412 100644 --- a/hybridse/examples/toydb/src/testing/toydb_engine_test.cc +++ b/hybridse/examples/toydb/src/testing/toydb_engine_test.cc @@ -61,7 +61,7 @@ TEST_P(EngineTest, TestBatchRequestEngineForLastRow) { TEST_P(EngineTest, TestClusterRequestEngine) { ParamType sql_case = GetParam(); EngineOptions options; - options.set_cluster_optimized(true); + options.SetClusterOptimized(true); LOG(INFO) << "ID: " << sql_case.id() << ", DESC: " << sql_case.desc(); if (!boost::contains(sql_case.mode(), "request-unsupport") && !boost::contains(sql_case.mode(), "rtidb-unsupport") && @@ -74,7 +74,7 @@ TEST_P(EngineTest, TestClusterRequestEngine) { TEST_P(EngineTest, TestClusterBatchRequestEngine) { ParamType sql_case = GetParam(); EngineOptions options; - options.set_cluster_optimized(true); + options.SetClusterOptimized(true); LOG(INFO) << "ID: " << sql_case.id() << ", DESC: " << sql_case.desc(); if (!boost::contains(sql_case.mode(), "request-unsupport") && !boost::contains(sql_case.mode(), "rtidb-unsupport") && @@ -90,7 +90,7 @@ TEST_P(BatchRequestEngineTest, TestBatchRequestEngine) { ParamType sql_case = GetParam(); LOG(INFO) << "ID: " << sql_case.id() << ", DESC: " << sql_case.desc(); EngineOptions options; - options.set_cluster_optimized(false); + options.SetClusterOptimized(false); if (!boost::contains(sql_case.mode(), "batch-request-unsupport")) { EngineCheck(sql_case, options, kBatchRequestMode); } else { @@ -101,7 +101,7 @@ TEST_P(BatchRequestEngineTest, TestClusterBatchRequestEngine) { ParamType sql_case = GetParam(); LOG(INFO) << "ID: " << sql_case.id() << ", DESC: " << sql_case.desc(); EngineOptions options; - options.set_cluster_optimized(true); + options.SetClusterOptimized(true); if (!boost::contains(sql_case.mode(), "batch-request-unsupport") && !boost::contains(sql_case.mode(), "cluster-unsupport")) { EngineCheck(sql_case, options, kBatchRequestMode); diff --git a/hybridse/examples/toydb/src/testing/toydb_engine_test_base.cc b/hybridse/examples/toydb/src/testing/toydb_engine_test_base.cc index 4b56ad3b6..0a10e7fb9 100644 --- a/hybridse/examples/toydb/src/testing/toydb_engine_test_base.cc +++ b/hybridse/examples/toydb/src/testing/toydb_engine_test_base.cc @@ -86,7 +86,7 @@ bool InitToydbEngineCatalog( return false; } name_table_map[table_def.name()] = table; - if (engine_options.is_cluster_optimzied()) { + if (engine_options.IsClusterOptimzied()) { // add table with local tablet if (!AddTable(catalog, table_def, table, engine.get())) { return false; diff --git a/hybridse/include/vm/engine.h b/hybridse/include/vm/engine.h index 0c9620b55..3b0126751 100644 --- a/hybridse/include/vm/engine.h +++ b/hybridse/include/vm/engine.h @@ -47,81 +47,81 @@ class EngineOptions { EngineOptions(); /// Set `true` to enable storing ir results into SqlContext, default `false`. - inline void set_keep_ir(bool flag) { this->keep_ir_ = flag; } + inline void SetKeepIR(bool flag) { this->keep_ir_ = flag; } /// Return if support to store ir results into SqlContext. - inline bool is_keep_ir() const { return this->keep_ir_; } + inline bool IsKeepIR() const { return this->keep_ir_; } /// Set `true` if only support to compile SQL, default `false` /// /// If set `true`, the engine won't generate runner plan as well. - inline void set_compile_only(bool flag) { this->compile_only_ = flag; } + inline void SetCompileOnly(bool flag) { this->compile_only_ = flag; } /// Return if only support to compile physical plan. - inline bool is_compile_only() const { return compile_only_; } + inline bool IsCompileOnly() const { return compile_only_; } /// Set `true` if the engine only generate physical plan, default `false`. /// /// If set `true`, the engine won't build llvm jit. - inline void set_plan_only(bool flag) { plan_only_ = flag; } + inline void SetPlanOnly(bool flag) { plan_only_ = flag; } /// Return `true` if the engine only generate physical plan. - inline bool is_plan_only() const { return plan_only_; } + inline bool IsPlanOnly() const { return plan_only_; } /// Set `true` if the engine is performance sensitive, default `true`. /// /// Normally, the engine can support more abilities under performance un-sensitive mode. - inline void set_performance_sensitive(bool flag) { + inline void SetPerformanceSensitive(bool flag) { performance_sensitive_ = flag; } /// Return `true` if the engine is performance sensitive. - inline bool is_performance_sensitive() const { + inline bool IsPerformanceSensitive() const { return performance_sensitive_; } /// Set `true` to enable cluster optimization, default `false` - inline EngineOptions* set_cluster_optimized(bool flag) { + inline EngineOptions* SetClusterOptimized(bool flag) { cluster_optimized_ = flag; return this; } /// Return if the engine support cluster optimization. - inline bool is_cluster_optimzied() const { return cluster_optimized_; } + inline bool IsClusterOptimzied() const { return cluster_optimized_; } /// Set `true` to enable batch request optimization, default `true`. - inline EngineOptions* set_batch_request_optimized(bool flag) { + inline EngineOptions* SetBatchRequestOptimized(bool flag) { batch_request_optimized_ = flag; return this; } /// Return if the engine support batch request optimization. - inline bool is_batch_request_optimized() const { return batch_request_optimized_; } + inline bool IsBatchRequestOptimized() const { return batch_request_optimized_; } /// Set `true` to enable expression optimization, default `true`. - inline EngineOptions* set_enable_expr_optimize(bool flag) { + inline EngineOptions* SetEnableExprOptimize(bool flag) { enable_expr_optimize_ = flag; return this; } /// Return if the engine support expression optimization - inline bool is_enable_expr_optimize() const { return enable_expr_optimize_; } + inline bool IsEnableExprOptimize() const { return enable_expr_optimize_; } /// Set `true` to enable batch window parallelization, default `false`. - inline EngineOptions* set_enable_batch_window_parallelization(bool flag) { + inline EngineOptions* SetEnableBatchWindowParallelization(bool flag) { enable_batch_window_parallelization_ = flag; return this; } /// Return if the engine support batch window parallelization. - inline bool is_enable_batch_window_parallelization() const { + inline bool IsEnableBatchWindowParallelization() const { return enable_batch_window_parallelization_; } /// Set the maximum number of cache entries, default is `50`. - inline void set_max_sql_cache_size(uint32_t size) { + inline void SetMaxSQLCacheSize(uint32_t size) { max_sql_cache_size_ = size; } /// Return the maximum number of entries we can hold for compiling cache. - inline uint32_t max_sql_cache_size() const { return max_sql_cache_size_; } + inline uint32_t GetMaxSQLCacheSize() const { return max_sql_cache_size_; } /// Set `true` to enable spark unsafe row format, default `false`. - EngineOptions* set_enable_spark_unsaferow_format(bool flag); + EngineOptions* SetEnableSparkUnsaferowFormat(bool flag); /// Return if the engine can support can support spark unsafe row format. - inline bool is_enable_spark_unsaferow_format() const { + inline bool IsEnableSparkUnsaferowFormat() const { return enable_spark_unsaferow_format_; } diff --git a/hybridse/include/vm/engine_context.h b/hybridse/include/vm/engine_context.h index 59931a33b..300865d6c 100644 --- a/hybridse/include/vm/engine_context.h +++ b/hybridse/include/vm/engine_context.h @@ -82,17 +82,17 @@ class CompileInfoCache { class JitOptions { public: - bool is_enable_mcjit() const { return enable_mcjit_; } - void set_enable_mcjit(bool flag) { enable_mcjit_ = flag; } + bool IsEnableMCJIT() const { return enable_mcjit_; } + void SetEnableMCJIT(bool flag) { enable_mcjit_ = flag; } - bool is_enable_vtune() const { return enable_vtune_; } - void set_enable_vtune(bool flag) { enable_vtune_ = flag; } + bool IsEnableVTune() const { return enable_vtune_; } + void SetEnableVTune(bool flag) { enable_vtune_ = flag; } - bool is_enable_gdb() const { return enable_gdb_; } - void set_enable_gdb(bool flag) { enable_gdb_ = flag; } + bool IsEnableGDB() const { return enable_gdb_; } + void SetEnableGDB(bool flag) { enable_gdb_ = flag; } - bool is_enable_perf() const { return enable_perf_; } - void set_enable_perf(bool flag) { enable_perf_ = flag; } + bool IsEnablePerf() const { return enable_perf_; } + void SetEnablePerf(bool flag) { enable_perf_ = flag; } private: bool enable_mcjit_ = false; diff --git a/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/JitManager.java b/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/JitManager.java index a5eac7085..aac9b9907 100644 --- a/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/JitManager.java +++ b/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/JitManager.java @@ -70,22 +70,22 @@ public class JitManager { String enableMcJit = prop.getProperty("fesql.jit.enable_mcjit"); if (enableMcJit != null && enableMcJit.toLowerCase().equals("true")) { logger.info("Try enable llvm legacy mcjit support"); - options.set_enable_mcjit(true); + options.SetEnableMCJIT(true); } String enableVtune = prop.getProperty("fesql.jit.enable_vtune"); if (enableVtune != null && enableVtune.toLowerCase().equals("true")) { logger.info("Try enable intel jit events support"); - options.set_enable_vtune(true); + options.SetEnableVTune(true); } String enablePerf = prop.getProperty("fesql.jit.enable_perf"); if (enablePerf != null && enablePerf.toLowerCase().equals("true")) { logger.info("Try enable perf jit events support"); - options.set_enable_perf(true); + options.SetEnablePerf(true); } String enableGdb = prop.getProperty("fesql.jit.enable_gdb"); if (enableGdb != null && enableGdb.toLowerCase().equals("true")) { logger.info("Try enable gdb jit events support"); - options.set_enable_gdb(true); + options.SetEnableGDB(true); } } catch (IOException ex) { logger.debug("Can not find jit.properties", ex); diff --git a/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/RequestEngine.java b/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/RequestEngine.java index 4027c36dc..0070657af 100644 --- a/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/RequestEngine.java +++ b/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/RequestEngine.java @@ -51,9 +51,9 @@ public class RequestEngine implements AutoCloseable { */ public RequestEngine(String sql, TypeOuterClass.Database database) throws UnsupportedHybridSeException { options = new EngineOptions(); - options.set_keep_ir(true); - options.set_compile_only(true); - options.set_performance_sensitive(false); + options.SetKeepIR(true); + options.SetCompileOnly(true); + options.SetPerformanceSensitive(false); catalog = new SimpleCatalog(); session = new RequestRunSession(); catalog.AddDatabase(database); diff --git a/hybridse/java/hybridse-sdk/src/test/java/com/_4paradigm/hybridse/sdk/SqlEngineTest.java b/hybridse/java/hybridse-sdk/src/test/java/com/_4paradigm/hybridse/sdk/SqlEngineTest.java index f82140e4d..23405afbc 100644 --- a/hybridse/java/hybridse-sdk/src/test/java/com/_4paradigm/hybridse/sdk/SqlEngineTest.java +++ b/hybridse/java/hybridse-sdk/src/test/java/com/_4paradigm/hybridse/sdk/SqlEngineTest.java @@ -149,7 +149,7 @@ public class SqlEngineTest { } try { EngineOptions options = createDefaultEngineOptions(); - options.set_enable_batch_window_parallelization(true); + options.SetEnableBatchWindowParallelization(true); SqlEngine engine = new SqlEngine(sql, db.build(), options); Assert.assertNotNull(engine.getPlan()); } catch (UnsupportedHybridSeException e) { @@ -231,7 +231,7 @@ public class SqlEngineTest { } try { EngineOptions options = createDefaultEngineOptions(); - options.set_enable_batch_window_parallelization(true); + options.SetEnableBatchWindowParallelization(true); SqlEngine engine = new SqlEngine(sql, Lists.newArrayList(db.build(), db2.build()), options, defaultDbName); Assert.assertNotNull(engine.getPlan()); @@ -324,7 +324,7 @@ public class SqlEngineTest { } try { EngineOptions options = createDefaultEngineOptions(); - options.set_enable_batch_window_parallelization(true); + options.SetEnableBatchWindowParallelization(true); SqlEngine engine = new SqlEngine(sql, Lists.newArrayList(db.build(), db2.build()), options, defaultDbName); Assert.assertNull(engine.getPlan()); diff --git a/hybridse/src/passes/physical/batch_request_optimize_test.cc b/hybridse/src/passes/physical/batch_request_optimize_test.cc index afad652b5..f2909e7e5 100644 --- a/hybridse/src/passes/physical/batch_request_optimize_test.cc +++ b/hybridse/src/passes/physical/batch_request_optimize_test.cc @@ -148,7 +148,7 @@ void CheckOptimizePlan(const SqlCase& sql_case_org, auto catalog = std::make_shared(); InitSimpleCataLogFromSqlCase(sql_case, catalog); EngineOptions options; - options.set_compile_only(true); + options.SetCompileOnly(true); auto engine = std::make_shared(catalog, options); std::string sql_str = sql_case.sql_str(); for (int j = 0; j < sql_case.CountInputs(); ++j) { diff --git a/hybridse/src/testing/engine_test_base.h b/hybridse/src/testing/engine_test_base.h index 7b5901f29..2928857f8 100644 --- a/hybridse/src/testing/engine_test_base.h +++ b/hybridse/src/testing/engine_test_base.h @@ -359,7 +359,7 @@ class BatchRequestEngineTestRunner : public EngineTestRunner { size_t request_schema_size = static_cast(request_schema.size()); if (common_column_indices.empty() || common_column_indices.size() == request_schema_size || - !options_.is_batch_request_optimized()) { + !options_.IsBatchRequestOptimized()) { request_rows_ = original_request_data; } else { std::vector non_common_column_indices; diff --git a/hybridse/src/vm/engine.cc b/hybridse/src/vm/engine.cc index 13c448f46..58335a621 100644 --- a/hybridse/src/vm/engine.cc +++ b/hybridse/src/vm/engine.cc @@ -55,7 +55,7 @@ EngineOptions::EngineOptions() FLAGS_enable_spark_unsaferow_format = enable_spark_unsaferow_format_; } -EngineOptions* EngineOptions::set_enable_spark_unsaferow_format(bool flag) { +EngineOptions* EngineOptions::SetEnableSparkUnsaferowFormat(bool flag) { enable_spark_unsaferow_format_ = flag; FLAGS_enable_spark_unsaferow_format = flag; return this; @@ -78,8 +78,8 @@ bool Engine::GetDependentTables(const std::string& sql, const std::string& db, E info->get_sql_context().sql = sql; info->get_sql_context().db = db; info->get_sql_context().engine_mode = engine_mode; - SqlCompiler compiler(std::atomic_load_explicit(&cl_, std::memory_order_acquire), options_.is_keep_ir(), false, - options_.is_plan_only()); + SqlCompiler compiler(std::atomic_load_explicit(&cl_, std::memory_order_acquire), options_.IsKeepIR(), false, + options_.IsPlanOnly()); bool ok = compiler.Parse(info->get_sql_context(), status); if (!ok || 0 != status.code) { // TODO(chenjing): do clean @@ -198,11 +198,11 @@ bool Engine::Get(const std::string& sql, const std::string& db, RunSession& sess sql_context.sql = sql; sql_context.db = db; sql_context.engine_mode = session.engine_mode(); - sql_context.is_performance_sensitive = options_.is_performance_sensitive(); - sql_context.is_cluster_optimized = options_.is_cluster_optimzied(); - sql_context.is_batch_request_optimized = options_.is_batch_request_optimized(); - sql_context.enable_batch_window_parallelization = options_.is_enable_batch_window_parallelization(); - sql_context.enable_expr_optimize = options_.is_enable_expr_optimize(); + sql_context.is_performance_sensitive = options_.IsPerformanceSensitive(); + sql_context.is_cluster_optimized = options_.IsClusterOptimzied(); + sql_context.is_batch_request_optimized = options_.IsBatchRequestOptimized(); + sql_context.enable_batch_window_parallelization = options_.IsEnableBatchWindowParallelization(); + sql_context.enable_expr_optimize = options_.IsEnableExprOptimize(); sql_context.jit_options = options_.jit_options(); if (session.engine_mode() == kBatchMode) { sql_context.parameter_types = dynamic_cast(&session)->GetParameterSchema(); @@ -211,13 +211,13 @@ bool Engine::Get(const std::string& sql, const std::string& db, RunSession& sess sql_context.batch_request_info.common_column_indices = batch_req_sess->common_column_indices(); } - SqlCompiler compiler(std::atomic_load_explicit(&cl_, std::memory_order_acquire), options_.is_keep_ir(), false, - options_.is_plan_only()); + SqlCompiler compiler(std::atomic_load_explicit(&cl_, std::memory_order_acquire), options_.IsKeepIR(), false, + options_.IsPlanOnly()); bool ok = compiler.Compile(info->get_sql_context(), status); if (!ok || 0 != status.code) { return false; } - if (!options_.is_compile_only()) { + if (!options_.IsCompileOnly()) { ok = compiler.BuildClusterJob(info->get_sql_context(), status); if (!ok || 0 != status.code) { LOG(WARNING) << "fail to build cluster job: " << status.msg; @@ -261,8 +261,8 @@ bool Engine::Explain(const std::string& sql, const std::string& db, EngineMode e ctx.sql = sql; ctx.db = db; ctx.parameter_types = parameter_schema; - ctx.is_performance_sensitive = options_.is_performance_sensitive(); - ctx.is_cluster_optimized = options_.is_cluster_optimzied(); + ctx.is_performance_sensitive = options_.IsPerformanceSensitive(); + ctx.is_cluster_optimized = options_.IsClusterOptimzied(); ctx.is_batch_request_optimized = !common_column_indices.empty(); ctx.batch_request_info.common_column_indices = common_column_indices; SqlCompiler compiler(std::atomic_load_explicit(&cl_, std::memory_order_acquire), true, true, true); @@ -363,7 +363,7 @@ bool Engine::SetCacheLocked(const std::string& db, const std::string& sql, Engin using BoostLRU = boost::compute::detail::lru_cache>; std::map::iterator db_iter = mode_cache.find(db); if (db_iter == mode_cache.end()) { - db_iter = mode_cache.insert(db_iter, {db, BoostLRU(options_.max_sql_cache_size())}); + db_iter = mode_cache.insert(db_iter, {db, BoostLRU(options_.GetMaxSQLCacheSize())}); } auto& lru = db_iter->second; auto value = lru.get(sql); diff --git a/hybridse/src/vm/engine_compile_test.cc b/hybridse/src/vm/engine_compile_test.cc index 5e099e8a7..db153f3c9 100644 --- a/hybridse/src/vm/engine_compile_test.cc +++ b/hybridse/src/vm/engine_compile_test.cc @@ -58,8 +58,8 @@ TEST_F(EngineCompileTest, EngineLRUCacheTest) { // Simple Engine EngineOptions options; - options.set_compile_only(true); - options.set_max_sql_cache_size(1); + options.SetCompileOnly(true); + options.SetMaxSQLCacheSize(1); Engine engine(catalog, options); std::string sql = "select col1, col2 from t1;"; @@ -104,8 +104,8 @@ TEST_F(EngineCompileTest, EngineWithParameterizedLRUCacheTest) { // Simple Engine EngineOptions options; - options.set_compile_only(true); - options.set_max_sql_cache_size(1); + options.SetCompileOnly(true); + options.SetMaxSQLCacheSize(1); Engine engine(catalog, options); hybridse::codec::Schema parameter_schema; @@ -189,7 +189,7 @@ TEST_F(EngineCompileTest, EngineEmptyDefaultDBLRUCacheTest) { } EngineOptions options; - options.set_performance_sensitive(false); + options.SetPerformanceSensitive(false); Engine engine(catalog, options); std::string sql = "select db1.t1.col1, db1.t1.col2,db2.t2.col3,db2.t2.col4 from db1.t1 last join db2.t2 ORDER BY db2.t2.col5 " @@ -259,8 +259,8 @@ TEST_F(EngineCompileTest, EngineCompileOnlyTest) { "SELECT t1.COL1, t1.COL2, t2.COL1, t2.COL2 FROM t1 last join t2 " "order by t2.col5 on t1.col1 = t2.col2;"}; EngineOptions options; - options.set_compile_only(true); - options.set_performance_sensitive(false); + options.SetCompileOnly(true); + options.SetPerformanceSensitive(false); Engine engine(catalog, options); base::Status get_status; for (auto sqlstr : sql_str_list) { @@ -282,7 +282,7 @@ TEST_F(EngineCompileTest, EngineCompileOnlyTest) { "on " "t1.col1 = t2.col2;"}; EngineOptions options; - options.set_performance_sensitive(false); + options.SetPerformanceSensitive(false); Engine engine(catalog, options); base::Status get_status; for (auto sqlstr : sql_str_list) { @@ -425,8 +425,8 @@ TEST_F(EngineCompileTest, RouterTest) { "window w1 as (partition by col2 \n" "order by col5 rows between 3 preceding and current row);"; EngineOptions options; - options.set_compile_only(true); - options.set_performance_sensitive(false); + options.SetCompileOnly(true); + options.SetPerformanceSensitive(false); Engine engine(catalog, options); ExplainOutput explain_output; codec::Schema empty_parameter_schema; @@ -470,8 +470,8 @@ TEST_F(EngineCompileTest, ExplainBatchRequestTest) { "window w1 as (partition by col2 \n" "order by col5 rows between 3 preceding and current row);"; EngineOptions options; - options.set_compile_only(true); - options.set_performance_sensitive(false); + options.SetCompileOnly(true); + options.SetPerformanceSensitive(false); Engine engine(catalog, options); ExplainOutput explain_output; base::Status status; @@ -528,7 +528,7 @@ TEST_F(EngineCompileTest, EngineCompileWithoutDefaultDBTest) { "select db1.t1.col1, db1.t1.col2,db2.t2.col3,db2.t2.col4 from db1.t1 last join db2.t2 ORDER BY db2.t2.col5 " "on db1.t1.col1=db2.t2.col1;"}; EngineOptions options; - options.set_performance_sensitive(false); + options.SetPerformanceSensitive(false); Engine engine(catalog, options); base::Status get_status; for (auto sqlstr : sql_str_list) { diff --git a/hybridse/src/vm/jit.cc b/hybridse/src/vm/jit.cc index 44e214b4f..5ac8b944d 100644 --- a/hybridse/src/vm/jit.cc +++ b/hybridse/src/vm/jit.cc @@ -253,7 +253,7 @@ bool HybridSeMcJitWrapper::AddModule( } else { execution_engine_->addModule(std::move(module)); } - if (jit_options_.is_enable_vtune()) { + if (jit_options_.IsEnableVTune()) { auto listener = ::llvm::JITEventListener::createIntelJITEventListener(); if (listener == nullptr) { LOG(WARNING) << "Intel jit events is not enabled"; @@ -261,7 +261,7 @@ bool HybridSeMcJitWrapper::AddModule( execution_engine_->RegisterJITEventListener(listener); } } - if (jit_options_.is_enable_gdb()) { + if (jit_options_.IsEnableGDB()) { auto listener = ::llvm::JITEventListener::createGDBRegistrationListener(); if (listener == nullptr) { @@ -276,7 +276,7 @@ bool HybridSeMcJitWrapper::AddModule( #endif } } - if (jit_options_.is_enable_perf()) { + if (jit_options_.IsEnablePerf()) { auto listener = ::llvm::JITEventListener::createPerfJITEventListener(); if (listener == nullptr) { LOG(WARNING) << "Perf jit events is not enabled"; diff --git a/hybridse/src/vm/jit_wrapper.cc b/hybridse/src/vm/jit_wrapper.cc index f12dcd5fd..d758f80ab 100644 --- a/hybridse/src/vm/jit_wrapper.cc +++ b/hybridse/src/vm/jit_wrapper.cc @@ -70,7 +70,7 @@ HybridSeJitWrapper* HybridSeJitWrapper::Create() { } HybridSeJitWrapper* HybridSeJitWrapper::Create(const JitOptions& jit_options) { - if (jit_options.is_enable_mcjit()) { + if (jit_options.IsEnableMCJIT()) { #ifdef LLVM_EXT_ENABLE LOG(INFO) << "Create McJit engine"; return new HybridSeMcJitWrapper(jit_options); @@ -79,8 +79,8 @@ HybridSeJitWrapper* HybridSeJitWrapper::Create(const JitOptions& jit_options) { return new HybridSeLlvmJitWrapper(); #endif } else { - if (jit_options.is_enable_vtune() || jit_options.is_enable_perf() || - jit_options.is_enable_gdb()) { + if (jit_options.IsEnableVTune() || jit_options.IsEnablePerf() || + jit_options.IsEnableGDB()) { LOG(WARNING) << "LLJIT do not support jit events"; } return new HybridSeLlvmJitWrapper(); diff --git a/hybridse/src/vm/jit_wrapper_test.cc b/hybridse/src/vm/jit_wrapper_test.cc index f163ee29b..f148e4f7d 100644 --- a/hybridse/src/vm/jit_wrapper_test.cc +++ b/hybridse/src/vm/jit_wrapper_test.cc @@ -101,26 +101,26 @@ void simple_test(const EngineOptions &options) { TEST_F(JitWrapperTest, test) { EngineOptions options; - options.set_keep_ir(true); + options.SetKeepIR(true); simple_test(options); } #ifdef LLVM_EXT_ENABLE TEST_F(JitWrapperTest, test_mcjit) { EngineOptions options; - options.set_keep_ir(true); - options.jit_options().set_enable_mcjit(true); - options.jit_options().set_enable_gdb(true); - options.jit_options().set_enable_perf(true); - options.jit_options().set_enable_vtune(true); + options.SetKeepIR(true); + options.jit_options().SetEnableMCJIT(true); + options.jit_options().SetEnableGDB(true); + options.jit_options().SetEnablePerf(true); + options.jit_options().SetEnableVTune(true); simple_test(options); } #endif TEST_F(JitWrapperTest, test_window) { EngineOptions options; - options.set_keep_ir(true); - options.set_performance_sensitive(false); + options.SetKeepIR(true); + options.SetPerformanceSensitive(false); auto catalog = GetTestCatalog(); auto compile_info = Compile( "select col_1, sum(col_2) over w, " diff --git a/hybridse/src/vm/schemas_context_test.cc b/hybridse/src/vm/schemas_context_test.cc index 88fac4b15..1e05c922a 100644 --- a/hybridse/src/vm/schemas_context_test.cc +++ b/hybridse/src/vm/schemas_context_test.cc @@ -173,7 +173,7 @@ PhysicalOpNode* GetTestSqlPlan(SqlCase& sql_case, // NOLINT std::map idx_to_table_dict; auto catalog = std::make_shared(); EngineOptions options; - options.set_plan_only(true); + options.SetPlanOnly(true); auto engine = std::make_shared(catalog, options); InitSimpleCataLogFromSqlCase(sql_case, catalog); diff --git a/src/base/ddl_parser.h b/src/base/ddl_parser.h index 917a11868..88fa65ce1 100644 --- a/src/base/ddl_parser.h +++ b/src/base/ddl_parser.h @@ -237,9 +237,9 @@ class DDLParser { // TODO(hw): engine is input, do not create in here ::hybridse::vm::Engine::InitializeGlobalLLVM(); ::hybridse::vm::EngineOptions options; - options.set_keep_ir(true); - options.set_compile_only(true); - options.set_performance_sensitive(false); + options.SetKeepIR(true); + options.SetCompileOnly(true); + options.SetPerformanceSensitive(false); auto engine = std::make_shared(catalog, options); ::hybridse::base::Status status; diff --git a/src/catalog/sdk_catalog_test.cc b/src/catalog/sdk_catalog_test.cc index e1f707565..522f68642 100644 --- a/src/catalog/sdk_catalog_test.cc +++ b/src/catalog/sdk_catalog_test.cc @@ -61,7 +61,7 @@ TEST_F(SDKCatalogTest, sdk_smoke_test) { Procedures procedures; ASSERT_TRUE(catalog->Init(tables, procedures)); ::hybridse::vm::EngineOptions options; - options.set_compile_only(true); + options.SetCompileOnly(true); ::hybridse::vm::Engine engine(catalog, options); std::string sql = "select col1, col2 + 1 from t1;"; ::hybridse::vm::BatchRunSession session; @@ -81,7 +81,7 @@ TEST_F(SDKCatalogTest, sdk_window_smoke_test) { Procedures procedures; ASSERT_TRUE(catalog->Init(tables, procedures)); ::hybridse::vm::EngineOptions options; - options.set_compile_only(true); + options.SetCompileOnly(true); ::hybridse::vm::Engine engine(catalog, options); std::string sql = "select sum(col2) over w1, t1.col1, t1.col2 from t1 window w1 " @@ -106,7 +106,7 @@ TEST_F(SDKCatalogTest, sdk_lastjoin_smoke_test) { Procedures procedures; ASSERT_TRUE(catalog->Init(tables, procedures)); ::hybridse::vm::EngineOptions options; - options.set_compile_only(true); + options.SetCompileOnly(true); ::hybridse::vm::Engine engine(catalog, options); std::string sql = "select t1.col1 as c1, t1.col2 as c2 , t2.col1 as c3, t2.col2 as c4 " diff --git a/src/sdk/db_sdk.cc b/src/sdk/db_sdk.cc index 1407fe2ce..8fbcf51e7 100644 --- a/src/sdk/db_sdk.cc +++ b/src/sdk/db_sdk.cc @@ -74,8 +74,8 @@ bool ClusterSDK::Init() { << ",session timeout " << options_.session_timeout << " and session id " << zk_client_->GetSessionTerm(); ::hybridse::vm::EngineOptions eopt; - eopt.set_compile_only(true); - eopt.set_plan_only(true); + eopt.SetCompileOnly(true); + eopt.SetPlanOnly(true); engine_ = new ::hybridse::vm::Engine(catalog_, eopt); ok = BuildCatalog(); @@ -406,8 +406,8 @@ std::vector> DBSDK::GetProcedureIn bool StandAloneSDK::Init() { ::hybridse::vm::EngineOptions opt; - opt.set_compile_only(true); - opt.set_plan_only(true); + opt.SetCompileOnly(true); + opt.SetPlanOnly(true); engine_ = new ::hybridse::vm::Engine(catalog_, opt); return PeriodicRefresh(); } diff --git a/src/tablet/tablet_impl.cc b/src/tablet/tablet_impl.cc index 725e472c4..6051ab4c3 100644 --- a/src/tablet/tablet_impl.cc +++ b/src/tablet/tablet_impl.cc @@ -149,7 +149,7 @@ bool TabletImpl::Init(const std::string& real_endpoint) { bool TabletImpl::Init(const std::string& zk_cluster, const std::string& zk_path, const std::string& endpoint, const std::string& real_endpoint) { ::hybridse::vm::EngineOptions options; - options.set_cluster_optimized(FLAGS_enable_distsql); + options.SetClusterOptimized(FLAGS_enable_distsql); engine_ = std::unique_ptr<::hybridse::vm::Engine>(new ::hybridse::vm::Engine(catalog_, options)); catalog_->SetLocalTablet( std::shared_ptr<::hybridse::vm::Tablet>(new ::hybridse::vm::LocalTablet(engine_.get(), sp_cache_))); -- Gitee From f9fe506210be619ca4dc294dfff90c560c9d0350 Mon Sep 17 00:00:00 2001 From: frankszn Date: Sat, 16 Oct 2021 01:24:40 +0800 Subject: [PATCH 3/4] fixed the corresponding api of EngineOptions in engine_test_base.cc --- hybridse/src/testing/engine_test_base.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hybridse/src/testing/engine_test_base.cc b/hybridse/src/testing/engine_test_base.cc index 86bd967bb..eb5d272f0 100644 --- a/hybridse/src/testing/engine_test_base.cc +++ b/hybridse/src/testing/engine_test_base.cc @@ -433,9 +433,9 @@ void EngineTestRunner::RunCheck() { if (!sql_case_.batch_plan().empty() && engine_mode == kBatchMode) { ASSERT_EQ(oss.str(), sql_case_.batch_plan()); } else if (!sql_case_.cluster_request_plan().empty() && engine_mode == kRequestMode && - options_.is_cluster_optimzied()) { + options_.IsClusterOptimzied()) { ASSERT_EQ(oss.str(), sql_case_.cluster_request_plan()); - } else if (!sql_case_.request_plan().empty() && engine_mode == kRequestMode && !options_.is_cluster_optimzied()) { + } else if (!sql_case_.request_plan().empty() && engine_mode == kRequestMode && !options_.IsClusterOptimzied()) { ASSERT_EQ(oss.str(), sql_case_.request_plan()); } status = PrepareData(); -- Gitee From d4ae2ef76990e750c77c65e2a3979dc34dda54d6 Mon Sep 17 00:00:00 2001 From: frankszn Date: Sat, 16 Oct 2021 10:03:35 +0800 Subject: [PATCH 4/4] fixed the corresponding api of EngineOptions in SqlEngine.java --- .../main/java/com/_4paradigm/hybridse/sdk/SqlEngine.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/SqlEngine.java b/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/SqlEngine.java index 76dfe4230..0b14d86d1 100644 --- a/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/SqlEngine.java +++ b/hybridse/java/hybridse-sdk/src/main/java/com/_4paradigm/hybridse/sdk/SqlEngine.java @@ -99,9 +99,9 @@ public class SqlEngine implements AutoCloseable { */ public static EngineOptions createDefaultEngineOptions() { EngineOptions engineOptions = new EngineOptions(); - engineOptions.set_keep_ir(true); - engineOptions.set_compile_only(true); - engineOptions.set_performance_sensitive(false); + engineOptions.SetKeepIR(true); + engineOptions.SetCompileOnly(true); + engineOptions.SetPerformanceSensitive(false); return engineOptions; } -- Gitee