From 6ccc64b69641a86043b5cfe1a411e300c9d6718e Mon Sep 17 00:00:00 2001 From: zangqianglei <785165816@qq.com> Date: Thu, 3 Aug 2023 18:38:37 +0800 Subject: [PATCH 1/4] Recommended Knob Settings result adds a column for the current value of the parameter --- dbmind/components/xtuner/tuner/knob.py | 8 ++-- dbmind/components/xtuner/tuner/recommend.py | 41 ++++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/dbmind/components/xtuner/tuner/knob.py b/dbmind/components/xtuner/tuner/knob.py index 2e15152..742adbf 100644 --- a/dbmind/components/xtuner/tuner/knob.py +++ b/dbmind/components/xtuner/tuner/knob.py @@ -51,11 +51,11 @@ class RecommendedKnobs: print(construct_dividing_line('Recommended Knob Settings', padding='*')) knob_table = PrettyTable() - knob_table.field_names = ['name', 'recommend', 'min', 'max', 'restart'] + knob_table.field_names = ['name', 'recommend', 'min', 'max', 'current', 'restart'] for knob in self.all_knobs: knob_table.add_row( - (knob.name, knob.recommend, knob.min, knob.max, knob.restart) + (knob.name, knob.recommend, knob.min, knob.max, knob.current, knob.restart) ) print(knob_table) @@ -116,6 +116,7 @@ class Knob: original: Str type. A setting value from `pg_settings` or user's configuration. min: Optional. Int, float type. max: Optional. Int, float type. + current: Optional. Int, float type. type: String type. Constrained by Knob.TYPE. restart: Boolean type. @@ -126,6 +127,7 @@ class Knob: self.recommend = kwargs.get('recommend') self.original = self.current = None self.user_set = kwargs.get('default') + self.current_value = kwargs.get('current') self._min = kwargs.get('min') self._max = kwargs.get('max') self.type = kwargs.get('type') @@ -140,7 +142,6 @@ class Knob: if str in (type(self._min), type(self._max)): raise ValueError("'min', and 'max' fields of knob should not be str type.") - # Refresh scale. self._scale = None self.fresh_scale() @@ -226,6 +227,7 @@ class Knob: 'restart': self.restart, 'default': self.user_set if self.user_set is not None else self.original, 'recommend': self.to_string(self.current), + 'current_value': self.to_string(self.current_value), 'min': self._min, 'max': self._max }} diff --git a/dbmind/components/xtuner/tuner/recommend.py b/dbmind/components/xtuner/tuner/recommend.py index 289f414..f6afb4d 100644 --- a/dbmind/components/xtuner/tuner/recommend.py +++ b/dbmind/components/xtuner/tuner/recommend.py @@ -32,19 +32,22 @@ def round4(v): return int(v + (4 - v % 4)) -def instantiate_knob(name, recommend, knob_type, value_min=0, value_max=1, restart=False): +def instantiate_knob(name, recommend, knob_type, current, value_min=0, value_max=1, restart=False): if knob_type not in Knob.TYPE.ITEMS: raise TypeError("The type of parameter 'knob_type' is incorrect.") if knob_type == Knob.TYPE.INT: recommend = int(recommend) + current = int(current) value_max = int(value_max) value_min = int(value_min) elif knob_type == Knob.TYPE.FLOAT: recommend = float(recommend) + current = float(current) value_max = float(value_max) value_min = float(value_min) elif knob_type == Knob.TYPE.BOOL: + current_value_type = type(current) value_type = type(recommend) if value_type is bool: recommend = 1 if recommend else 0 @@ -54,12 +57,22 @@ def instantiate_knob(name, recommend, knob_type, value_min=0, value_max=1, resta pass else: raise ValueError - + # Convert the current value to the same type as the recommended value. + if current_value_type is bool: + current = 1 if current else 0 + elif current_value_type is str and current_value_type in ('on', 'off'): + current = 1 if current == 'on' else 0 + elif current_value_type is int: + pass + else: + raise ValueError + value_max = 1 value_min = 0 return Knob(name, recommend=recommend, + current=current, min=value_min, max=value_max, type=knob_type, @@ -202,6 +215,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_process_memory", recommend=suitable_mem, knob_type=Knob.TYPE.INT, + current=self.metric["max_process_memory"] * self.metric.block_size, value_max=suitable_mem, value_min=min_mem, restart=True) @@ -257,6 +271,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_connections", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["max_connections"], value_max=upper, value_min=lower, restart=True) @@ -279,6 +294,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_connections", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["max_connections"], value_max=recommend * 2, value_min=lower, restart=True) @@ -305,6 +321,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_prepared_transactions", recommend=max_conn.recommend, knob_type=Knob.TYPE.INT, + current=max_pt, value_max=max_conn.max, value_min=max_conn.min, restart=True) @@ -340,6 +357,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="shared_buffers", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["shared_buffers"] * self.metric.block_size, value_max=upper, value_min=lower, restart=True) @@ -350,6 +368,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="shared_buffers", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["shared_buffers"] * self.metric.block_size, value_max=max(upper, lower), value_min=lower, restart=True) @@ -379,6 +398,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="work_mem", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["work_mem"], value_max=upper, value_min=lower, restart=False) @@ -389,6 +409,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="work_mem", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["work_mem"], value_max=upper, value_min=lower, restart=False) @@ -398,6 +419,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="maintenance_work_mem", recommend=self.work_mem.recommend * 10, knob_type=Knob.TYPE.INT, + current=self.metric["maintenance_work_mem"], value_max=self.work_mem.recommend * 20, value_min=self.work_mem.recommend, restart=False) @@ -411,6 +433,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_cache_size", recommend=lower, knob_type=Knob.TYPE.INT, + current=self.metric["effective_cache_size"] * self.metric.block_size, value_max=upper, value_min=lower, restart=False) @@ -418,6 +441,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_cache_size", recommend=upper, knob_type=Knob.TYPE.INT, + current=self.metric["effective_cache_size"] * self.metric.block_size, value_max=upper, value_min=lower, restart=False) @@ -431,6 +455,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_io_concurrency", recommend=2, knob_type=Knob.TYPE.INT, + current=self.metric["effective_io_concurrency"], value_max=4, value_min=0, restart=False) @@ -438,6 +463,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_io_concurrency", recommend=200, knob_type=Knob.TYPE.INT, + current=self.metric["effective_io_concurrency"], value_max=250, value_min=150, restart=False) @@ -459,6 +485,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="wal_buffers", recommend=self.shared_buffers.recommend * 1 / 32, knob_type=Knob.TYPE.INT, + current=self.metric["wal_buffers"] * self.metric.block_size, value_max=max(self.shared_buffers.recommend * 1 / 32, blocks_16m), value_min=min(self.shared_buffers.recommend * 1 / 64, blocks_16m), restart=True) @@ -472,6 +499,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="wal_buffers", recommend=self.shared_buffers.recommend * 1 / 32, knob_type=Knob.TYPE.INT, + current=self.metric["wal_buffers"], value_max=self.shared_buffers.recommend * 1 / 32, value_min=blocks_16m, restart=True) @@ -479,6 +507,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="wal_buffers", recommend=-1, knob_type=Knob.TYPE.INT, + current=self.metric["wal_buffers"] * self.metric.block_size, value_max=-1, value_min=-1, restart=True) @@ -491,6 +520,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="random_page_cost", recommend=3, knob_type=Knob.TYPE.FLOAT, + current=self.metric["random_page_cost"], value_max=4, value_min=2, restart=False) @@ -498,6 +528,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="random_page_cost", recommend=1, knob_type=Knob.TYPE.FLOAT, + current=self.metric["random_page_cost"], value_max=2, value_min=1, restart=False) @@ -530,6 +561,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="default_statistics_target", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["default_statistics_target"], value_max=1000, value_min=50, restart=False) @@ -546,6 +578,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="default_statistics_target", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["default_statistics_target"], value_max=150, value_min=10, restart=False) @@ -553,6 +586,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="default_statistics_target", recommend=100, knob_type=Knob.TYPE.INT, + current=self.metric["default_statistics_target"], value_max=300, value_min=80, restart=False) @@ -565,6 +599,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="enable_nestloop", recommend=0, knob_type=Knob.TYPE.BOOL, + current=self.metric["enable_nestloop"], value_max=1, value_min=0, restart=False) @@ -577,6 +612,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="enable_mergejoin", recommend=0, knob_type=Knob.TYPE.BOOL, + current=self.metric["enable_mergejoin"], value_max=1, value_min=0, restart=False) @@ -589,6 +625,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="enable_hashjoin", recommend=1, knob_type=Knob.TYPE.BOOL, + current=self.metric["enable_hashjoin"], value_max=1, value_min=0, restart=False) -- Gitee From b2d08eb44483e6201a562e58ebc51d9c83449a6a Mon Sep 17 00:00:00 2001 From: zangqianglei <785165816@qq.com> Date: Mon, 7 Aug 2023 10:26:30 +0800 Subject: [PATCH 2/4] The recommend mode in the xtuner feature displays a new column of the current value of the parameter in the recommendation result --- dbmind/components/xtuner/tuner/knob.py | 8 ++-- dbmind/components/xtuner/tuner/recommend.py | 41 ++++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/dbmind/components/xtuner/tuner/knob.py b/dbmind/components/xtuner/tuner/knob.py index 2e15152..327b69b 100644 --- a/dbmind/components/xtuner/tuner/knob.py +++ b/dbmind/components/xtuner/tuner/knob.py @@ -51,11 +51,11 @@ class RecommendedKnobs: print(construct_dividing_line('Recommended Knob Settings', padding='*')) knob_table = PrettyTable() - knob_table.field_names = ['name', 'recommend', 'min', 'max', 'restart'] + knob_table.field_names = ['name', 'recommend', 'min', 'max', 'current', 'restart'] for knob in self.all_knobs: knob_table.add_row( - (knob.name, knob.recommend, knob.min, knob.max, knob.restart) + (knob.name, knob.recommend, knob.min, knob.max, knob.current_value, knob.restart) ) print(knob_table) @@ -116,6 +116,7 @@ class Knob: original: Str type. A setting value from `pg_settings` or user's configuration. min: Optional. Int, float type. max: Optional. Int, float type. + current_value: Optional. Int, float type. type: String type. Constrained by Knob.TYPE. restart: Boolean type. @@ -126,6 +127,7 @@ class Knob: self.recommend = kwargs.get('recommend') self.original = self.current = None self.user_set = kwargs.get('default') + self.current_value = kwargs.get('current') self._min = kwargs.get('min') self._max = kwargs.get('max') self.type = kwargs.get('type') @@ -140,7 +142,6 @@ class Knob: if str in (type(self._min), type(self._max)): raise ValueError("'min', and 'max' fields of knob should not be str type.") - # Refresh scale. self._scale = None self.fresh_scale() @@ -226,6 +227,7 @@ class Knob: 'restart': self.restart, 'default': self.user_set if self.user_set is not None else self.original, 'recommend': self.to_string(self.current), + 'current_value': self.to_string(self.current_value), 'min': self._min, 'max': self._max }} diff --git a/dbmind/components/xtuner/tuner/recommend.py b/dbmind/components/xtuner/tuner/recommend.py index 289f414..f6afb4d 100644 --- a/dbmind/components/xtuner/tuner/recommend.py +++ b/dbmind/components/xtuner/tuner/recommend.py @@ -32,19 +32,22 @@ def round4(v): return int(v + (4 - v % 4)) -def instantiate_knob(name, recommend, knob_type, value_min=0, value_max=1, restart=False): +def instantiate_knob(name, recommend, knob_type, current, value_min=0, value_max=1, restart=False): if knob_type not in Knob.TYPE.ITEMS: raise TypeError("The type of parameter 'knob_type' is incorrect.") if knob_type == Knob.TYPE.INT: recommend = int(recommend) + current = int(current) value_max = int(value_max) value_min = int(value_min) elif knob_type == Knob.TYPE.FLOAT: recommend = float(recommend) + current = float(current) value_max = float(value_max) value_min = float(value_min) elif knob_type == Knob.TYPE.BOOL: + current_value_type = type(current) value_type = type(recommend) if value_type is bool: recommend = 1 if recommend else 0 @@ -54,12 +57,22 @@ def instantiate_knob(name, recommend, knob_type, value_min=0, value_max=1, resta pass else: raise ValueError - + # Convert the current value to the same type as the recommended value. + if current_value_type is bool: + current = 1 if current else 0 + elif current_value_type is str and current_value_type in ('on', 'off'): + current = 1 if current == 'on' else 0 + elif current_value_type is int: + pass + else: + raise ValueError + value_max = 1 value_min = 0 return Knob(name, recommend=recommend, + current=current, min=value_min, max=value_max, type=knob_type, @@ -202,6 +215,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_process_memory", recommend=suitable_mem, knob_type=Knob.TYPE.INT, + current=self.metric["max_process_memory"] * self.metric.block_size, value_max=suitable_mem, value_min=min_mem, restart=True) @@ -257,6 +271,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_connections", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["max_connections"], value_max=upper, value_min=lower, restart=True) @@ -279,6 +294,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_connections", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["max_connections"], value_max=recommend * 2, value_min=lower, restart=True) @@ -305,6 +321,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="max_prepared_transactions", recommend=max_conn.recommend, knob_type=Knob.TYPE.INT, + current=max_pt, value_max=max_conn.max, value_min=max_conn.min, restart=True) @@ -340,6 +357,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="shared_buffers", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["shared_buffers"] * self.metric.block_size, value_max=upper, value_min=lower, restart=True) @@ -350,6 +368,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="shared_buffers", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["shared_buffers"] * self.metric.block_size, value_max=max(upper, lower), value_min=lower, restart=True) @@ -379,6 +398,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="work_mem", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["work_mem"], value_max=upper, value_min=lower, restart=False) @@ -389,6 +409,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="work_mem", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["work_mem"], value_max=upper, value_min=lower, restart=False) @@ -398,6 +419,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="maintenance_work_mem", recommend=self.work_mem.recommend * 10, knob_type=Knob.TYPE.INT, + current=self.metric["maintenance_work_mem"], value_max=self.work_mem.recommend * 20, value_min=self.work_mem.recommend, restart=False) @@ -411,6 +433,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_cache_size", recommend=lower, knob_type=Knob.TYPE.INT, + current=self.metric["effective_cache_size"] * self.metric.block_size, value_max=upper, value_min=lower, restart=False) @@ -418,6 +441,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_cache_size", recommend=upper, knob_type=Knob.TYPE.INT, + current=self.metric["effective_cache_size"] * self.metric.block_size, value_max=upper, value_min=lower, restart=False) @@ -431,6 +455,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_io_concurrency", recommend=2, knob_type=Knob.TYPE.INT, + current=self.metric["effective_io_concurrency"], value_max=4, value_min=0, restart=False) @@ -438,6 +463,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="effective_io_concurrency", recommend=200, knob_type=Knob.TYPE.INT, + current=self.metric["effective_io_concurrency"], value_max=250, value_min=150, restart=False) @@ -459,6 +485,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="wal_buffers", recommend=self.shared_buffers.recommend * 1 / 32, knob_type=Knob.TYPE.INT, + current=self.metric["wal_buffers"] * self.metric.block_size, value_max=max(self.shared_buffers.recommend * 1 / 32, blocks_16m), value_min=min(self.shared_buffers.recommend * 1 / 64, blocks_16m), restart=True) @@ -472,6 +499,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="wal_buffers", recommend=self.shared_buffers.recommend * 1 / 32, knob_type=Knob.TYPE.INT, + current=self.metric["wal_buffers"], value_max=self.shared_buffers.recommend * 1 / 32, value_min=blocks_16m, restart=True) @@ -479,6 +507,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="wal_buffers", recommend=-1, knob_type=Knob.TYPE.INT, + current=self.metric["wal_buffers"] * self.metric.block_size, value_max=-1, value_min=-1, restart=True) @@ -491,6 +520,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="random_page_cost", recommend=3, knob_type=Knob.TYPE.FLOAT, + current=self.metric["random_page_cost"], value_max=4, value_min=2, restart=False) @@ -498,6 +528,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="random_page_cost", recommend=1, knob_type=Knob.TYPE.FLOAT, + current=self.metric["random_page_cost"], value_max=2, value_min=1, restart=False) @@ -530,6 +561,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="default_statistics_target", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["default_statistics_target"], value_max=1000, value_min=50, restart=False) @@ -546,6 +578,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="default_statistics_target", recommend=recommend, knob_type=Knob.TYPE.INT, + current=self.metric["default_statistics_target"], value_max=150, value_min=10, restart=False) @@ -553,6 +586,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="default_statistics_target", recommend=100, knob_type=Knob.TYPE.INT, + current=self.metric["default_statistics_target"], value_max=300, value_min=80, restart=False) @@ -565,6 +599,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="enable_nestloop", recommend=0, knob_type=Knob.TYPE.BOOL, + current=self.metric["enable_nestloop"], value_max=1, value_min=0, restart=False) @@ -577,6 +612,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="enable_mergejoin", recommend=0, knob_type=Knob.TYPE.BOOL, + current=self.metric["enable_mergejoin"], value_max=1, value_min=0, restart=False) @@ -589,6 +625,7 @@ class OpenGaussKnobAdvisor: return instantiate_knob(name="enable_hashjoin", recommend=1, knob_type=Knob.TYPE.BOOL, + current=self.metric["enable_hashjoin"], value_max=1, value_min=0, restart=False) -- Gitee From 2e7a61c427ebb87650ab790d8fbd0357f60b0430 Mon Sep 17 00:00:00 2001 From: zangqianglei <785165816@qq.com> Date: Mon, 7 Aug 2023 10:32:16 +0800 Subject: [PATCH 3/4] Only parameters whose recommended value differs from the current value are displayed. --- dbmind/components/xtuner/tuner/knob.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dbmind/components/xtuner/tuner/knob.py b/dbmind/components/xtuner/tuner/knob.py index 4218e00..e27eca5 100644 --- a/dbmind/components/xtuner/tuner/knob.py +++ b/dbmind/components/xtuner/tuner/knob.py @@ -54,9 +54,10 @@ class RecommendedKnobs: knob_table.field_names = ['name', 'recommend', 'min', 'max', 'current', 'restart'] for knob in self.all_knobs: - knob_table.add_row( - (knob.name, knob.recommend, knob.min, knob.max, knob.current_value, knob.restart) - ) + if knob.recommend != knob.current_value: # Only parameters whose recommended value differs from the current value are displayed. + knob_table.add_row( + (knob.name, knob.recommend, knob.min, knob.max, knob.current_value, knob.restart) + ) print(knob_table) def dump(self, fp, dump_report_knobs=True): -- Gitee From 7a27cb316c52eea1f42d3a05f119fc07fd590852 Mon Sep 17 00:00:00 2001 From: zangqianglei <785165816@qq.com> Date: Thu, 17 Aug 2023 16:51:19 +0800 Subject: [PATCH 4/4] fix get database name error --- dbmind/common/opengauss_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbmind/common/opengauss_driver.py b/dbmind/common/opengauss_driver.py index 4d0884b..4cf0a75 100644 --- a/dbmind/common/opengauss_driver.py +++ b/dbmind/common/opengauss_driver.py @@ -302,7 +302,7 @@ class DriverBundle: ) discovered = set() for dbname in all_db_list: - if dbname in ('template0', 'template1'): # Skip these useless databases. + if dbname[0] in ('template0', 'template1'): # Skip these useless databases. continue discovered.add(dbname[0]) -- Gitee