name : db_governor_lib.py # -*- coding: utf-8 -*- # db_governer_lib.py - Library for MySQL governor config files manipulating for cloudlinux-config utility # # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT import os import subprocess from typing import Optional # NOQA from lxml import etree from clcommon.clexception import FormattedException from clcommon.lib.mysql_governor_lib import MySQLGovernor from .clconfig_utils import ( boolean_to_on_off, str_to_boolean, time_convertor_to_dict, time_unit_to_letter, ) class DBGovernorException(FormattedException): pass DEFAULTS_DICT = { 'modeOfOperation': 'off', 'gatherDataForDetailedStats': 'on', 'save_uid': 'off', 'logRestrictedUsersQueries': 'off', 'scriptPath': '', 'userMaxConnections': 30, 'restrictType': { 'mode': 'limit', 'unlimit': '60s' }, 'slowQueries': { 'kill': 'off', 'logPath': '', 'timeout': 30 }, 'restrictLog': { 'logPath': '', 'format': 'LONG' }, 'errorLog': { 'logPath': '', 'format': 'ERROR' } } DbGovConfig = '/etc/container/mysql-governor.xml' # For unit tests def _open(file_name): return open(file_name, encoding='utf-8') def get_data_from_xml(x, node, attrib, default=None): """ Gets data from specified node and attribute :param x: Parent node object :param node: Node name to extract data :param attrib: Attribute name to extract data :param default: Default value to return if specified node/attribute not found :return: Value """ try: path = '//' + str(node) + '/@' + str(attrib) data = x.xpath(path)[0] except (AttributeError, IndexError): return default return data def convert_time_value_to_seconds(value): """ Convert value in minutes, hours, days to seconds :param value: value of slow limit :return: value in seconds """ value = str(value).lower() if value.endswith('m'): result = int(value[:-1]) * 60 elif value.endswith('h'): result = int(value[:-1]) * 60 * 60 elif value.endswith('d'): result = int(value[:-1]) * 60 * 60 * 24 elif value.endswith('s'): result = int(value[:-1]) else: result = int(value) return result def get_slow_queries_timeout(x, default): """ Gets slow queries timeout :param x: Root node :param default: Default value to return if timeout not found :return: Timeout """ try: data = x.xpath("default/limit[@name='slow']/@current")[0] data = convert_time_value_to_seconds(data) return int(data) except (AttributeError, IndexError, ValueError): return default def set_slow_queries_timeout(node, slow_timeout): """ Sets slow queries timeout :param node: Root node :param slow_timeout: new timeout value :return: None """ path = "default/limit[@name='slow']" tag_list = node.xpath(path) if len(tag_list) == 0: # Create default/limit chain node_list = node.xpath('default') if len(node_list) == 0: new_node = etree.Element('default') node.append(new_node) # 'default' node present, but 'limit' node with name="slow" attribute is absent default_node = node.xpath('default')[0] # Create 'limit' node tag_to_set = etree.Element('limit') default_node.append(tag_to_set) # set name attribute tag_to_set.set('name', 'slow') else: tag_to_set = tag_list[0] # Set attribute value tag_to_set.set('current', str(slow_timeout)) def set_data_to_xml(x, node_name, attribute_name, value): """ Sets value to specified node/attribute :param x: Parent node :param node_name: Node name :param attribute_name: Attribute name :param value: Value to set :return: None """ path = '//' + str(node_name) tag_list = x.xpath(path) if len(tag_list) == 0: # Tag absent, add it new_node = etree.Element(node_name) x.append(new_node) tag_list = x.xpath(path) tag = tag_list[0] # Set specified attribute tag.set(attribute_name, value) def _get_timedict_from_xml(x, tag_name, attr_name): """ Get time from specified tag/attribute as seconds :param x: XML object :param tag_name: XML tag name :param attr_name: XML object name :return: Number of seconds. Throws DBGovernorException if any error """ value = get_data_from_xml(x, tag_name, attr_name) try: return time_convertor_to_dict(value) except Exception as e: raise DBGovernorException( f"{DbGovConfig} has invalid value in governer/{tag_name}/@{attr_name}" ) from e def get_gov_mode_operation(parsed_xml_tree=None): # type: (etree.ElementBase) -> Optional[str] """ Get MySQL Governor mode. We return None if governor config isn't exists All exceptions will be catched in top level functions :param parsed_xml_tree: parsed governor config """ if not os.path.exists(DbGovConfig): return None if parsed_xml_tree is None: f = _open(DbGovConfig) x = etree.parse(f).getroot() else: x = parsed_xml_tree return get_data_from_xml(x, 'lve', 'use', DEFAULTS_DICT['modeOfOperation']) def get_db_gov_config(): """ Get MySQL Governer config :return: Dictionary with config data """ try: if not os.path.exists(DbGovConfig): return {} f = _open(DbGovConfig) x = etree.parse(f).getroot() result_dict = {} result_dict['modeOfOperation'] = get_gov_mode_operation(parsed_xml_tree=x) result_dict['gatherDataForDetailedStats'] = str_to_boolean( get_data_from_xml(x, 'statistic', 'mode', DEFAULTS_DICT['gatherDataForDetailedStats'])) result_dict['logRestrictedUsersQueries'] = str_to_boolean( get_data_from_xml(x, 'logqueries', 'use', DEFAULTS_DICT['logRestrictedUsersQueries'])) result_dict['scriptPath'] = get_data_from_xml(x, 'restrict', 'script', DEFAULTS_DICT['scriptPath']) result_dict['userMaxConnections'] = int( get_data_from_xml(x, 'restrict', 'user_max_connections', DEFAULTS_DICT['userMaxConnections'])) result_dict['restrictedTimePeriods'] = { 'level1': _get_timedict_from_xml(x, 'restrict', 'level1'), 'level2': _get_timedict_from_xml(x, 'restrict', 'level2'), 'level3': _get_timedict_from_xml(x, 'restrict', 'level3'), 'level4': _get_timedict_from_xml(x, 'restrict', 'level4'), 'timeout': _get_timedict_from_xml(x, 'restrict', 'timeout') } result_dict['restrictType'] = { 'mode': get_data_from_xml(x, 'restrict_mode', 'use', DEFAULTS_DICT['restrictType']['mode']), 'unlimit': time_convertor_to_dict( get_data_from_xml(x, 'restrict_mode', 'unlimit', DEFAULTS_DICT['restrictType']['unlimit'])) } result_dict['slowQueries'] = {'kill': str_to_boolean(get_data_from_xml(x, 'slow_queries', 'run', DEFAULTS_DICT['slowQueries']['kill'])), 'logPath': get_data_from_xml(x, 'slow_queries', 'log', DEFAULTS_DICT['slowQueries']['logPath']), 'timeout': get_slow_queries_timeout(x, DEFAULTS_DICT['slowQueries']['timeout']) } result_dict['restrictLog'] = { 'logPath': get_data_from_xml(x, 'restrict', 'log', DEFAULTS_DICT['restrictLog']['logPath']), 'format': get_data_from_xml(x, 'restrict', 'format', DEFAULTS_DICT['restrictLog']['format']) } result_dict['errorLog'] = {'logPath': get_data_from_xml(x, 'log', 'file', DEFAULTS_DICT['errorLog']['logPath']), 'level': get_data_from_xml(x, 'log', 'mode', DEFAULTS_DICT['errorLog']['format']) } result_dict.update(get_db_gov_stat_info()) return {'mySQLGovSettings': result_dict} except (OSError, IOError, etree.XMLSyntaxError, etree.XMLSchemaParseError, etree.XMLSchemaError, DBGovernorException) as e: raise DBGovernorException(f"{DbGovConfig} read/parse error: {e}") from e def set_db_gov_config(parameters_dict): """ Set MySQL governer config :param parameters_dict: Parameters to set dictionary. For example: { u'modeOfOperation': u'abusers', u'scriptPath': u'', u'gatherDataForDetailedStats': True, u'userMaxConnections': 30, u'logRestrictedUsersQueries': False, u'restrictLog': {u'logPath': u'/var/log/dbgovernor-restrict.log', u'format': u'LONG'}, u'restrictedTimePeriods': {u'level1': {u'unitOfTime': u'seconds', u'period': 60}, u'level2': {u'unitOfTime': u'minutes', u'period': 15}, u'level3': {u'unitOfTime': u'hours', u'period': 1}, u'level4': {u'unitOfTime': u'days', u'period': 1}, u'timeout': {u'unitOfTime': u'hours', u'period': 1}}, u'errorLog': {u'logPath': u'/var/log/dbgovernor-error-my.log', u'level': u'DEBUG'}, u'slowQueries': {u'logPath': u'/var/log/dbgovernor-kill.log', u'kill': True, u'timeout': 30}, u'restrictType': {u'mode': u'limit', u'unlimit': {u'unitOfTime': u'seconds', u'period': 60}} } Any key in dictionary can absent, appropriate MySQL Governor parameter will not changed :return: None. Throws DBGovernorException if error """ try: # Do nothing if MySQL Governor config does not exist if not os.path.exists(DbGovConfig): return # Open config and parse it f = _open(DbGovConfig) x = etree.parse(f).getroot() f.close() ################################## # 1. Process single node/attribute # Control data list struct: # (input_dictionary_keys, xml_tag_name, xml_attrubute_name) control_data_list = [ ("['modeOfOperation']", 'lve', 'use', str), # lve/@use ("['scriptPath']", 'restrict', 'script', str), # restrict/@script ("['gatherDataForDetailedStats']", 'statistic', 'mode', boolean_to_on_off), # statistic/@mode ("['userMaxConnections']", 'restrict', 'user_max_connections', str), # restrict/@user_max_connections ("['logRestrictedUsersQueries']", 'logqueries', 'use', boolean_to_on_off), # logqueries/@use ("['restrictLog']['logPath']", 'restrict', 'log', str), # restrict/@log ("['restrictLog']['format']", 'restrict', 'format', str), # restrict/@format ("['errorLog']['logPath']", 'log', 'file', str), # log/@file ("['errorLog']['level']", 'log', 'mode', str), # log/@mode ("['slowQueries']['logPath']", 'slow_queries', 'log', str), # slow_queries/@log ("['slowQueries']['kill']", 'slow_queries', 'run', boolean_to_on_off), # slow_queries/@run ("['restrictType']['mode']", 'restrict_mode', 'use', str) # restrict_mode/@use ] for control_data_item in control_data_list: try: input_dict_keys, xml_tag_name, xml_attribute_name, func_converter = control_data_item # Get input dictionary value. Generates KeyError exception if key(s) not exist val = eval(f"parameters_dict{input_dict_keys}") set_data_to_xml(x, xml_tag_name, xml_attribute_name, func_converter(val)) except KeyError: pass ################################## # 2. Process values pairs # Item structure: (period_indexes, period_unit_of_time, xml_tag_name, xml_attrubute_name) control_data_pairs_list = [ ("['restrictType']['unlimit']['period']", "['restrictType']['unlimit']['unitOfTime']", 'restrict_mode', 'unlimit'), # restrict_mode/@unlimit ("['restrictedTimePeriods']['level1']['period']", "['restrictedTimePeriods']['level1']['unitOfTime']", 'restrict', 'level1'), # restrict/@level1 ("['restrictedTimePeriods']['level2']['period']", "['restrictedTimePeriods']['level2']['unitOfTime']", 'restrict', 'level2'), # restrict_mode/@level2 ("['restrictedTimePeriods']['level3']['period']", "['restrictedTimePeriods']['level3']['unitOfTime']", 'restrict', 'level3'), # restrict_mode/@level3 ("['restrictedTimePeriods']['level4']['period']", "['restrictedTimePeriods']['level4']['unitOfTime']", 'restrict', 'level4'), # restrict_mode/@level4 ("['restrictedTimePeriods']['timeout']['period']", "['restrictedTimePeriods']['timeout']['unitOfTime']", 'restrict', 'timeout'), # restrict_mode/@timeout ] for control_data_item in control_data_pairs_list: period_indexes, period_unit_of_time_indexes, xml_tag_name, xml_attribute_name = control_data_item try: # period_indexes, period_unit_of_time_indexes, xml_tag_name, xml_attribute_name = control_data_item # Get input dictionary value. Generates KeyError exception if key(s) not exist period_val = eval(f"parameters_dict{period_indexes}") period_unit_of_time = time_unit_to_letter(eval(f"parameters_dict{period_unit_of_time_indexes}")) set_data_to_xml(x, xml_tag_name, xml_attribute_name, f"{period_val}{period_unit_of_time}") except KeyError: pass ################################## # 3. Process special values # Write slow queries timeout try: slow_queries_timeout = parameters_dict['slowQueries']['timeout'] set_slow_queries_timeout(x, slow_queries_timeout) except KeyError: pass ################################## # 4. Save result s_xml = etree.tostring(x) with open(DbGovConfig, 'wb') as f: f.write(b'<?xml version="1.0" ?>\n%s\n' % s_xml) except (OSError, IOError, etree.XMLSyntaxError, etree.XMLSchemaParseError, etree.XMLSchemaError) as e: raise DBGovernorException({'message': "%(dbgov_cfg)s error: " + str(e), 'context': {'dbgov_cfg': DbGovConfig}}) from e ################################## # 5. Restart governer # /sbin/service db_governor restart subprocess.run('/sbin/service db_governor restart &>/dev/null &', shell=True, executable='/bin/bash', check=False) def get_db_gov_stat_info(): """ Get db_governor status and version info :return: dict( 'dbGovVersion': <version>, 'dbGovStatus': enabled|error|disabled, 'dbGovError': error details ) """ db_gov = MySQLGovernor() status, error = db_gov.get_governor_status() return { 'dbGovVersion': db_gov.get_governor_version(), 'dbGovStatus': status, 'dbGovError': str(error) }