name : cloudlinux_top.py # coding=utf-8 # # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT import json import logging import time from typing import Callable # NOQA import lvestats.lib.commons.decorators from clcommon import FormattedException, cpapi from clcommon.cpapi import NotSupported from clcommon.cpapi.pluginlib import getuser from clcommon.lib import GovernorStatus, MySQLGovernor, MySQLGovException from lvestats.lib import dbengine from lvestats.lib.commons import func, server_status from lvestats.lib.commons.func import get_users_for_reseller, skip_user_by_maxuid from lvestats.lib.commons.logsetup import setup_logging from lvestats.lib.commons.users_manager import g_users_manager from lvestats.lib.config import HIDE_MAX_UID_LVE_PARAMETER, str_to_bool from lvestats.lib.parsers.cloudlinux_top_argparse import cloudlinux_top_parser from lvestats.lib.uidconverter import uid_to_username NOT_AVAILABLE = "N/A" class CloudLinuxTop(object): def __init__(self, cnf, engine=None): self._log = setup_logging( config=cnf, caller_name="CloudLinuxTop", file_level=logging.WARNING, console_level=logging.FATAL) self.fname = '/var/lve/cloudlinux_top.json' self.fname_db = '/var/lve/cloudlinux_dbtop.json' self.touch_fname = '/var/lve/governor.ts' self.server_id = cnf.get('server_id', 'localhost') self.hide_maxuid_lve = str_to_bool(cnf.get(HIDE_MAX_UID_LVE_PARAMETER, 'true')) self.engine_error = None if engine is not None: self.engine = engine else: try: self.engine = dbengine.make_db_engine(cnf) except dbengine.MakeDbException as e: self.engine_error = e self.mysql_governor = MySQLGovernor() self.governor_status, self.governor_error = \ self.mysql_governor.get_governor_status() self.governor_mode = func.get_governor_mode(self.mysql_governor) self._admins = self._get_admins() if getuser() == 'root' else set() @staticmethod def _get_admins(): try: return cpapi.admins() except (NotSupported, AttributeError): return set() def _get_username_by_id(self, uid): return uid_to_username( uid=uid, local_server_id=self.server_id, server_id=self.server_id, db_engine=self.engine ) def _get_order_key(self, order, cl_data): # type: (str, dict) -> Callable[dict] """returns key function for sorted""" def mysql_key(value): def key(k): username = self._get_username_by_id(k["id"]) if username not in cl_data["dbgov_data"]: return 0 else: dbgov_stats = cl_data["dbgov_data"][username] return dbgov_stats[value] return key order_key_map = { "cpu": lambda k: k["usage"]["cpu"]["all"], "io": lambda k: k["usage"]["io"]["all"], "mysql_cpu": mysql_key("cpu"), "mysql_io": mysql_key("io"), "iops": lambda k: k["usage"].get("iops", 0), "ep": lambda k: k["usage"]["ep"], "nproc": lambda k: k["usage"].get("pno", 0), "pmem": lambda k: k["usage"].get("mem", 0), } return order_key_map[order] def _load_from_json_file(self, fname, default=None): """ Try load and parse json data from file; return <default> if loading or parsing wrong :param fname: path to parsed file :param None|dict default: default data if loaded file is wrong :return None| dict: return loaded data """ try: with open(fname, encoding='utf-8') as json_file: cl_data = json.load(json_file) except (IOError, OSError): cl_data = default return cl_data def _touch_dbtop(self): """ Trying touch the file to signal new data """ try: func.touch(self.touch_fname) except (IOError, OSError): self._log.error("An error occurred while touching file %s. " "Database statistics may be not available", self.touch_fname) def _load_data(self): """ Load and join data from separate files :return dict: loaded data """ cl_data = self._load_from_json_file(self.fname, {"users": [], "resellers": []}) cl_data.update(self._load_from_json_file(self.fname_db, {"dbgov_data": {}})) return cl_data def _get_data(self, username_filter=None, domain_filter=None, limit=None, show_mysql=True, order="cpu", reseller=None): """ :type order: str :type show_mysql: bool :type limit: int|None :type domain_filter: str|None :type username_filter: str|None :param None|str reseller: get information about given reseller :rtype: tuple[list, list] """ users_result = [] resellers_result = [] users_list = [] current_user_name = getuser() current_user_is_admin = current_user_name == 'root' if current_user_is_admin or reseller: self._touch_dbtop() # touch file as signal to save dbgov data to file g_users_manager.build_users_cache(reseller) if reseller: users_list = get_users_for_reseller(reseller or getuser()) # show reseller information too else: users_list = [current_user_name] cl_data = self._load_data() cl_users_data_ordered = sorted(cl_data["users"], key=self._get_order_key(order, cl_data), reverse=True) for user in cl_users_data_ordered: # skip users with maxuid lve ids due to some client`s complaints (details: LVES-929) if self.hide_maxuid_lve and skip_user_by_maxuid(user['id']): continue username = self._get_username_by_id(user["id"]) or NOT_AVAILABLE # users must not be skipped if current user is root and when --for-reseller is not passed should_skip_users = not current_user_is_admin or reseller is not None if should_skip_users and username not in users_list or username in self._admins: continue if username == NOT_AVAILABLE: user["domain"] = NOT_AVAILABLE user["reseller"] = NOT_AVAILABLE else: user["reseller"] = g_users_manager.get_reseller(username, raise_exc=False) or NOT_AVAILABLE user["domain"] = g_users_manager.get_domain(username, raise_exc=bool(domain_filter)) or NOT_AVAILABLE if (domain_filter and (user["domain"] == NOT_AVAILABLE or domain_filter not in user["domain"])) or \ (username_filter and (username == NOT_AVAILABLE or username_filter not in username)): continue user["username"] = username if show_mysql and self.governor_status == GovernorStatus.ENABLED: self._add_mysql_data(user, cl_data, username) users_result.append(user) if limit is not None and 0 < limit <= len(users_result): break cl_resellers_data_ordered = \ sorted(cl_data.get("resellers", []), key=self._get_order_key(order, cl_data), reverse=True) if current_user_is_admin: for reseller_info in cl_resellers_data_ordered: username = self._get_username_by_id(reseller_info['id']) # get only reseller that we need (if needed) if reseller and username != (reseller or getuser()): continue if username is None: username = NOT_AVAILABLE reseller_info["name"] = username resellers_result.append(reseller_info) return users_result, resellers_result def _add_mysql_data(self, user, cl_data, username): """ add mysql statistics to the user dict :type cl_data: dict :type user: dict :type username: str """ dbgov_stats = cl_data["dbgov_data"].get( username, {"cpu": 0, "io": 0, "cause_of_restrict": "-", "time_of_restrict": 0}, ) if self.governor_mode != "all": try: mysql_limits = self.mysql_governor.get_limits_by_user(username) except MySQLGovException as e: self.governor_status = GovernorStatus.ERROR self.governor_error = e else: user["limit"]["cpu"]["mysql"] = mysql_limits[0] user["limit"]["io"]["mysql"] = mysql_limits[1] * 1024 # in bytes user["usage"]["cpu"]["mysql"] = dbgov_stats["cpu"] user["usage"]["io"]["mysql"] = dbgov_stats["io"] else: user["limit"]["cpu"]["mysql"] = "-" user["limit"]["io"]["mysql"] = "-" user["usage"]["cpu"]["mysql"] = min(dbgov_stats["cpu"], user["usage"]["cpu"]["all"]) user["usage"]["io"]["mysql"] = min(dbgov_stats["io"], user["usage"]["io"]["all"]) if dbgov_stats["cause_of_restrict"] != "-": user["restricted"] = True user["restriction"] = { "time": dbgov_stats["time_of_restrict"], "reason": f"Violated the {dbgov_stats['cause_of_restrict']} limit"} else: user["restricted"] = False @staticmethod def to_json(value): """ :type value: dict :rtype: str """ return json.dumps(value, indent=4, sort_keys=True) @lvestats.lib.commons.decorators.no_sigpipe def main(self, *args): """ :rtype: (str, int) """ parser = cloudlinux_top_parser() result = {"timestamp": time.time()} try: namespace = parser.parse_args(args) if not namespace.json: return "Use --json option, other modes currently unsupported", 1 except ValueError as e: result["result"] = e.args[0] result["context"] = {} return self.to_json(result), 0 if getuser() != 'root' and namespace.for_reseller is not None and namespace.for_reseller != getuser(): error_msg = 'Permission denied. Reseller can view ' \ 'information only about himself. ' \ f'Option --for-reseller={namespace.for_reseller} is forbidden.' result['result'] = error_msg result["context"] = {} return self.to_json(result), 1 lvestats_status = server_status.check_server_status() if lvestats_status != server_status.RUNNING: result["result"] = server_status.status_description[lvestats_status] elif self.engine_error is not None: result["result"] = self.engine_error.message result["context"] = self.engine_error.context else: try: users, resellers = self._get_data( username_filter=namespace.username, domain_filter=namespace.domain, limit=namespace.max, show_mysql=not namespace.hide_mysql, order=namespace.order_by, reseller=namespace.for_reseller) except IOError as e: result["result"] = str(e) result["context"] = {} except FormattedException as e: result["result"] = e.message result["context"] = e.context else: result.update({ "result": "success", "users": users, "resellers": resellers, "mySqlGov": self.governor_status, "mySqlGovMode": self.governor_mode}) # broken governor is a non-fatal problem # and lve statistics should works even in that case # so let's threat "error" status as "warning" # and show "warning" message in web ui # anyway, at the moment lvemanager ignores "errors" field if self.governor_status == GovernorStatus.ERROR: result["warning"] = self.governor_error.message try: result["context"] = self.governor_error.context except AttributeError: result["context"] = {} return self.to_json(result), 0