????
Current Path : /proc/self/root/proc/self/root/lib/Acronis/PyShell/site-tools/ |
Current File : //proc/self/root/proc/self/root/lib/Acronis/PyShell/site-tools/cep.py |
from datetime import datetime, timedelta from urllib import parse import acrort import argparse import itertools import json import pprint import prettytable import requests def fmt_sizeof(num, suffix='B'): for unit in ['','K','M','G','T','P','E','Z']: if abs(num) < 1024.0: return "%3.1f%s%s" % (num, unit, suffix) num /= 1024.0 return "%.1f%s%s" % (num, 'Y', suffix) class PrettyTable(prettytable.PrettyTable): PrettyFormat = 'default' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def __repr__(self): return self.__str__() def __str__(self): if self.PrettyFormat == 'json': return self.get_json_string() else: return self.get_string() def get_json_string(self, **kwargs): options = self._get_options(kwargs) result = [self._field_names] for row in self._get_rows(options): result.append(row) return json.dumps(result) class DistributionTable: def build(self, map, name): t = PrettyTable([name]+["VALUE", '%%']) total = sum([v for _, v in map.items()]) for k, v in map.items(): t.add_row([k, v, '{:1.1f}'.format(v / total * 100)]) t.align[name] = 'l' t.align["VALUE"] = 'r' t.align['%%'] = 'r' return t class MachineReport: def build_ams_pretty(self, cn): rep = self.build_ams_report(cn) t = PrettyTable(["ACRONIS MANAGEMENT SERVER", "VALUE"]) t.add_row(["Report date", str(datetime.now().date())]) t.add_row(["Version", rep['version']]) t.add_row(["CPU model", rep['cpu']]) t.add_row(["RAM", rep['memory']]) t.add_row(["OS", rep['os']]) t.align["ACRONIS MANAGEMENT SERVER"] = 'l' t.align["VALUE"] = 'r' return t def build_ams_report(self, cn): pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'MachineManagement::Machine'), ('.Info.Role', 'dword', 1) ]) resp = cn.dml.select(acrort.dml.ViewSpec(pt)) assert len(resp) == 1, "MachineManagement::Machine role=1 exists" return self.parse_machine(resp[0]) def build_agent_pretty(self, cn): rep = self.build_agent_report(cn) tt = [] tt.append(DistributionTable().build(rep['agent_version'], "AGENT VERSIONS")) tt.append(DistributionTable().build(rep['cpu'], "AGENT CPUs")) tt.append(DistributionTable().build(rep['os'], "AGENT OS")) tt.append(DistributionTable().build(rep['memory'], "AGENT RAM")) return tt def build_agent_report(self, cn): reader = DmlReader(cn) pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'MachineManagement::Machine'), ('.Info.Role', 'dword', 0) ]) opts = acrort.plain.Unit(flat=[ ('.Mask.Agents', 'nil', ''), ('.Mask.Info', 'nil', '') ]) stat = [self.parse_machine(m) for m in reader.read(pt, opts)] return { 'os' : self.count_values([m['os'] for m in stat]), 'cpu' : self.count_values([m['cpu'] for m in stat]), 'memory' : self.count_values([m['memory'] for m in stat]), 'agent_version' : self.count_values([m['version'] for m in stat]), } def count_values(self, values): stat = {} for v in values: stat[v] = stat.get(v, 0) + 1 return stat def parse_machine(self, m): return { 'version' : [v.Version.ref for _, v in m.Info.Agents if v.Id.ref == ''][0], 'cpu_freq' : m.Info.Hardware.ProcessorFrequency.ref, 'cpu' : m.Info.Hardware.ProcessorName.ref, 'memory' : fmt_sizeof(m.Info.Hardware.MemorySize.ref), 'os' : m.Info.OS.Name.ref, } class ApiGwReader: def __init__(self, url, user, pswd): self.url = url self.user = user self.pswd = pswd self.token = self.get_token(url, user, pswd) def get_token(self, url, user, pswd): resp = requests.post( url=url+"/idp/token", data={ 'grant_type' : 'password', 'username' : user, 'password' : pswd, }, ) assert resp.status_code == 200, "Get token success" return resp.json()['access_token'] class ActivityByDateReport: def build_pretty(self, token, url, days_limit): report = self.build(token, url, days_limit) t = PrettyTable(["ACTIVITY BY DATE", "TOTAL", "SUCCESS", "FAILED"]) fmt = lambda x : '-' if x == 0 else x for r in report: t.add_row([str(r[0]), fmt(r[1]+r[2]), fmt(r[1]), fmt(r[2])]) return t def build(self, token, url, days_limit): now = datetime.now() round4 = lambda x : x.replace(hour=x.hour // 4 * 4, minute=0, second=0, microsecond=0) top = round4(now) bottom = top - timedelta(days=days_limit) buckets = {} for a in self.activity_reader(url, token=token): time, status = self.strptime(a['finishTime']), a['status'] if time < bottom: break b = buckets.get(round4(time), [0, 0, []]) b[2].append(time) if status in ['ok', 'warning']: b[0] += 1 else: b[1] += 1 buckets[round4(time)] = b res = [] while top >= bottom: b = buckets.get(top, [0, 0, []]) res.append((top, b[0], b[1], b[2])) top -= timedelta(hours=4) return sorted(res, key=lambda x: -x[0].timestamp()) def activity_reader(self, tm, token): usn = None limit = 100 while True: url = tm+"/api/task_manager/x/activities?state=completed&order=usn.desc&limit={}".format(limit) if usn: url = url + "&usn_ls={}".format(usn) resp = requests.get( url=url, headers={ 'Authorization': 'Bearer ' + token, } ) assert resp.status_code == 200, "Get task successful" activities = resp.json() for a in activities: if usn: assert a['usn'] < usn, "usn is decreasing" usn = a['usn'] yield a if len(activities) < limit: break def strptime(self, t): return datetime.strptime(t, '%Y-%m-%dT%H:%M:%SZ') class DmlReader: def __init__(self, cn): self.cn = cn def read(self, pt, opts): limit = 100 if not opts: opts = acrort.plain.Unit(flat=[('.LimitOptions', 'dword', limit)]) opts = opts.consolidate(acrort.plain.Unit(flat=[('.LimitOptions', 'dword', limit)])) if opts.get_branch('.Mask', None): opts = opts.consolidate(acrort.plain.Unit(flat=[('.Mask.DmlTimeStamp', 'nil', '')])) stamp = 0 while True: spec = acrort.dml.ViewSpec(pt.consolidate(self.stamp_greater(stamp)), opts.consolidate(self.sort_stamp_asc())) stampmax = stamp for obj in self.cn.dml.select(spec): yield obj stampmax = obj.DmlTimeStamp.ref if stampmax - stamp < limit: break stamp = stampmax def sort_stamp_asc(self): return acrort.plain.Unit(flat=[ ('.SortingOptions.DmlTimeStamp', 'sqword', 0)]) def stamp_greater(self, ts): return acrort.plain.Unit(flat=[('.DmlTimeStamp', 'sqword', 0), ('.DmlTimeStamp^Greater', 'sqword', ts)]) class BackupPlanScheduleReport: def __init__(self): pass def build_pretty(self, cn): rep = self.build(cn) tt = [] tt.append(DistributionTable().build(rep['schemes'], "BACKUP PLAN SCHEMES")) dow = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"] str_hour = lambda x : '{:02}-{:02}'.format(x, x+2) dow_stat = { dow[k] : v for k, v in rep['days'].items() } tt.append(DistributionTable().build(dow_stat, "DAY OF WEEK")) dow_time_stat = PrettyTable(["DOW AND TIME", "COUNT"]) for i in [1, 2, 3, 4, 5, 6, 0]: for j in range(12): v = rep['schedules'].get((i, j * 2), 0) dow_time_stat.add_row(['{}, {}'.format(dow[i], str_hour(j * 2)), v]) dow_time_stat.align["DOW AND TIME"] = 'l' dow_time_stat.align["COUNT"] = 'r' tt.append(dow_time_stat) return tt def build(self, cn): schedules = {} schemes = {} names = {} days_stat = {} for name, scheme, days, start in self.parse_schedules(cn): #print(name, scheme, days, start) if not scheme: schemes['others'] = schemes.get('others', 0) + 1 continue schemes[scheme] = schemes.get(scheme, 0) + 1 if not days: continue for d in days: buck_id = d, start[0] // 2 * 2 schedules[buck_id] = schedules.get(buck_id, 0) + 1 names[buck_id] = names.get(buck_id, []) + [name] days_stat[d] = days_stat.get(d, 0) + 1 return { 'schemes' : schemes, 'schedules' : schedules, 'days' : days_stat, } def parse_schedules(self, cn): reader = DmlReader(cn) pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'Gtob::Dto::ProtectionPlan'), ]) opts = acrort.plain.Unit(flat=[ ('.Mask.Scheme', 'nil', ''), ('.Mask.Name', 'nil', ''), ('.Mask.Enabled', 'nil', ''), ]) days_of_week = [0] * 7 for obj in reader.read(pt, opts): if obj.Scheme.Type.ref == 5: for _, item in obj.Scheme.Parameters.Items: for days, startAt in self.process_schedule(item.Schedule): yield obj.Name.ref, 'custom', days, startAt continue if obj.Scheme.Type.ref == 9: yield obj.Name.ref, 'replication_once', None, None continue if obj.Scheme.Type.ref in [10, 20, 21, 22, 23]: names = { 10 : 'replication_simple', 20 : 'always_full', 21 : 'always_incr', 22 : 'full_daily_incr', 23 : 'mwd', } for days, startAt in self.process_schedule(obj.Scheme.Parameters.BackupSchedule.Schedule): yield obj.Name.ref, names[obj.Scheme.Type.ref], days, startAt continue yield obj.Name.ref, None, None, None def from_array(self, ar): return [x.ref for _, x in ar] def from_mask_days(self, m): forb = [i for i, ch in enumerate(bin(m)[2:][::-1]) if ch == '1'] return [i for i in range(7) if i not in forb] def process_schedule(self, sch): assert 'ScheduleManagement::Schedule' in [v for n, v in sch.traits if n == 'Is'] alarms = [al for _, al in sch.Alarms if al.Alarm.polyType.ref == 1] if not alarms: return for al in alarms: impl = al.Alarm.impl if impl.Calendar.Calendar.polyType.ref != 2: continue days = self.from_mask_days(impl.Calendar.Calendar.impl.Days.ref) startAt = impl.StartTime.Hour.ref, impl.StartTime.Minute.ref if impl.RepeatAtDay.TimeInterval.ref > 0: step = impl.RepeatAtDay.TimeInterval.ref // 60 endAt = (impl.RepeatAtDay.EndTime.Hour.ref, impl.RepeatAtDay.EndTime.Minute.ref) while startAt <= endAt: yield days, startAt min = startAt[1] + step startAt = (startAt[0] + min // 60, min % 60) continue yield days, startAt class AgentOnlineReport: def build_pretty(self, cn): rep = self.build(cn) return DistributionTable().build(rep, "AGENT AVAILABILITY") def build(self, cn): reader = DmlReader(cn) pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'MachineManagement::Machine'), ('.Info.Role', 'dword', 0), ]) opts = acrort.plain.Unit(flat=[ ('.Mask.Status', 'nil', ''), ]) status = {} for m in reader.read(pt, opts): st = m.Status.ref status[st] = status.get(st, 0) + 1 return { { 1 : 'offline', 0 : 'online' }[k] : v for k, v in status.items() } class BackupPlanDataTypeReport: def build_pretty(self, cn): rep = self.build(cn) return DistributionTable().build(rep, "BACKUP PLAN DATA TYPES") def build(self, cn): mm = { 'ams::instances::physical_instance' : 'Machines/Disks/Volumes', 'ams::instances::virtual_instance' : 'VMs', 'ams::resources::group' : 'Groups', 'arx::ams::gct::mailbox' : 'MS Exchange mailboxes', 'mms::disk::disk' : 'Machines/Disks/Volumes', 'mms::disk::volume' : 'Machines/Disks/Volumes', 'mms::file::dir' : 'Files/Folders', 'mms::file::file' : 'Files/Folders', 'mms::smb::dir' : 'Files/Folders', } stat = {} for t, host, resource in self.parse_inclusions(cn): key = mm[t] counter = stat.get(key, 0) counter += 1 stat[key] = counter return stat def parse_inclusions(self, cn): reader = DmlReader(cn) pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'Gtob::Dto::ProtectionPlan'), ]) opts = acrort.plain.Unit(flat=[ ('.Mask.Target', 'nil', ''), ('.Mask.Name', 'nil', ''), ]) for p in reader.read(pt, opts): for _, item in p.Target.Inclusions: it = item.Key.ItemType.ref id = item.Key.LocalID.ref if it in ['ams::instances::virtual_instance', 'ams::instances::physical_instance']: id = id.split('@') yield it, id[1], id[0] continue if it == 'ams::resources::group': yield it, None, id continue if it == 'arx::ams::gct::mailbox': url = parse.unquote(id.split("ArxUri=")[1]) pp = parse.urlsplit(url) assert pp.scheme == 'arx' instance, host = pp.path[1:].split('@') yield it, host, instance continue if not item.get_branch('.Key.0B781614-5AED-4A10-9B79-0A607CB7EEAE', None) is None: yield it, item.get_branch('.Key.0B781614-5AED-4A10-9B79-0A607CB7EEAE').ref, None continue raise Exception("Please add support of new type here") class ProtectedResourcesReport: def build_pretty(self, cn): rep = self.build(cn) tt = [] tt.append(DistributionTable().build(rep['virtual'], "PROTECTED VIRTUAL MACHINES")) tt.append(DistributionTable().build(rep['physical'], "PROTECTED PHYSICAL MACHINES")) tt.append(DistributionTable().build(rep['mailbox'], "PROTECTED MAILBOXES")) tt.append(DistributionTable().build(rep['db'], "PROTECTED DATABASE SERVERS")) tt.append(DistributionTable().build(rep['exchange'], "PROTECTED EXCHANGE SERVERS")) pr = PrettyTable(["PROTECTED RESOURCES", "COUNT"]) nm = { 'ams::instances::physical_instance::gct::disks' : 'Physical Machines', 'arx::ams::gct::mailbox' : 'Mailbox', 'ams::instances::virtual_instance' : 'Virtual Machines', 'ams::instances::physical_instance::gct::files' : 'Files/Folders', 'ams::instances::sql_server' : 'Database Servers', 'arx::ams::gct::exchange_instance' : 'MS Exchange Servers', } mapped = { nm[k] : v for k, v in rep['protected_resources'].items() } for k, v in mapped.items(): pr.add_row([k, v]) pr.align["PROTECTED RESOURCES"] = 'l' pr.align["COUNT"] = 'r' tt.append(pr) return tt def build(self, cn): stat = {} protected = {} for _, typ, os, gtob in self.protected_instance_reader(cn): protected[gtob] = protected.get(gtob, 0) + 1 subgroup = stat.get(typ, {}) subgroup[os] = subgroup.get(os, 0) + 1 stat[typ] = subgroup return { 'protected_resources' : protected, 'physical' : stat.get('physical', {}), 'virtual' : stat.get('virtual', {}), 'mailbox' : stat.get('mailbox', {}), 'db' : stat.get('db', {}), 'exchange' : stat.get('exchange', {}), } def protected_instance_reader(self, cn): for group in self.splitter(100, self.enum_protected_instances(cn)): ids = list(set([id for id, _ in group])) ii = {} for i in self.get_instance_os_type(cn, ids): id = str(i.ID.ref) if i.Type.ref == 1: ii[id] = 'physical', i.Parameters.OperatingSystem[0].ref continue if i.Type.ref in [4, 5]: os = i.Parameters.OperatingSystem[0].ref if not os and i.Parameters.Type[0].ref == 'mshyperv': os = '!HyperV!' ii[id] = 'virtual', os continue if i.Type.ref == 24: ii[id] = 'mailbox', i.Parameters.OperatingSystem[0].ref continue if i.Type.ref == 2: ii[id] = 'db', i.Parameters.OperatingSystem[0].ref continue if i.Type.ref == 6: ii[id] = 'exchange', i.Parameters.OperatingSystem[0].ref continue for id, type in group: found = ii.get(id, None) if found is None: continue yield id, found[0], found[1], type def splitter(self, size, ids): group = [] for x in ids: group.append(x) if len(group) == size: yield group group = [] if group: yield group def get_instance_os_type(self, cn, ids): por_value_in = [acrort.plain.Unit(flat=[('', 'guid', id)]) for id in ids] pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'InstanceManagement::Instance'), ('.ID', 'guid', str(acrort.common.Guid())), ('.ID^ValueIn', 'array', por_value_in), #('.ID', 'guid', 'C9743CCD-6FAB-4CD8-ADEA-A3B6DED1E375') ]) opts = acrort.plain.Unit(flat=[ ('.Mask.Parameters.OperatingSystem', 'nil', ''), ('.Mask.Parameters.Type', 'nil', ''), ('.Mask.Type', 'nil', ''), ]) ii = [i for i in cn.dml.select(acrort.dml.ViewSpec(pt, opts))] return ii def enum_protected_instances(self, cn): reader = DmlReader(cn) pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'Gtob::Dto::ItemProtection') ]) opts = acrort.plain.Unit(flat=[ ('.Mask.InstanceID', 'nil', ''), ('.Mask.Centralized.Subject', 'nil', ''), ]) for ip in reader.read(pt, opts): it = ip.Centralized.Subject.ItemType.ref if it == 'ams::instances::physical_instance': proj = ip.get_branch('.Centralized.Subject.4B2A7A93-A44F-4155-BDE3-A023C57C9431', '') it += '::' + proj.ref yield str(ip.InstanceID.ref), it class ConsumedStorageReport: def build_pretty(self, cn): rep = self.build(cn) t = PrettyTable(["CONSUMED STORAGE", "PHYSICAL SIZE", "LOGICAL SIZE"]) for k, v in rep.items(): t.add_row([k, v, 'N/A']) t.align["CONSUMED STORAGE"] = 'l' t.align["PHYSICAL SIZE"] = 'r' return t def build(self, cn): reader = DmlReader(cn) pt = acrort.plain.Unit(flat=[ ('^Is', 'string', 'DMS::BackupLocation'), ]) opts = acrort.plain.Unit(flat=[ ('.Mask.Info.DisplayName', 'nil', ''), ('.Mask.OccupiedSpace', 'nil', ''), ('.Mask.TotalSpace', 'nil', ''), ('.Mask.Info.Kind.LocationKind', 'nil', ''), ]) stats = {} for x in reader.read(pt, opts): kind = x.Info.Kind.LocationKind.ref sum = stats.get(kind, 0) occup = x.get_branch('.OccupiedSpace', None) if not occup is None: sum += occup.ref stats[kind] = sum kinds = { 1 : 'Local folder', 2 : 'Network share', 3 : 'FTP Location', 4 : 'SFTP Location', 5 : 'CD Location', 6 : 'Tape Location', 7 : 'Acronis Storage Node', 8 : 'Acronis Secure Zone', 9 : 'Removable drive', 10 : 'Cloud Storage', 11 : 'NFS Location', 12 : 'ESX Location' } return { kinds[k] : fmt_sizeof(v) for k, v in stats.items() } class TenantsHierarchyReport: def __init__(self, token, url): self._url = url self._token = token def build_pretty(self): rep = self.build() tt = [] pt1 = PrettyTable(["TENANTS HIERARCHY", "COUNT"]) pt1.add_row(['Organizations', rep['customer']]) pt1.add_row(['Units', rep['unit']]) pt1.add_row(['Folders', rep['folder']]) pt1.align["TENANTS HIERARCHY"] = 'l' pt1.align["COUNT"] = 'r' tt.append(pt1) pt2 = PrettyTable(["UNITS PER ORGANIZATION", "COUNT"]) pt2.align["UNITS PER ORGANIZATION"] = 'l' pt2.align["COUNT"] = 'r' tt.append(pt2) return tt def build(self): headers = { 'Authorization': 'Bearer ' + self._token } resp = requests.get(self._url + '/api/2/users/me', headers=headers) assert resp.status_code == 200, "Get users/me successful" tenant_id = resp.json()['tenant_id'] resp = requests.get(self._url + '/api/2/tenants/%s/children' % tenant_id, headers=headers) assert resp.status_code == 200, "Get tenants/children successful" tenants_ids = resp.json()['items'] tenants = { 'customer': 0, 'partner': 0, 'unit': 0, 'folder': 0, } for uuids in [tenants_ids[i:i+1000] for i in range(0, len(tenants_ids), 1000)]: resp = requests.get(self._url + '/api/2/tenants?uuids=' + ','.join(uuids), headers=headers) assert resp.status_code == 200, "Get tenants data successful" for item in resp.json()['items']: kind = item['kind'] tenants[kind] = tenants[kind] + 1 return tenants def main(): parser = argparse.ArgumentParser(description='CEP report') parser.add_argument('--activity', nargs=1, help='Activity by date report', dest='activity') parser.add_argument('--agents', action='store_true', help='Agent machine report', dest='agents') parser.add_argument('--ams', action='store_true', help='Ams machine report', dest='ams') parser.add_argument('--consumed', action='store_true', help='Plan report', dest='consumed') parser.add_argument('--online', action='store_true', help='Ams machine report', dest='online') parser.add_argument('--plans', action='store_true', help='Plan report', dest='plans') parser.add_argument('--protected', action='store_true', help='Plan report', dest='protected') parser.add_argument('--remote', nargs=3, help='Specify computer user pass', dest ='remote', metavar=('computer','user','pass'), required=True) parser.add_argument('--schedule', action='store_true', help='Schedule report', dest='schedule') parser.add_argument('--hierarchy', action='store_true', help='Tenants hierarchy report', dest='hierarchy') parser.add_argument('--all', action='store_true', help='All report', dest='all') parser.add_argument('--fast', action='store_true', help='Fast reports only', dest='fast') parser.add_argument('--json', action='store_true', help='JSON output format', dest='json') args = parser.parse_args() result = [] if args.json: PrettyTable.PrettyFormat = 'json' if args.all: args.activity = [1] args.agents = args.ams = args.consumed = args.online = args.plans = args.protected = True args.schedule = args.hierarchy = True if args.fast: args.agents = args.ams = args.consumed = args.online = args.plans = args.protected = True args.schedule = True def connector(): return acrort.connectivity.Connection(service='ams', computer=args.remote[0], cred=(args.remote[1], args.remote[2]), client_session_data={'identity_disabled':True}) if args.activity or args.hierarchy: apigw = ApiGwReader('http://' + args.remote[0]+':9877', args.remote[1], args.remote[2]) if args.ams: result.append(MachineReport().build_ams_pretty(connector())) if args.activity: period = int(args.activity[0]) result.append(ActivityByDateReport().build_pretty(apigw.token, url='http://' + args.remote[0] + ':9877', days_limit=period)) if args.schedule: for t in BackupPlanScheduleReport().build_pretty(connector()): result.append(t) if args.online: result.append(AgentOnlineReport().build_pretty(connector())) if args.agents: for t in MachineReport().build_agent_pretty(connector()): result.append(t) if args.plans: result.append(BackupPlanDataTypeReport().build_pretty(connector())) if args.protected: for t in ProtectedResourcesReport().build_pretty(connector()): result.append(t) if args.consumed: result.append(ConsumedStorageReport().build_pretty(connector())) if args.hierarchy: for t in TenantsHierarchyReport(apigw.token, 'http://' + args.remote[0]+':30678').build_pretty(): result.append(t) if args.json: print(result) else: for r in result: print(r) if __name__ == '__main__': main()