updates

server/pegasus-health-json

@@ -49,6 +49,25 @@ def output():
         match = re.match(r'^\[(OK\s*|WARN|FAIL)\]\s+(.+?)(?:\s+—\s+(.*))?$', line)
         if match:
             checks.append({'name': match[2].strip(), 'status': match[1].strip().lower(), 'detail': (match[3] or '').strip(), 'section': section})
+    # Noncritical firmware metadata failures are warnings.
+    # Other failed systemd units remain critical.
+    for item in checks:
+        if item['name'] != 'Failed systemd units':
+            continue
+        if item['status'] != 'fail':
+            continue
+
+        failed_units = re.findall(
+            r'([A-Za-z0-9@_.:-]+\\.service)\\s+loaded\\s+failed\\s+failed',
+            item['detail']
+        )
+
+        if failed_units and all(
+            unit == 'fwupd-refresh.service'
+            for unit in failed_units
+        ):
+            item['status'] = 'warn'
+
     lookup = {c['name']: c['status'] for c in checks}
     groups = []
     for title, names in GROUPS:
@@ -62,7 +81,12 @@ def output():
         groups.append({'name':label, 'status':state, 'missing':[n for n in relevant if n not in lookup]})
     counts_match = re.search(r'SUMMARY:\s*(\d+) OK\s*\|\s*(\d+) WARN\s*\|\s*(\d+) FAIL', plain)
     counts = dict(zip(('ok','warn','fail'), (int(x) for x in counts_match.groups()))) if counts_match else {'ok':0,'warn':0,'fail':1}
-    if not checks:
+    if checks:
+        counts = {
+            state: sum(c['status'] == state for c in checks)
+            for state in ('ok', 'warn', 'fail')
+        }
+    else:
         counts['fail'] = max(1, counts['fail'])
     overall = 'fail' if counts['fail'] or any(g['status']=='fail' for g in groups) else 'warn' if counts['warn'] or any(g['status']=='warn' for g in groups) else 'ok'
     print(json.dumps({'schema':1, 'host':'pegasus', 'uptimeSeconds':system_uptime_seconds(), 'generatedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(), 'overall':overall, 'counts':counts,'groups':groups,'checks':checks, 'diagnostics':plain[-24000:], 'commandExitCode':p.returncode}, ensure_ascii=False))

server/remote-health-json

@@ -62,7 +62,26 @@ for directory in ('/', '/backup'):
 systemctl = run(['systemctl', '--failed', '--no-legend', '--plain'])
 if systemctl is not None:
     failed = [line for line in systemctl.stdout.splitlines() if line.strip()]
-    check('Services', 'Failed systemd units', 'fail' if failed else 'ok',
+    # Firmware metadata refresh is noncritical.
+    # Any other failed systemd unit remains critical.
+    optional_units = {'fwupd-refresh.service'}
+
+    failed_names = [
+        line.split()[0].lstrip('●')
+        for line in failed
+        if line.split()
+    ]
+
+    if not failed:
+        severity = 'ok'
+    elif failed_names and all(
+        unit in optional_units for unit in failed_names
+    ):
+        severity = 'warn'
+    else:
+        severity = 'fail'
+
+    check('Services', 'Failed systemd units', severity,
           ', '.join(failed[:5]) if failed else 'None')
 
 if shutil.which('docker'):

server/remote-health-json-legacy

@@ -0,0 +1,311 @@
+#!/usr/bin/env python
+import os
+import sys
+import time
+import socket
+import commands
+
+def command(args):
+    try:
+        quoted = []
+        for arg in args:
+            value = str(arg)
+            quoted.append("'" + value.replace("'", "'\\''") + "'")
+
+        cmd = " ".join(quoted)
+        return commands.getoutput(cmd).strip()
+    except:
+        return ""
+
+def escape(value):
+    value = str(value)
+    return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r")
+
+def string(value):
+    return '"' + escape(value) + '"'
+
+def status(name, state, detail):
+    return '{"name":%s,"state":%s,"detail":%s}' % (
+        string(name), string(state), string(detail)
+    )
+
+def check(section, name, state, detail):
+    return '{"section":%s,"name":%s,"status":%s,"detail":%s}' % (
+        string(section), string(name), string(state), string(detail)
+    )
+
+def group(name, state):
+    return '{"name":%s,"status":%s,"missing":[]}' % (
+        string(name), string(state)
+    )
+
+def read_file(path):
+    try:
+        f = open(path)
+        data = f.read()
+        f.close()
+        return data
+    except:
+        return ""
+
+def process_running(name):
+    output = command(["ps", "ax"])
+    return name in output
+
+hostname = socket.gethostname()
+
+# CPU
+try:
+    load = float(read_file("/proc/loadavg").split()[0])
+except:
+    load = 0.0
+
+try:
+    cpu_count = len([
+        x for x in read_file("/proc/cpuinfo").splitlines()
+        if x.startswith("processor")
+    ])
+    if cpu_count < 1:
+        cpu_count = 1
+except:
+    cpu_count = 1
+
+cpu_state = "ok"
+if load > cpu_count * 2:
+    cpu_state = "fail"
+elif load > cpu_count:
+    cpu_state = "warn"
+
+# Memory
+mem = {}
+for line in read_file("/proc/meminfo").splitlines():
+    parts = line.replace(":", "").split()
+    if len(parts) >= 2:
+        try:
+            mem[parts[0]] = int(parts[1])
+        except:
+            pass
+
+total = mem.get("MemTotal", 1)
+free = (
+    mem.get("MemFree", 0) +
+    mem.get("Buffers", 0) +
+    mem.get("Cached", 0)
+)
+
+memory_percent = int(100 * (total - free) / total)
+
+memory_state = "ok"
+if memory_percent >= 95:
+    memory_state = "fail"
+elif memory_percent >= 85:
+    memory_state = "warn"
+
+# Disk
+disk_percent = 0
+disk_state = "ok"
+
+try:
+    st = os.statvfs("/")
+    disk_percent = int(
+        100 * (st.f_blocks - st.f_bavail) / st.f_blocks
+    )
+except:
+    pass
+
+if disk_percent >= 95:
+    disk_state = "fail"
+elif disk_percent >= 85:
+    disk_state = "warn"
+
+
+# Notes data partition
+notes_disk_state = "ok"
+notes_disk_percent = 0
+notes_disk_detail = "Not checked"
+
+try:
+    mounts = open("/proc/mounts").read()
+    mounted = 0
+
+    for line in mounts.splitlines():
+        parts = line.split()
+        if len(parts) >= 2 and parts[1] == "/local/notesdata":
+            mounted = 1
+            break
+
+    if not mounted:
+        notes_disk_state = "fail"
+        notes_disk_detail = "/local/notesdata is not mounted"
+    else:
+        st = os.statvfs("/local/notesdata")
+
+        if st.f_blocks > 0:
+            notes_disk_percent = int(
+                100 * (st.f_blocks - st.f_bavail) / st.f_blocks
+            )
+
+            notes_disk_detail = "%d%% used" % notes_disk_percent
+
+            if notes_disk_percent >= 95:
+                notes_disk_state = "fail"
+            elif notes_disk_percent >= 85:
+                notes_disk_state = "warn"
+        else:
+            notes_disk_state = "fail"
+            notes_disk_detail = "Filesystem size unavailable"
+
+except:
+    notes_disk_state = "fail"
+    notes_disk_detail = "Cannot read Notes data filesystem"
+
+# Uptime
+try:
+    uptime = int(float(read_file("/proc/uptime").split()[0]))
+except:
+    uptime = 0
+
+checks = [
+    check(
+        "System",
+        "CPU load",
+        cpu_state,
+        "1m load %.2f; %d CPUs" % (load, cpu_count)
+    ),
+    check(
+        "System",
+        "Memory",
+        memory_state,
+        "%d%% used" % memory_percent
+    ),
+    check(
+        "Storage",
+        "Disk /",
+        disk_state,
+        "%d%% used" % disk_percent
+    ),
+    check(
+        "Storage",
+        "Notes data disk",
+        notes_disk_state,
+        notes_disk_detail
+    )
+]
+
+# Legacy SysV services
+services = []
+
+service_names = [
+    ("ssh", "sshd"),
+    ("apache2", "apache2"),
+    ("nginx", "nginx"),
+    ("mysql", "mysqld"),
+    ("postgresql", "postgres"),
+    ("redis-server", "redis-server"),
+    ("domino", "server")
+]
+
+for service_name, process_name in service_names:
+    init_script = "/etc/init.d/" + service_name
+
+    if not os.path.exists(init_script):
+        services.append(
+            status(service_name, "not_installed", "SysV service not found")
+        )
+        continue
+
+    running = process_running(process_name)
+
+    if running:
+        service_state = "running"
+    else:
+        service_state = "stopped"
+
+    services.append(
+        status(
+            service_name,
+            service_state,
+            "SysV process check"
+        )
+    )
+
+# Domino is a required service on Europa.
+domino_state = "ok"
+domino_detail = "Domino process running"
+
+if not process_running("server"):
+    domino_state = "fail"
+    domino_detail = "Domino server process not running"
+
+checks.append(
+    check(
+        "Services",
+        "Domino",
+        domino_state,
+        domino_detail
+    )
+)
+
+states = [
+    cpu_state,
+    memory_state,
+    disk_state,
+    notes_disk_state,
+    domino_state
+]
+
+overall = "ok"
+if "fail" in states:
+    overall = "fail"
+elif "warn" in states:
+    overall = "warn"
+
+ok_count = states.count("ok")
+warn_count = states.count("warn")
+fail_count = states.count("fail")
+
+if cpu_state == "fail" or memory_state == "fail":
+    system_state = "fail"
+elif cpu_state == "warn" or memory_state == "warn":
+    system_state = "warn"
+else:
+    system_state = "ok"
+
+if disk_state == "fail" or notes_disk_state == "fail":
+    storage_state = "fail"
+elif disk_state == "warn" or notes_disk_state == "warn":
+    storage_state = "warn"
+else:
+    storage_state = "ok"
+
+groups = [
+    group("System", system_state),
+    group("Storage", storage_state),
+    group("Services", domino_state)
+]
+
+output = """{
+"schema":1,
+"host":%s,
+"generatedAt":%s,
+"overall":%s,
+"counts":{"ok":%d,"warn":%d,"fail":%d},
+"groups":[%s],
+"checks":[%s],
+"diagnostics":null,
+"services":[%s],
+"uptimeSeconds":%d
+}""" % (
+    string(hostname),
+    string(time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())),
+    string(overall),
+    ok_count,
+    warn_count,
+    fail_count,
+    ",".join(groups),
+    ",".join(checks),
+    ",".join(services),
+    uptime
+)
+
+print output