214 lines
9.6 KiB
Python
214 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only post-reboot audit for Pegasus (Ubuntu). Python 3 stdlib only."""
|
|
import datetime
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
OK = 0
|
|
WARN = 0
|
|
FAIL = 0
|
|
|
|
def run(*args, timeout=12):
|
|
try:
|
|
p = subprocess.run(args, text=True, capture_output=True, timeout=timeout)
|
|
return p.returncode, (p.stdout + p.stderr).strip()
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
return 124, str(exc)
|
|
|
|
def check(label, success, detail='', severity='fail'):
|
|
global OK, WARN, FAIL
|
|
if success:
|
|
OK += 1
|
|
tag = 'OK '
|
|
elif severity == 'warn':
|
|
WARN += 1
|
|
tag = 'WARN'
|
|
else:
|
|
FAIL += 1
|
|
tag = 'FAIL'
|
|
print(f'[{tag}] {label}' + (f' — {detail}' if detail else ''))
|
|
|
|
def section(label):
|
|
print(f'\n=== {label} ===')
|
|
|
|
def active(unit, required=True):
|
|
rc, state = run('systemctl', 'is-active', unit)
|
|
check(unit, rc == 0 and state == 'active', state.splitlines()[0] if state else 'unknown',
|
|
'fail' if required else 'warn')
|
|
|
|
def port(label, port, host='127.0.0.1', required=False):
|
|
try:
|
|
with socket.create_connection((host, port), timeout=2):
|
|
check(label, True, f'{host}:{port} listening')
|
|
except OSError as exc:
|
|
check(label, False, f'{host}:{port} not reachable ({exc})', 'fail' if required else 'warn')
|
|
|
|
def web(label, url, required=False):
|
|
if not shutil.which('curl'):
|
|
check(label, False, 'curl not installed', 'warn')
|
|
return
|
|
rc, value = run('curl', '-L', '-sS', '-o', '/dev/null', '--connect-timeout', '4', '--max-time', '12',
|
|
'-w', '%{http_code}', url, timeout=15)
|
|
code = value[-3:] if len(value) >= 3 else value
|
|
# 401/403 may be expected for protected applications.
|
|
success = rc == 0 and code.isdigit() and 200 <= int(code) < 400
|
|
check(label, success, f'HTTP {code}' if rc == 0 else value[:120],
|
|
'fail' if required else 'warn')
|
|
|
|
now = datetime.datetime.now().astimezone()
|
|
print('PEGASUS — POST-REBOOT SYSTEM HEALTH')
|
|
print('Time:', now.isoformat(timespec='seconds'))
|
|
print('Host:', socket.gethostname(), '| User:', os.environ.get('USER', '?'))
|
|
if os.geteuid() != 0:
|
|
print('NOTE: Run with sudo for full Docker, kernel journal and SMART visibility.')
|
|
|
|
section('BOOT AND SYSTEM')
|
|
rc, value = run('uptime', '-p')
|
|
print('Uptime:', value)
|
|
rc, value = run('systemctl', '--failed', '--no-legend', '--plain', '--no-pager')
|
|
failed = [line for line in value.splitlines() if '.service' in line or '.mount' in line or '.socket' in line]
|
|
check('Failed systemd units', rc == 0 and not failed, '; '.join(failed[:8]) or 'none')
|
|
rc, value = run('journalctl', '-b', '-p', 'err', '--no-pager', '-n', '15')
|
|
if rc == 0 and value and 'No entries' not in value:
|
|
print('Recent boot errors (may include recovered/transient errors):')
|
|
for line in value.splitlines()[-15:]:
|
|
print(' ', line[:200])
|
|
|
|
section('STORAGE')
|
|
rc, value = run('findmnt', '-rn', '-o', 'SOURCE,FSTYPE', '--target', '/backup')
|
|
backed = rc == 0 and value and value.split()[0] != '/dev/mapper/ubuntu--vg-ubuntu--lv'
|
|
# Strict mountpoint test avoids interpreting root filesystem as a backup mount.
|
|
rc2, _ = run('mountpoint', '-q', '/backup')
|
|
backed = backed and rc2 == 0
|
|
check('/backup mounted', backed, value if backed else 'backup unavailable — DO NOT RUN BACKUPS')
|
|
if backed:
|
|
rc, uuid = run('findmnt', '-n', '-o', 'UUID', '--target', '/backup')
|
|
expected = '008c9f5d-5274-4d27-9ebe-ba8a6986f5ad'
|
|
check('Backup filesystem UUID', rc == 0 and uuid.strip() == expected,
|
|
f'found {uuid.strip() or "unknown"}; expected {expected}')
|
|
for path in ('/', '/backup', '/mnt/timemachine'):
|
|
if path == '/backup' and not backed:
|
|
continue
|
|
if path == '/mnt/timemachine':
|
|
rc, _ = run('mountpoint', '-q', path)
|
|
if rc:
|
|
check('Time Machine mount', False, f'{path} not mounted (if expected)', 'warn')
|
|
continue
|
|
try:
|
|
usage = shutil.disk_usage(path)
|
|
used = 100 * usage.used / usage.total
|
|
check(f'Space {path}', used < 90, f'{used:.1f}% used, {usage.free / 2**30:.1f} GiB free',
|
|
'warn' if used < 95 else 'fail')
|
|
except OSError as exc:
|
|
check(f'Space {path}', False, str(exc))
|
|
for folder in ('/backup/borg', '/backup/vms', '/backup/sebrant'):
|
|
if backed:
|
|
check(folder, Path(folder).is_dir(), 'directory present' if Path(folder).is_dir() else 'missing', 'warn')
|
|
|
|
section('CORE SERVICES')
|
|
for unit in ('nginx', 'mysql', 'postgresql', 'redis-server', 'docker', 'containerd',
|
|
'gitea', 'cloudflared', 'cron', 'libvirtd'):
|
|
active(unit)
|
|
for unit in ('php8.3-fpm', 'php8.4-fpm', 'netdata', 'cockpit.socket',
|
|
'openvpn', 'strongswan-starter', 'olilo-l2tp', 'gitea-mcp',
|
|
'pegasus-backup.timer', 'pegasus-backup.service'):
|
|
rc, output = run('systemctl', 'show', '-p', 'LoadState', '--value', unit)
|
|
if rc == 0 and output.strip() not in ('not-found', ''):
|
|
if unit.endswith('.service') and ('backup' in unit):
|
|
rc2, state = run('systemctl', 'is-active', unit)
|
|
check(unit, state in ('active', 'inactive'),
|
|
f'{state} (oneshot jobs can normally be inactive)', 'warn')
|
|
else:
|
|
active(unit, required=False)
|
|
rc, value = run('nginx', '-t')
|
|
check('Nginx configuration', rc == 0, value.splitlines()[-1][:120] if value else 'no result')
|
|
|
|
section('DATABASE AND SERVICE PORTS')
|
|
for label, portno, required in (
|
|
('Nginx HTTP', 80, True), ('Nginx HTTPS', 443, True),
|
|
('MySQL', 3306, True), ('Redis', 6379, True),
|
|
('PostgreSQL', 5432, False), ('Gitea HTTP', 3000, False),
|
|
('Cockpit', 9090, False), ('Netdata', 19999, False),
|
|
('Domino NRPC', 1352, False), ('Mail SMTP', 25, False),
|
|
('Mail submission', 587, False), ('Mail IMAPS', 993, False),
|
|
):
|
|
port(label, portno, required=required)
|
|
|
|
section('DOCKER CONTAINERS')
|
|
if shutil.which('docker'):
|
|
rc, value = run('docker', 'ps', '-a', '--format', '{{.Names}}|{{.State}}|{{.Status}}', timeout=25)
|
|
if rc != 0:
|
|
check('Docker query', False, value[:150])
|
|
else:
|
|
lines = [line for line in value.splitlines() if '|' in line]
|
|
for line in sorted(lines):
|
|
name, state, status = line.split('|', 2)
|
|
# Some stacks are intentionally stopped; report, do not declare them failed.
|
|
expected = {'sonarqube', 'sonarqube-db', 'zap', 'mailpit', 'mailserver',
|
|
'komodo-core', 'komodo-periphery', 'komodo-ferretdb',
|
|
'komodo-postgres', 'registry'}
|
|
required = name in expected
|
|
healthy = state == 'running' and '(unhealthy)' not in status
|
|
check(f'Container {name}', healthy, status, 'fail' if required else 'warn')
|
|
check('Docker containers discovered', bool(lines), f'{len(lines)} present')
|
|
else:
|
|
check('Docker CLI', False, 'not installed')
|
|
|
|
section('VIRTUAL MACHINES')
|
|
if shutil.which('virsh'):
|
|
rc, value = run('virsh', 'list', '--all', '--name')
|
|
names = [n.strip() for n in value.splitlines() if n.strip()] if rc == 0 else []
|
|
if 'domino651' in names:
|
|
rc, state = run('virsh', 'domstate', 'domino651')
|
|
check('domino651 VM', rc == 0 and state.strip() == 'running', state)
|
|
else:
|
|
check('domino651 VM', False, 'not found via default virsh connection', 'warn')
|
|
rc, value = run('virsh', 'dominfo', 'domino651')
|
|
if rc == 0:
|
|
autostart = next((x for x in value.splitlines() if x.startswith('Autostart:')), '')
|
|
check('domino651 autostart', 'enable' in autostart.lower(), autostart.strip(), 'warn')
|
|
|
|
section('NETWORK AND VPN')
|
|
for device in ('br0', 'ppp0', 'tun0'):
|
|
rc, value = run('ip', '-br', 'addr', 'show', device)
|
|
check(f'Interface {device}', rc == 0 and ' UP ' in f' {value} ', value[:120],
|
|
'fail' if device in ('br0', 'ppp0') else 'warn')
|
|
rc, value = run('ip', 'route', 'show', 'default')
|
|
check('Default route', rc == 0 and bool(value), value.replace('\n', '; ')[:150])
|
|
rc, value = run('getent', 'ahostsv4', 'cloudflare.com')
|
|
check('DNS resolution', rc == 0 and bool(value), value.splitlines()[0][:100] if value else 'failed')
|
|
for label, host, service_port in (('OpenVPN UDP', '0.0.0.0', 1194), ('IKEv2 UDP', '0.0.0.0', 500),
|
|
('IPsec NAT-T UDP', '0.0.0.0', 4500)):
|
|
rc, value = run('ss', '-H', '-lun')
|
|
present = rc == 0 and any(f':{service_port} ' in line for line in value.splitlines())
|
|
check(label, present, f'UDP {service_port} listening' if present else 'not listening', 'warn')
|
|
|
|
section('PUBLIC ENDPOINTS (EXTERNAL NETWORK / DNS DEPENDENT)')
|
|
for label, url in (
|
|
('absdev.net', 'https://absdev.net'), ('git.absdev.net', 'https://git.absdev.net'),
|
|
('notes.absdev.net', 'https://notes.absdev.net'),
|
|
('cloud.absdev.net', 'https://cloud.absdev.net')):
|
|
web(label, url, required=False)
|
|
|
|
section('BACKUP JOBS')
|
|
for unit in ('pegasus-backup.service', 'pegasus-backup.timer'):
|
|
rc, value = run('systemctl', 'show', unit, '-p', 'LoadState', '-p', 'ActiveState',
|
|
'-p', 'Result', '-p', 'UnitFileState')
|
|
if rc == 0 and 'LoadState=not-found' not in value:
|
|
print(unit + ': ' + value.replace('\n', ' | '))
|
|
rc, value = run('systemctl', 'list-timers', '--all', '--no-pager', '--plain')
|
|
if rc == 0:
|
|
for line in value.splitlines():
|
|
if any(word in line.lower() for word in ('backup', 'borg', 'certbot')):
|
|
print(line[:200])
|
|
|
|
print('\n' + '=' * 58)
|
|
print(f'SUMMARY: {OK} OK | {WARN} WARN | {FAIL} FAIL')
|
|
print('Warnings include intentionally stopped or optional services; review them.')
|
|
print('This checks availability, not complete application behavior or backup integrity.')
|
|
sys.exit(2 if FAIL else 1 if WARN else 0)
|