#!/bin/bash

# =============================================================================
# Server Configuration Audit Script
# =============================================================================
# Checks all servers for consistent MariaDB, Apache, PHP-FPM configuration,
# hardware specs, software versions, error log routing, and more.
# Generates an HTML report similar to rev.sh / inst_rev_ld.sh
# =============================================================================

# List of servers (same as inst_rev_ld.sh)
SERVERS=("localhost" "s21.tasko.cloud" "s20.tasko.cloud" "s19.tasko.cloud"  "s18.tasko.cloud" "s17.tasko.cloud" "s16.tasko.cloud" "s15.tasko.cloud" "s14.tasko.cloud" "s13.tasko.cloud" "s12.tasko.cloud" "s11.tasko.cloud" "s10.tasko.cloud" )
#SERVERS=("localhost")

# SSH user
SSH_USER="deployer"

audit_server() {
    local server="$1"

    ssh "$SSH_USER@$server" "bash -s" << 'REMOTE_EOF'

    # =========================================================================
    # HTML escape helper - prevents XSS / broken markup from raw values
    # =========================================================================
    h() {
        local s="$1"
        s="${s//&/&amp;}"
        s="${s//</&lt;}"
        s="${s//>/&gt;}"
        s="${s//\"/&quot;}"
        echo "$s"
    }

    # =========================================================================
    # 1. SYSTEM INFO
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>System Information</b></td></tr>"
    echo "<tr><th>Property</th><th colspan=5>Value</th></tr>"

    UBUNTU_VER=$(lsb_release -d 2>/dev/null | cut -f2)
    echo "<tr><td>OS</td><td colspan=5>$(h "${UBUNTU_VER:-unknown}")</td></tr>"

    echo "<tr><td>Hostname</td><td colspan=5>$(h "$(hostname -f 2>/dev/null || hostname)")</td></tr>"

    echo "<tr><td>Uptime</td><td colspan=5>$(h "$(uptime -p 2>/dev/null || uptime)")</td></tr>"

    # =========================================================================
    # 2. HARDWARE SPECS
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>Hardware</b></td></tr>"
    echo "<tr><th>Resource</th><th>Total</th><th>Used</th><th>Free</th><th>Usage %</th><th>Notes</th></tr>"

    # RAM
    TOTAL_RAM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}')
    AVAIL_RAM_KB=$(grep MemAvailable /proc/meminfo | awk '{print $2}')
    TOTAL_RAM_MB=$((TOTAL_RAM_KB / 1024))
    AVAIL_RAM_MB=$((AVAIL_RAM_KB / 1024))
    USED_RAM_MB=$((TOTAL_RAM_MB - AVAIL_RAM_MB))
    RAM_USAGE_PCT=$((USED_RAM_MB * 100 / TOTAL_RAM_MB))
    if [ "$RAM_USAGE_PCT" -gt 90 ]; then
        RAM_COLOR="background:#ffcdd2"
    elif [ "$RAM_USAGE_PCT" -gt 75 ]; then
        RAM_COLOR="background:#fff9c4"
    else
        RAM_COLOR="background:#c8e6c9"
    fi
    echo "<tr><td>RAM</td><td>${TOTAL_RAM_MB} MB</td><td>${USED_RAM_MB} MB</td><td>${AVAIL_RAM_MB} MB</td><td style='${RAM_COLOR}'>${RAM_USAGE_PCT}%</td><td></td></tr>"

    # Swap
    TOTAL_SWAP_KB=$(grep SwapTotal /proc/meminfo | awk '{print $2}')
    FREE_SWAP_KB=$(grep SwapFree /proc/meminfo | awk '{print $2}')
    TOTAL_SWAP_MB=$((TOTAL_SWAP_KB / 1024))
    USED_SWAP_MB=$(( (TOTAL_SWAP_KB - FREE_SWAP_KB) / 1024 ))
    if [ "$TOTAL_SWAP_KB" -gt 0 ]; then
        SWAP_USAGE_PCT=$((USED_SWAP_MB * 100 / TOTAL_SWAP_MB))
    else
        SWAP_USAGE_PCT=0
    fi
    if [ "$SWAP_USAGE_PCT" -gt 50 ]; then
        SWAP_COLOR="background:#ffcdd2"
    elif [ "$SWAP_USAGE_PCT" -gt 20 ]; then
        SWAP_COLOR="background:#fff9c4"
    else
        SWAP_COLOR="background:#c8e6c9"
    fi
    echo "<tr><td>Swap</td><td>${TOTAL_SWAP_MB} MB</td><td>${USED_SWAP_MB} MB</td><td>$((FREE_SWAP_KB / 1024)) MB</td><td style='${SWAP_COLOR}'>${SWAP_USAGE_PCT}%</td><td></td></tr>"

    # CPU
    CPU_CORES=$(nproc 2>/dev/null || grep -c ^processor /proc/cpuinfo)
    CPU_MODEL=$(grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2 | xargs)
    LOAD_AVG=$(awk '{print $1, $2, $3}' /proc/loadavg)
    echo "<tr><td>CPU</td><td>${CPU_CORES} cores</td><td colspan=3>$(h "$CPU_MODEL")</td><td>Load: ${LOAD_AVG}</td></tr>"

    # Disk
    echo "<tr><td>Disk</td><td colspan=5><pre style='margin:0'>"
    df -h --total 2>/dev/null | grep -E '(Filesystem|/dev/|total)' | while IFS= read -r line; do h "$line"; done
    echo "</pre></td></tr>"

    # =========================================================================
    # 3. PHP CONFIGURATION
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>PHP Configuration</b></td></tr>"
    echo "<tr><th>Parameter</th><th>Value</th><th>Expected</th><th>Status</th><th colspan=2>Notes</th></tr>"

    # PHP version
    PHP_VER=$(php -v 2>/dev/null | head -1 | awk '{print $2}')
    echo "<tr><td>PHP Version</td><td>$(h "${PHP_VER:-not found}")</td><td>8.3.x</td>"
    if echo "$PHP_VER" | grep -q "^8\.3"; then
        echo "<td style='background:#c8e6c9'>OK</td>"
    else
        echo "<td style='background:#ffcdd2'>MISMATCH</td>"
    fi
    echo "<td colspan=2></td></tr>"

    # PHP-FPM pool config - try multiple PHP version paths
    FPM_CONF=""
    FPM_PHP_VER=""
    for v in 8.3 8.2 8.1 8.0; do
        if [ -f "/etc/php/${v}/fpm/pool.d/www.conf" ]; then
            FPM_CONF="/etc/php/${v}/fpm/pool.d/www.conf"
            FPM_PHP_VER="$v"
            break
        fi
    done

    PHP_INI_FPM=""
    for v in 8.3 8.2 8.1 8.0; do
        if [ -f "/etc/php/${v}/fpm/php.ini" ]; then
            PHP_INI_FPM="/etc/php/${v}/fpm/php.ini"
            break
        fi
    done

    if [ -n "$FPM_CONF" ]; then
        # Parse FPM pool values - allow leading whitespace, = with spaces
        PM_MAX_CHILDREN=$(grep -E "^\s*pm\.max_children\s*=" "$FPM_CONF" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
        PM_MODE=$(grep -E "^\s*pm\s*=" "$FPM_CONF" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
        PM_START=$(grep -E "^\s*pm\.start_servers\s*=" "$FPM_CONF" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
        PM_MIN_SPARE=$(grep -E "^\s*pm\.min_spare_servers\s*=" "$FPM_CONF" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
        PM_MAX_SPARE=$(grep -E "^\s*pm\.max_spare_servers\s*=" "$FPM_CONF" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
        PM_MAX_REQ=$(grep -E "^\s*pm\.max_requests\s*=" "$FPM_CONF" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')

        # Get memory_limit from FPM php.ini (not CLI)
        MEMORY_LIMIT=""
        if [ -n "$PHP_INI_FPM" ]; then
            MEMORY_LIMIT=$(grep -E "^\s*memory_limit\s*=" "$PHP_INI_FPM" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
        fi
        if [ -z "$MEMORY_LIMIT" ]; then
            MEMORY_LIMIT=$(php -r 'echo ini_get("memory_limit");' 2>/dev/null)
        fi

        # Parse memory_limit to MB
        MEM_LIMIT_NUM=$(echo "$MEMORY_LIMIT" | grep -oE '[0-9]+')
        MEM_LIMIT_UNIT=$(echo "$MEMORY_LIMIT" | grep -oE '[A-Za-z]+$')
        case "$MEM_LIMIT_UNIT" in
            G|g) MEM_LIMIT_MB=$((MEM_LIMIT_NUM * 1024)) ;;
            M|m) MEM_LIMIT_MB=$MEM_LIMIT_NUM ;;
            K|k) MEM_LIMIT_MB=$((MEM_LIMIT_NUM / 1024)) ;;
            *)   MEM_LIMIT_MB=$MEM_LIMIT_NUM ;;
        esac

        # Observe actual FPM worker RSS for a more realistic estimate
        AVG_WORKER_RSS_KB=$(ps --no-headers -o rss -C "php-fpm${FPM_PHP_VER}" 2>/dev/null | awk '{sum+=$1; n++} END {if(n>0) printf "%d", sum/n; else print 0}')
        if [ "$AVG_WORKER_RSS_KB" -eq 0 ] 2>/dev/null; then
            AVG_WORKER_RSS_KB=$(ps --no-headers -o rss -C php-fpm 2>/dev/null | awk 'NR>1{sum+=$1; n++} END {if(n>0) printf "%d", sum/n; else print 0}')
        fi
        AVG_WORKER_RSS_MB=$((AVG_WORKER_RSS_KB / 1024))

        # Reserve ~20% of RAM for MariaDB + Apache + OS, use 80% for PHP-FPM
        AVAILABLE_FOR_PHP_MB=$((TOTAL_RAM_MB * 80 / 100))

        # Use actual RSS if available, otherwise fall back to memory_limit
        if [ "$AVG_WORKER_RSS_MB" -gt 0 ] 2>/dev/null; then
            RECOMMENDED_MAX_CHILDREN=$((AVAILABLE_FOR_PHP_MB / AVG_WORKER_RSS_MB))
            ESTIMATE_SOURCE="actual avg RSS: ${AVG_WORKER_RSS_MB}MB"
        elif [ "$MEM_LIMIT_MB" -gt 0 ] 2>/dev/null; then
            RECOMMENDED_MAX_CHILDREN=$((AVAILABLE_FOR_PHP_MB / MEM_LIMIT_MB))
            ESTIMATE_SOURCE="memory_limit: ${MEMORY_LIMIT} (rough estimate, no running workers found)"
        else
            RECOMMENDED_MAX_CHILDREN="N/A"
            ESTIMATE_SOURCE="cannot determine"
        fi

        echo "<tr><td>FPM Pool Config</td><td colspan=5>$(h "$FPM_CONF")</td></tr>"
        echo "<tr><td>pm mode</td><td>$(h "${PM_MODE:-not set}")</td><td>dynamic</td>"
        if [ "$PM_MODE" = "dynamic" ]; then
            echo "<td style='background:#c8e6c9'>OK</td>"
        else
            echo "<td style='background:#fff9c4'>CHECK</td>"
        fi
        echo "<td colspan=2></td></tr>"

        # Actual worker memory
        if [ "$AVG_WORKER_RSS_MB" -gt 0 ] 2>/dev/null; then
            WORKER_COUNT=$(ps --no-headers -o rss -C php-fpm 2>/dev/null | wc -l)
            echo "<tr><td>FPM worker avg RSS</td><td><b>${AVG_WORKER_RSS_MB} MB</b></td><td></td><td></td><td colspan=2>${WORKER_COUNT} workers running</td></tr>"
        fi

        echo "<tr><td><b>pm.max_children</b></td><td><b>$(h "${PM_MAX_CHILDREN:-not set}")</b></td><td><b>~${RECOMMENDED_MAX_CHILDREN}</b></td>"
        if [ -n "$PM_MAX_CHILDREN" ] && [ "$RECOMMENDED_MAX_CHILDREN" != "N/A" ]; then
            if [ "$PM_MAX_CHILDREN" -le 5 ] 2>/dev/null; then
                echo "<td style='background:#ffcdd2'><b>TOO LOW</b></td>"
                echo "<td colspan=2>TICKET: server reached pm.max_children (${PM_MAX_CHILDREN}). Based on ${ESTIMATE_SOURCE}</td>"
            elif [ "$PM_MAX_CHILDREN" -lt "$((RECOMMENDED_MAX_CHILDREN / 2))" ] 2>/dev/null; then
                echo "<td style='background:#fff9c4'><b>LOW</b></td>"
                echo "<td colspan=2>Consider raising. Based on ${ESTIMATE_SOURCE}</td>"
            elif [ "$PM_MAX_CHILDREN" -gt "$((RECOMMENDED_MAX_CHILDREN * 2))" ] 2>/dev/null; then
                echo "<td style='background:#fff9c4'><b>HIGH</b></td>"
                echo "<td colspan=2>Risk of OOM. Based on ${ESTIMATE_SOURCE}</td>"
            else
                echo "<td style='background:#c8e6c9'>OK</td>"
                echo "<td colspan=2>Based on ${ESTIMATE_SOURCE}</td>"
            fi
        else
            echo "<td style='background:#ffcdd2'>ERROR</td><td colspan=2>Cannot calculate (${ESTIMATE_SOURCE})</td>"
        fi
        echo "</tr>"

        echo "<tr><td>pm.start_servers</td><td>$(h "${PM_START:-not set}")</td><td></td><td></td><td colspan=2></td></tr>"
        echo "<tr><td>pm.min_spare_servers</td><td>$(h "${PM_MIN_SPARE:-not set}")</td><td></td><td></td><td colspan=2></td></tr>"
        echo "<tr><td>pm.max_spare_servers</td><td>$(h "${PM_MAX_SPARE:-not set}")</td><td></td><td></td><td colspan=2></td></tr>"
        echo "<tr><td>pm.max_requests</td><td>$(h "${PM_MAX_REQ:-not set}")</td><td>500</td>"
        if [ -z "$PM_MAX_REQ" ] || [ "$PM_MAX_REQ" = "0" ]; then
            echo "<td style='background:#fff9c4'>NOT SET</td><td colspan=2>Recommend setting to prevent memory leaks</td>"
        else
            echo "<td style='background:#c8e6c9'>OK</td><td colspan=2></td>"
        fi
        echo "</tr>"
    else
        echo "<tr><td>FPM Pool Config</td><td colspan=5 style='background:#ffcdd2'>NOT FOUND</td></tr>"
    fi

    # PHP ini settings (from FPM php.ini)
    echo "<tr><td colspan=6 style='background:#e8eaf6'><b>PHP ini Settings (FPM)</b></td></tr>"
    if [ -n "$PHP_INI_FPM" ]; then
        echo "<tr><td>php.ini path</td><td colspan=5>$(h "$PHP_INI_FPM")</td></tr>"

        # Helper to parse ini value allowing whitespace
        ini_val() { grep -E "^\s*${1}\s*=" "$PHP_INI_FPM" 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}'; }

        MEM_LIMIT_VAL=$(ini_val memory_limit)
        echo "<tr><td>memory_limit</td><td>$(h "${MEM_LIMIT_VAL:-not set}")</td><td>512M</td>"
        if [ "$MEM_LIMIT_VAL" = "512M" ]; then
            echo "<td style='background:#c8e6c9'>OK</td>"
        else
            echo "<td style='background:#fff9c4'>DIFFERS</td>"
        fi
        echo "<td colspan=2></td></tr>"

        UPLOAD_MAX=$(ini_val upload_max_filesize)
        echo "<tr><td>upload_max_filesize</td><td>$(h "${UPLOAD_MAX:-not set}")</td><td>256M</td>"
        if [ "$UPLOAD_MAX" = "256M" ]; then
            echo "<td style='background:#c8e6c9'>OK</td>"
        else
            echo "<td style='background:#fff9c4'>DIFFERS</td>"
        fi
        echo "<td colspan=2></td></tr>"

        POST_MAX=$(ini_val post_max_size)
        echo "<tr><td>post_max_size</td><td>$(h "${POST_MAX:-not set}")</td><td>&gt;=256M</td>"
        echo "<td></td><td colspan=2>Should be &gt;= upload_max_filesize</td></tr>"

        MAX_EXEC=$(ini_val max_execution_time)
        echo "<tr><td>max_execution_time</td><td>$(h "${MAX_EXEC:-not set}")</td><td>300</td><td></td><td colspan=2></td></tr>"

        MAX_INPUT=$(ini_val max_input_time)
        echo "<tr><td>max_input_time</td><td>$(h "${MAX_INPUT:-not set}")</td><td>300</td><td></td><td colspan=2></td></tr>"

        MAX_INPUT_VARS=$(ini_val max_input_vars)
        echo "<tr><td>max_input_vars</td><td>$(h "${MAX_INPUT_VARS:-not set}")</td><td>3000</td><td></td><td colspan=2></td></tr>"

        DISPLAY_ERRORS=$(ini_val display_errors)
        echo "<tr><td>display_errors</td><td>$(h "${DISPLAY_ERRORS:-not set}")</td><td>Off</td>"
        if [ "$DISPLAY_ERRORS" = "Off" ] || [ "$DISPLAY_ERRORS" = "off" ]; then
            echo "<td style='background:#c8e6c9'>OK</td>"
        elif [ -z "$DISPLAY_ERRORS" ]; then
            echo "<td></td>"
        else
            echo "<td style='background:#fff9c4'>CHECK</td>"
        fi
        echo "<td colspan=2>Should be Off in production</td></tr>"

        LOG_ERRORS=$(ini_val log_errors)
        echo "<tr><td>log_errors</td><td>$(h "${LOG_ERRORS:-not set}")</td><td>On</td>"
        if [ "$LOG_ERRORS" = "On" ] || [ "$LOG_ERRORS" = "on" ]; then
            echo "<td style='background:#c8e6c9'>OK</td>"
        else
            echo "<td style='background:#ffcdd2'>OFF</td>"
        fi
        echo "<td colspan=2>Must be On for error tracking</td></tr>"

        ERROR_LOG=$(ini_val error_log)
        echo "<tr><td>error_log</td><td colspan=5>$(h "${ERROR_LOG:-syslog (default)}")</td></tr>"

        # OPcache
        OPCACHE_ENABLE=$(ini_val "opcache\.enable")
        OPCACHE_MEM=$(ini_val "opcache\.memory_consumption")
        OPCACHE_STRINGS=$(ini_val "opcache\.interned_strings_buffer")
        OPCACHE_FILES=$(ini_val "opcache\.max_accelerated_files")
        OPCACHE_REVALIDATE=$(ini_val "opcache\.revalidate_freq")
        echo "<tr><td>opcache.enable</td><td>$(h "${OPCACHE_ENABLE:-not set}")</td><td>1</td>"
        if [ "$OPCACHE_ENABLE" = "1" ]; then
            echo "<td style='background:#c8e6c9'>OK</td>"
        else
            echo "<td style='background:#fff9c4'>CHECK</td>"
        fi
        echo "<td colspan=2></td></tr>"
        echo "<tr><td>opcache.memory_consumption</td><td>$(h "${OPCACHE_MEM:-not set}")</td><td>256</td><td></td><td colspan=2></td></tr>"
        echo "<tr><td>opcache.interned_strings_buffer</td><td>$(h "${OPCACHE_STRINGS:-not set}")</td><td>16</td><td></td><td colspan=2></td></tr>"
        echo "<tr><td>opcache.max_accelerated_files</td><td>$(h "${OPCACHE_FILES:-not set}")</td><td>10000</td><td></td><td colspan=2></td></tr>"
        echo "<tr><td>opcache.revalidate_freq</td><td>$(h "${OPCACHE_REVALIDATE:-not set}")</td><td>2</td><td></td><td colspan=2></td></tr>"
    else
        echo "<tr><td colspan=6 style='background:#ffcdd2'>PHP FPM ini not found</td></tr>"
    fi

    # =========================================================================
    # 4. APACHE CONFIGURATION
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>Apache Configuration</b></td></tr>"

    APACHE_VER=$(apache2 -v 2>/dev/null | head -1)
    echo "<tr><td>Apache Version</td><td colspan=5>$(h "${APACHE_VER:-not found}")</td></tr>"

    # MPM module - take first match only
    APACHE_MPM=$(apache2ctl -M 2>/dev/null | grep -oE 'mpm_\w+' | head -1)
    echo "<tr><td>MPM Module</td><td>$(h "${APACHE_MPM:-not found}")</td><td>mpm_prefork or mpm_event</td><td></td><td colspan=2></td></tr>"

    # MaxRequestWorkers - search key config files only (not all of /etc/apache2/ recursively)
    MAX_REQ_WORKERS=""
    MAX_REQ_SOURCE=""
    for conf_file in /etc/apache2/mods-enabled/mpm_*.conf /etc/apache2/apache2.conf; do
        if [ -f "$conf_file" ]; then
            val=$(grep -E "^\s*MaxRequestWorkers" "$conf_file" 2>/dev/null | grep -v "#" | tail -1 | awk '{print $2}')
            if [ -n "$val" ]; then
                MAX_REQ_WORKERS="$val"
                MAX_REQ_SOURCE="$conf_file"
            fi
        fi
    done
    echo "<tr><td><b>MaxRequestWorkers</b></td><td><b>$(h "${MAX_REQ_WORKERS:-not set (default 256)}")</b></td><td></td>"
    if [ -z "$MAX_REQ_WORKERS" ]; then
        echo "<td style='background:#fff9c4'>DEFAULT</td><td colspan=2>Using default 256. Review based on load.</td>"
    else
        echo "<td style='background:#c8e6c9'>SET</td><td colspan=2>Source: $(h "$MAX_REQ_SOURCE")</td>"
    fi
    echo "</tr>"

    # KeepAlive settings - from main config files only
    KEEP_ALIVE=$(grep -hE "^\s*KeepAlive\s" /etc/apache2/apache2.conf /etc/apache2/conf-enabled/*.conf 2>/dev/null | grep -v "#" | tail -1 | awk '{print $2}')
    echo "<tr><td>KeepAlive</td><td>$(h "${KEEP_ALIVE:-not set (default On)}")</td><td>On</td><td></td><td colspan=2></td></tr>"

    KEEP_ALIVE_TIMEOUT=$(grep -hE "^\s*KeepAliveTimeout" /etc/apache2/apache2.conf /etc/apache2/conf-enabled/*.conf 2>/dev/null | grep -v "#" | tail -1 | awk '{print $2}')
    echo "<tr><td>KeepAliveTimeout</td><td>$(h "${KEEP_ALIVE_TIMEOUT:-not set (default 5)}")</td><td>5</td><td></td><td colspan=2></td></tr>"

    TIMEOUT=$(grep -hE "^\s*Timeout\s" /etc/apache2/apache2.conf /etc/apache2/conf-enabled/*.conf 2>/dev/null | grep -v "#" | tail -1 | awk '{print $2}')
    echo "<tr><td>Timeout</td><td>$(h "${TIMEOUT:-not set (default 300)}")</td><td>300</td><td></td><td colspan=2></td></tr>"

    # Apache error log locations - global + per-site
    APACHE_GLOBAL_ERRORLOG=$(grep -hE "^\s*ErrorLog" /etc/apache2/apache2.conf 2>/dev/null | grep -v "#" | tail -1 | awk '{print $2}')
    echo "<tr><td>Global ErrorLog</td><td colspan=5>$(h "${APACHE_GLOBAL_ERRORLOG:-not found}")</td></tr>"

    # Unique ErrorLog paths from enabled sites
    SITE_LOGS=$(grep -rhE "^\s*ErrorLog" /etc/apache2/sites-enabled/ 2>/dev/null | grep -v "#" | awk '{print $2}' | sort -u | head -10)
    if [ -n "$SITE_LOGS" ]; then
        echo "<tr><td>Site ErrorLog paths</td><td colspan=5><small>"
        echo "$SITE_LOGS" | while IFS= read -r logpath; do h "$logpath"; echo "<br>"; done
        echo "</small></td></tr>"
    fi

    # Key enabled modules
    echo "<tr><td>Enabled Modules</td><td colspan=5>"
    apache2ctl -M 2>/dev/null | grep -E "(rewrite|ssl|headers|proxy|deflate|expires)" | awk '{print $1}' | sort | tr '\n' ', '
    echo "</td></tr>"

    # =========================================================================
    # 5. MariaDB CONFIGURATION
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>MariaDB / MySQL Configuration</b></td></tr>"

    # Detect which DB is installed
    DB_VER=$(mariadb --version 2>/dev/null || mysql --version 2>/dev/null)
    if echo "$DB_VER" | grep -qi mariadb 2>/dev/null; then
        DB_NAME="MariaDB"
    else
        DB_NAME="MySQL"
    fi
    echo "<tr><td>${DB_NAME} Version</td><td colspan=5>$(h "${DB_VER:-not found}")</td></tr>"

    # Detect service name dynamically
    DB_SERVICE=""
    for svc_candidate in mariadb mysql mysqld; do
        if systemctl list-unit-files "${svc_candidate}.service" 2>/dev/null | grep -q "${svc_candidate}"; then
            DB_SERVICE="${svc_candidate}"
            break
        fi
    done

    if [ -n "$DB_SERVICE" ]; then
        DB_SVC_STATUS=$(systemctl is-active "$DB_SERVICE" 2>/dev/null || echo "unknown")
        echo "<tr><td>${DB_NAME} Service (${DB_SERVICE})</td>"
        if [ "$DB_SVC_STATUS" = "active" ]; then
            echo "<td style='background:#c8e6c9' colspan=5>Running</td></tr>"
        else
            echo "<td style='background:#ffcdd2' colspan=5>$(h "$DB_SVC_STATUS")</td></tr>"
        fi
    else
        echo "<tr><td>${DB_NAME} Service</td><td style='background:#ffcdd2' colspan=5>No service found</td></tr>"
    fi

    # Use my_print_defaults for effective config (handles all !includedir)
    # Falls back to scanning .cnf files if my_print_defaults not available
    db_get_val() {
        local key="$1"
        local val=""
        # Try my_print_defaults first - gives effective merged config
        if command -v my_print_defaults &>/dev/null; then
            val=$(my_print_defaults mysqld 2>/dev/null | grep -E "^--${key}=" | tail -1 | awk -F'=' '{print $2}')
        fi
        # Fallback: scan all .cnf files
        if [ -z "$val" ]; then
            local files=$(find /etc/mysql/ -name '*.cnf' -type f 2>/dev/null)
            if [ -z "$files" ]; then
                files=$(find /etc/my.cnf.d/ -name '*.cnf' -type f 2>/dev/null)
            fi
            if [ -n "$files" ]; then
                val=$(grep -rhE "^\s*${key}\s*=" $files 2>/dev/null | tail -1 | awk -F'=' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}')
            fi
        fi
        echo "$val"
    }

    # Show which config files exist
    MYSQL_CONF_FILES=$(find /etc/mysql/ -name '*.cnf' -type f 2>/dev/null)
    if [ -n "$MYSQL_CONF_FILES" ]; then
        echo "<tr><td>Config Files</td><td colspan=5><small>"
        echo "$MYSQL_CONF_FILES" | while IFS= read -r f; do h "$f"; echo "<br>"; done
        echo "</small></td></tr>"
    fi

    INNODB_BUFFER=$(db_get_val innodb_buffer_pool_size)
    echo "<tr><td>innodb_buffer_pool_size</td><td>$(h "${INNODB_BUFFER:-not set (default 128M)}")</td><td>~50-70% of RAM for DB servers</td><td></td><td colspan=2></td></tr>"

    MAX_CONNECTIONS=$(db_get_val max_connections)
    echo "<tr><td>max_connections</td><td>$(h "${MAX_CONNECTIONS:-not set (default 151)}")</td><td>200-500</td><td></td><td colspan=2></td></tr>"

    SLOW_QUERY=$(db_get_val slow_query_log)
    SLOW_LOG_FILE=$(db_get_val slow_query_log_file)
    echo "<tr><td>slow_query_log</td><td>$(h "${SLOW_QUERY:-not set}")</td><td>1 (enabled)</td>"
    if [ "$SLOW_QUERY" = "1" ] || [ "$SLOW_QUERY" = "ON" ] || [ "$SLOW_QUERY" = "on" ]; then
        echo "<td style='background:#c8e6c9'>ON</td>"
    else
        echo "<td style='background:#fff9c4'>OFF</td>"
    fi
    echo "<td colspan=2>$(h "$SLOW_LOG_FILE")</td></tr>"

    LOG_ERROR=$(db_get_val log_error)
    echo "<tr><td>log_error</td><td colspan=5>$(h "${LOG_ERROR:-syslog (default)}")</td></tr>"

    MAX_ALLOWED_PACKET=$(db_get_val max_allowed_packet)
    echo "<tr><td>max_allowed_packet</td><td>$(h "${MAX_ALLOWED_PACKET:-not set (default 64M)}")</td><td>64M+</td><td></td><td colspan=2></td></tr>"

    WAIT_TIMEOUT=$(db_get_val wait_timeout)
    echo "<tr><td>wait_timeout</td><td>$(h "${WAIT_TIMEOUT:-not set (default 28800)}")</td><td>28800</td><td></td><td colspan=2></td></tr>"

    INNODB_LOG_SIZE=$(db_get_val innodb_log_file_size)
    echo "<tr><td>innodb_log_file_size</td><td>$(h "${INNODB_LOG_SIZE:-not set}")</td><td>256M</td><td></td><td colspan=2></td></tr>"

    KEY_BUFFER=$(db_get_val key_buffer_size)
    echo "<tr><td>key_buffer_size</td><td>$(h "${KEY_BUFFER:-not set}")</td><td>256M</td><td></td><td colspan=2></td></tr>"

    # =========================================================================
    # 6. ERROR LOG ROUTING (Elastic / Filebeat)
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>Error Log Routing to Elastic</b></td></tr>"

    # Check for Filebeat
    if command -v filebeat &>/dev/null || [ -f /usr/bin/filebeat ] || [ -f /usr/share/filebeat/bin/filebeat ]; then
        FB_VER=$(filebeat version 2>/dev/null | head -1)
        FB_RUNNING=$(systemctl is-active filebeat 2>/dev/null)
        echo "<tr><td>Filebeat</td><td>$(h "$FB_VER")</td><td></td>"
        if [ "$FB_RUNNING" = "active" ]; then
            echo "<td style='background:#c8e6c9'>Running</td>"
        else
            echo "<td style='background:#ffcdd2'>$(h "$FB_RUNNING")</td>"
        fi
        echo "<td colspan=2></td></tr>"

        # Check for specific log paths in filebeat config AND modules
        FB_PHP_OK=0
        FB_APACHE_OK=0

        # Check main config for actual log file paths
        if [ -f /etc/filebeat/filebeat.yml ]; then
            if grep -qE '/var/log/php.*fpm|php.*error' /etc/filebeat/filebeat.yml 2>/dev/null; then
                FB_PHP_OK=1
            fi
            if grep -qE '/var/log/apache2/(error|access)' /etc/filebeat/filebeat.yml 2>/dev/null; then
                FB_APACHE_OK=1
            fi
        fi

        # Check filebeat modules (e.g. apache module enabled)
        if [ -d /etc/filebeat/modules.d ]; then
            if [ -f /etc/filebeat/modules.d/apache.yml ] && ! echo /etc/filebeat/modules.d/apache.yml.disabled | grep -q disabled; then
                FB_APACHE_OK=1
            fi
            # There's no standard php module, so config-based detection is the right approach
        fi

        echo "<tr><td>Filebeat &rarr; PHP logs</td><td colspan=2>"
        if [ "$FB_PHP_OK" -eq 1 ]; then
            echo "Log path configured"
            echo "</td><td style='background:#c8e6c9'>OK</td>"
        else
            echo "No PHP log path found in config"
            echo "</td><td style='background:#ffcdd2'>MISSING</td>"
        fi
        echo "<td colspan=2>Expected: /var/log/php*-fpm.log or similar</td></tr>"

        echo "<tr><td>Filebeat &rarr; Apache logs</td><td colspan=2>"
        if [ "$FB_APACHE_OK" -eq 1 ]; then
            echo "Log path or module configured"
            echo "</td><td style='background:#c8e6c9'>OK</td>"
        else
            echo "No Apache log path found in config"
            echo "</td><td style='background:#ffcdd2'>MISSING</td>"
        fi
        echo "<td colspan=2>Expected: /var/log/apache2/error.log or apache module</td></tr>"
    else
        echo "<tr><td>Filebeat</td><td style='background:#ffcdd2' colspan=5>NOT INSTALLED - PHP and Apache error logs will NOT reach Elastic</td></tr>"
    fi

    # Check for rsyslog
    if systemctl is-active --quiet rsyslog 2>/dev/null; then
        echo "<tr><td>Rsyslog</td><td style='background:#c8e6c9' colspan=5>Running</td></tr>"
    fi

    # Show actual log file locations and sizes
    PHP_FPM_ERROR_LOG=""
    for v in 8.3 8.2 8.1; do
        if [ -f "/var/log/php${v}-fpm.log" ]; then
            PHP_FPM_ERROR_LOG="/var/log/php${v}-fpm.log"
            break
        fi
    done
    if [ -n "$PHP_FPM_ERROR_LOG" ]; then
        echo "<tr><td>PHP-FPM Error Log</td><td colspan=5>$(h "$PHP_FPM_ERROR_LOG") ($(ls -lh "$PHP_FPM_ERROR_LOG" 2>/dev/null | awk '{print $5}'))</td></tr>"
    fi

    if [ -f /var/log/apache2/error.log ]; then
        echo "<tr><td>Apache Error Log</td><td colspan=5>/var/log/apache2/error.log ($(ls -lh /var/log/apache2/error.log 2>/dev/null | awk '{print $5}'))</td></tr>"
    fi

    # =========================================================================
    # 7. RECENT ERROR LOG WARNINGS
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>Recent Critical Warnings (last 1000 lines)</b></td></tr>"

    # PHP-FPM: max_children reached - find the actual log file first
    FPM_MAX_CHILD_HITS=0
    if [ -n "$PHP_FPM_ERROR_LOG" ] && [ -r "$PHP_FPM_ERROR_LOG" ]; then
        FPM_MAX_CHILD_HITS=$(tail -1000 "$PHP_FPM_ERROR_LOG" 2>/dev/null | grep -c "max_children" || echo 0)
    fi
    echo "<tr><td>PHP-FPM max_children hits</td><td>${FPM_MAX_CHILD_HITS}</td><td>0</td>"
    if [ "$FPM_MAX_CHILD_HITS" -gt 0 ]; then
        echo "<td style='background:#ffcdd2'>TICKET</td><td colspan=2>WARNING: server reached pm.max_children</td>"
    else
        echo "<td style='background:#c8e6c9'>OK</td><td colspan=2></td>"
    fi
    echo "</tr>"

    # Apache: MaxRequestWorkers reached
    APACHE_MAX_WORKER_HITS=0
    if [ -r /var/log/apache2/error.log ]; then
        APACHE_MAX_WORKER_HITS=$(tail -1000 /var/log/apache2/error.log 2>/dev/null | grep -c "MaxRequestWorkers" || echo 0)
    fi
    echo "<tr><td>Apache MaxRequestWorkers hits</td><td>${APACHE_MAX_WORKER_HITS}</td><td>0</td>"
    if [ "$APACHE_MAX_WORKER_HITS" -gt 0 ]; then
        echo "<td style='background:#ffcdd2'>TICKET</td><td colspan=2>AH00161: server reached MaxRequestWorkers</td>"
    else
        echo "<td style='background:#c8e6c9'>OK</td><td colspan=2></td>"
    fi
    echo "</tr>"

    # Timeout errors
    TIMEOUT_HITS=0
    if [ -r /var/log/apache2/error.log ]; then
        TIMEOUT_HITS=$(tail -1000 /var/log/apache2/error.log 2>/dev/null | grep -c "timeout specified has expired" || echo 0)
    fi
    echo "<tr><td>Apache timeout errors</td><td>${TIMEOUT_HITS}</td><td>0</td>"
    if [ "$TIMEOUT_HITS" -gt 0 ]; then
        echo "<td style='background:#fff9c4'>WARN</td><td colspan=2>(70007)The timeout specified has expired</td>"
    else
        echo "<td style='background:#c8e6c9'>OK</td><td colspan=2></td>"
    fi
    echo "</tr>"

    # OOM kills - use journalctl (works with permissions), fallback dmesg
    OOM_KILLS=0
    OOM_SOURCE=""
    OOM_OUTPUT=$(journalctl -k --no-pager -q 2>/dev/null | grep -c "Out of memory" 2>/dev/null)
    if [ -n "$OOM_OUTPUT" ] && [ "$OOM_OUTPUT" -gt 0 ] 2>/dev/null; then
        OOM_KILLS=$OOM_OUTPUT
        OOM_SOURCE="journalctl -k"
    else
        OOM_OUTPUT=$(dmesg 2>&1)
        if echo "$OOM_OUTPUT" | grep -q "Operation not permitted\|Permission denied"; then
            OOM_KILLS=-1
            OOM_SOURCE="no permission (dmesg_restrict)"
        else
            OOM_KILLS=$(echo "$OOM_OUTPUT" | grep -c "Out of memory" || echo 0)
            OOM_SOURCE="dmesg"
        fi
    fi
    echo "<tr><td>OOM Kills</td><td>${OOM_KILLS}</td><td>0</td>"
    if [ "$OOM_KILLS" -eq -1 ] 2>/dev/null; then
        echo "<td style='background:#fff9c4'>UNKNOWN</td><td colspan=2>Cannot read kernel log (${OOM_SOURCE})</td>"
    elif [ "$OOM_KILLS" -gt 0 ] 2>/dev/null; then
        echo "<td style='background:#ffcdd2'>CRITICAL</td><td colspan=2>Out of Memory events detected! (${OOM_SOURCE})</td>"
    else
        echo "<td style='background:#c8e6c9'>OK</td><td colspan=2>(${OOM_SOURCE})</td>"
    fi
    echo "</tr>"

    # =========================================================================
    # 8. SOFTWARE VERSIONS
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>Software Versions</b></td></tr>"

    NODE_VER=$(node -v 2>/dev/null || echo "not installed")
    echo "<tr><td>Node.js</td><td>$(h "$NODE_VER")</td><td></td><td></td><td colspan=2></td></tr>"

    NPM_VER=$(npm -v 2>/dev/null || echo "not installed")
    echo "<tr><td>npm</td><td>$(h "$NPM_VER")</td><td></td><td></td><td colspan=2></td></tr>"

    COMPOSER_VER=$(composer --version 2>/dev/null | head -1 || echo "not installed")
    echo "<tr><td>Composer</td><td colspan=5>$(h "$COMPOSER_VER")</td></tr>"

    RUBY_VER=$(ruby -v 2>/dev/null || echo "not installed")
    echo "<tr><td>Ruby</td><td colspan=5>$(h "$RUBY_VER")</td></tr>"

    CAP_VER=$(cap --version 2>/dev/null || gem list capistrano 2>/dev/null | grep capistrano || echo "not installed")
    echo "<tr><td>Capistrano</td><td colspan=5>$(h "$CAP_VER")</td></tr>"

    # =========================================================================
    # 9. SERVICES STATUS (all dynamically detected)
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>Service Status</b></td></tr>"

    # Detect PHP-FPM service dynamically
    PHP_FPM_SVC=$(systemctl list-units --type=service --all 2>/dev/null | grep -oE 'php[0-9.]+-fpm' | head -1)
    if [ -z "$PHP_FPM_SVC" ]; then
        PHP_FPM_SVC="php-fpm"
    fi

    # DB_SERVICE already detected in section 5; fallback
    if [ -z "$DB_SERVICE" ]; then
        DB_SERVICE="mariadb"
    fi

    for svc in apache2 "$PHP_FPM_SVC" "$DB_SERVICE"; do
        SVC_STATUS=$(systemctl is-active "$svc" 2>/dev/null || echo "unknown")
        SVC_ENABLED=$(systemctl is-enabled "$svc" 2>/dev/null || echo "unknown")
        echo "<tr><td>$(h "$svc")</td><td>$(h "$SVC_STATUS")</td><td>active</td>"
        if [ "$SVC_STATUS" = "active" ]; then
            echo "<td style='background:#c8e6c9'>OK</td>"
        else
            echo "<td style='background:#ffcdd2'>$(h "$SVC_STATUS")</td>"
        fi
        echo "<td colspan=2>Enabled: $(h "$SVC_ENABLED")</td></tr>"
    done

    # =========================================================================
    # 10. SECURITY BASICS
    # =========================================================================
    echo "<tr><td colspan=6 style='background:#e3f2fd'><b>Security</b></td></tr>"

    UFW_STATUS=$(sudo ufw status 2>/dev/null | head -1 || echo "ufw not available")
    echo "<tr><td>Firewall (UFW)</td><td colspan=5>$(h "$UFW_STATUS")</td></tr>"

    # Use apt-get simulate for reliable count
    UPDATES=$(apt-get -s upgrade 2>/dev/null | grep -c "^Inst " || echo "?")
    echo "<tr><td>Pending apt updates</td><td>$(h "$UPDATES")</td><td></td>"
    if [ "$UPDATES" -gt 20 ] 2>/dev/null; then
        echo "<td style='background:#fff9c4'>MANY</td>"
    else
        echo "<td style='background:#c8e6c9'>OK</td>"
    fi
    echo "<td colspan=2></td></tr>"

    if dpkg -l unattended-upgrades 2>/dev/null | grep -q "^ii"; then
        echo "<tr><td>Unattended Upgrades</td><td style='background:#c8e6c9' colspan=5>Installed</td></tr>"
    else
        echo "<tr><td>Unattended Upgrades</td><td style='background:#fff9c4' colspan=5>Not installed</td></tr>"
    fi

REMOTE_EOF
}

# =============================================================================
# HTML OUTPUT
# =============================================================================
echo "<!DOCTYPE html>"
echo "<html><head>"
echo "<meta charset='UTF-8'>"
echo "<title>Server Configuration Audit</title>"
echo "<style>"
echo "  body { font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; margin: 20px; background: #fafafa; }"
echo "  h1 { color: #1565c0; }"
echo "  h2 { color: #1976d2; margin-top: 30px; }"
echo "  table { border-collapse: collapse; width: 100%; margin-bottom: 30px; box-shadow: 0 1px 3px rgba(0,0,0,0.12); }"
echo "  td, th { border: 1px solid #ccc; padding: 6px 10px; font-size: 13px; }"
echo "  th { background: #e0e0e0; text-align: left; }"
echo "  tr:hover { background: #f5f5f5; }"
echo "  pre { font-size: 12px; }"
echo "  .legend { margin: 10px 0; }"
echo "  .legend span { padding: 3px 10px; margin-right: 10px; font-size: 12px; }"
echo "</style>"
echo "</head><body>"
echo "<h1>Server Configuration Audit Report</h1>"
echo "<p>Generated: $(date '+%Y-%m-%d %H:%M:%S')</p>"
echo "<div class='legend'>"
echo "  <span style='background:#c8e6c9'>OK</span>"
echo "  <span style='background:#fff9c4'>Warning</span>"
echo "  <span style='background:#ffcdd2'>Error / Action needed</span>"
echo "</div>"

echo "<h2>Audit Checks</h2>"
echo "<ul>"
echo "<li>PHP-FPM: pm.max_children vs available RAM + actual worker RSS, memory_limit, upload_max_filesize, OPcache</li>"
echo "<li>Apache: MaxRequestWorkers, MPM module, KeepAlive, Timeout, per-site ErrorLog paths</li>"
echo "<li>MariaDB: effective config via my_print_defaults + all .cnf files, innodb_buffer_pool_size, max_connections, slow query log</li>"
echo "<li>Error Logs: PHP and Apache log paths verified in Filebeat config + modules</li>"
echo "<li>Recent warnings: max_children hits, MaxRequestWorkers hits, timeouts, OOM kills</li>"
echo "<li>Versions: PHP, Node.js, npm, Composer, Ruby, Capistrano</li>"
echo "<li>Hardware: RAM, CPU, Disk, Swap</li>"
echo "<li>Services: apache2, php-fpm (auto-detected), mariadb/mysql (auto-detected)</li>"
echo "<li>Security: UFW firewall, pending updates, unattended-upgrades</li>"
echo "</ul>"

for server in "${SERVERS[@]}"; do
    echo "<h2>${server}</h2>"
    echo "<table>"
    audit_server "$server"
    echo "</table>"
done

echo "<h2>Recommendations</h2>"
echo "<table>"
echo "<tr><th>Item</th><th>Description</th></tr>"
echo "<tr><td>pm.max_children formula</td><td>Available RAM (80% of total) / avg worker RSS = recommended max_children. Falls back to memory_limit if no workers running (rough estimate).</td></tr>"
echo "<tr><td>MaxRequestWorkers</td><td>Should match or be slightly higher than pm.max_children for prefork MPM</td></tr>"
echo "<tr><td>Error logs &rarr; Elastic</td><td>Install Filebeat and configure inputs for /var/log/apache2/error.log and /var/log/php*-fpm.log, or enable the apache module</td></tr>"
echo "<tr><td>pm.max_requests</td><td>Set to 500 to prevent PHP memory leaks from accumulating</td></tr>"
echo "<tr><td>OPcache</td><td>Enable with opcache.memory_consumption=256, opcache.max_accelerated_files=10000</td></tr>"
echo "<tr><td>Slow query log</td><td>Enable MariaDB slow query log with long_query_time=2 for performance monitoring</td></tr>"
echo "</table>"

echo "</body></html>"