require("rootpath")();
const express = require("express");
const cors = require("cors");
const bodyParser = require("body-parser");
const helmet = require("helmet"); // Security headers
const compression = require("compression"); // Compress responses
const rateLimit = require("express-rate-limit"); // Rate limiting
const morgan = require("morgan"); // Logging
const path = require("path"); // Untuk resolve path folder
const errorHandler = require("middlewares/error-handler");
const healthCheck = require("middlewares/health-check");
const connectionResilience = require("middlewares/connection-resilience");
const ispAnalytics = require("middlewares/isp-analytics");
const connectionTracker = require("middlewares/connection-tracker");
const { getErrorHandler } = require("./utils/errorHandler");
const { getCrashRecovery } = require("./utils/crashRecovery");
// Load environment variables first (for cPanel deployment)
require("dotenv").config();

const config = require("./config.json");

// Basic Auth untuk dashboard monitoring (sangat ringan, tidak mengganggu sistem lain)
const DASHBOARD_USER = process.env.DASHBOARD_USER || config.backendConfig.dashboardUser || "admin";
const DASHBOARD_PASS = process.env.DASHBOARD_PASS || config.backendConfig.dashboardPass || "password";
function dashboardAuth(req, res, next) {
	const auth = req.headers.authorization;
	if (!auth || !auth.startsWith("Basic ")) {
		res.set("WWW-Authenticate", 'Basic realm="Dashboard"');
		return res.status(401).send("Authentication required.");
	}
	const [user, pass] = Buffer.from(auth.split(" ")[1], "base64").toString().split(":");
	if (user === DASHBOARD_USER && pass === DASHBOARD_PASS) return next();
	res.set("WWW-Authenticate", 'Basic realm="Dashboard"');
	return res.status(401).send("Invalid credentials.");
}

// Override config with environment variables (priority: env > config.json)
// This is important for cPanel deployment where environment variables are set by Application Manager
if (process.env.NODE_ENV) {
	config.backendConfig.NODE_ENV = process.env.NODE_ENV;
	config.databaseConfigSAPPS.NODE_ENV = process.env.NODE_ENV;
}

// Pastikan dashboardConfig ada (sumber: config.json; tanpa ENV)
if (!config.dashboardConfig) config.dashboardConfig = {};

// Database environment overrides (opsional; jika tidak pakai env, gunakan databaseConfigSAPPS di config)
if (process.env.DB_HOST) config.databaseConfigSAPPS.host = process.env.DB_HOST;
if (process.env.DB_PORT) config.databaseConfigSAPPS.port = parseInt(process.env.DB_PORT, 10);
if (process.env.DB_USER) config.databaseConfigSAPPS.user = process.env.DB_USER;
if (process.env.DB_PASSWORD) config.databaseConfigSAPPS.password = process.env.DB_PASSWORD;
if (process.env.DB_NAME) config.databaseConfigSAPPS.database = process.env.DB_NAME;

// PORT is set by cPanel Application Manager automatically
// If not set, use config.json port, or default to 3000
if (!process.env.PORT && config.backendConfig.port) {
	process.env.PORT = config.backendConfig.port.toString();
}

// =============== CRASH RECOVERY INITIALIZATION ===============
// Initialize crash recovery system BEFORE anything else
const crashRecovery = getCrashRecovery({
	maxRestarts: 10,
	restartWindow: 60000, // 1 minute
	restartDelay: 2000, // 2 seconds initial delay
	maxRestartDelay: 30000, // 30 seconds max delay
});

// Initialize enhanced error handler
const enhancedErrorHandler = getErrorHandler();

const app = express();

/* Proxy has been disabled per request */
app.set("trust proxy", false);

// ================= TRAFFIC COUNTER (REALTIME) =================
const trafficStats = {
	total: 0,
	perEndpoint: {},
	perMinute: {},
};
app.use((req, res, next) => {
	// Hitungan hanya untuk API (bisa disesuaikan)
	if (req.path.startsWith("/sapps/") || req.path.startsWith("/api/")) {
		trafficStats.total++;
		// Per endpoint
		trafficStats.perEndpoint[req.path] = (trafficStats.perEndpoint[req.path] || 0) + 1;
		// Per menit
		const minute = new Date().toISOString().slice(0, 16); // yyyy-mm-ddThh:mm
		trafficStats.perMinute[minute] = (trafficStats.perMinute[minute] || 0) + 1;
	}

	next();
});

// Periodic cleanup for trafficStats to prevent memory leak
setInterval(() => {
	// Clean perMinute (keep last 60 minutes)
	const minutes = Object.keys(trafficStats.perMinute).sort();
	if (minutes.length > 60) {
		const toRemove = minutes.slice(0, minutes.length - 60);
		toRemove.forEach(m => delete trafficStats.perMinute[m]);
	}

	// Safety check for perEndpoint (prevent explosion from scanning or dynamic paths)
	const endpoints = Object.keys(trafficStats.perEndpoint);
	if (endpoints.length > 2000) {
		console.warn("Traffic stats warning: excessive unique endpoints. Resetting endpoint stats to prevent memory leak.");
		trafficStats.perEndpoint = {};
	}
}, 60 * 60 * 1000); // Check every hour

// Endpoint untuk dashboard visual
app.get("/analytics/traffic", (req, res) => {
	res.json({
		total: trafficStats.total,
		perEndpoint: trafficStats.perEndpoint,
		perMinute: trafficStats.perMinute,
		ts: Date.now(),
	});
});
// Endpoint monitoring system info (uptime, memory, CPU, error rate, recent error log)
const os = require("os");
const fs = require("fs");
const { exec } = require("child_process");
const { promisify } = require("util");
const execAsync = promisify(exec);
const DISK_CHECK_TIMEOUT = 3000; // avoid blocking event loop in production (exec can be slow on shared hosts)
app.get("/analytics/sysinfo", async (req, res) => {
	// Uptime
	const uptime = process.uptime();
	// Memory
	const mem = process.memoryUsage();
	// CPU
	const cpus = os.cpus();
	const load = os.loadavg();
	// Node version
	const nodeVersion = process.version;
	// Error rate (dari trafficStats, error = status >= 400)
	if (!trafficStats.errorCount) trafficStats.errorCount = 0;
	// Recent error log (async read to avoid blocking event loop; was readFileSync and caused 503 under load)
	let recentErrors = [];
	try {
		const logPath = path.join(__dirname, "logs", "login-requests.log");
		const content = await fs.promises.readFile(logPath, "utf8");
		const lines = content.trim().split("\n");
		recentErrors = lines.slice(-10).map((l) => {
			try {
				return JSON.parse(l);
			} catch {
				return l;
			}
		});
	} catch (_) {}

	// Disk usage (cross-platform): async exec + timeout
	let disk = { total: null, used: null, free: null, percent: null, raw: null };
	try {
		if (process.platform === "linux" || process.platform === "darwin") {
			// Use df -P . to get the disk usage of the current directory (where the app is installed)
			// -P ensures POSIX output (one line per filesystem)
			const { stdout } = await execAsync("df -P .", { encoding: "utf8", timeout: DISK_CHECK_TIMEOUT });
			disk.raw = stdout;
			const lines = stdout.trim().split("\n");
			// Header is line 0, data is line 1
			// Filesystem 1024-blocks Used Available Capacity Mounted on
			if (lines[1]) {
				const parts = lines[1].split(/\s+/);
				// parts[1] = total (in 1K blocks), parts[2] = used, parts[3] = available, parts[4] = capacity percentage
				const totalBlocks = parseInt(parts[1], 10);
				const usedBlocks = parseInt(parts[2], 10);
				const freeBlocks = parseInt(parts[3], 10);
				
				disk.total = (totalBlocks / 1024 / 1024).toFixed(2) + " GB"; // 1K blocks -> GB
				disk.used = (usedBlocks / 1024 / 1024).toFixed(2) + " GB";
				disk.free = (freeBlocks / 1024 / 1024).toFixed(2) + " GB";
				disk.percent = parts[4]; // Already includes %
			}
		} else if (process.platform === "win32") {
            const currentDrive = path.parse(process.cwd()).root.replace('\\', ''); // e.g., "C:"
			const { stdout } = await execAsync("wmic logicaldisk get size,freespace,caption", { encoding: "utf8", timeout: DISK_CHECK_TIMEOUT });
			disk.raw = stdout;
			const lines = stdout.trim().split("\n");
            
            // WMIC output usually: Caption  FreeSpace  Size
            // We find the line that starts with our drive letter
            const driveLine = lines.find(l => l.trim().toUpperCase().startsWith(currentDrive.toUpperCase()));

			if (driveLine) {
				const parts = driveLine.trim().split(/\s+/);
				// Assuming order: Caption, FreeSpace, Size (alphabetical columns)
                // If the drive letter matches first part, then parts[1] is FreeSpace, parts[2] is Size
				const free = parseInt(parts[1], 10);
				const size = parseInt(parts[2], 10);
                
				disk.total = (size / 1024 / 1024 / 1024).toFixed(2) + " GB";
				disk.free = (free / 1024 / 1024 / 1024).toFixed(2) + " GB";
				disk.used = ((size - free) / 1024 / 1024 / 1024).toFixed(2) + " GB";
				disk.percent = size ? (((size - free) / size) * 100).toFixed(1) + "%" : null;
			}
		}
	} catch (e) {
		// fallback: gunakan os.freemem/totalmem (ini RAM, bukan disk, tapi di beberapa dashboard monitoring dicampur)
        // Kita beri label khusus jika fallback
		disk.total = (os.totalmem() / 1024 / 1024 / 1024).toFixed(2) + " GB (RAM)";
		disk.free = (os.freemem() / 1024 / 1024 / 1024).toFixed(2) + " GB (RAM)";
		disk.used = ((os.totalmem() - os.freemem()) / 1024 / 1024 / 1024).toFixed(2) + " GB (RAM)";
		disk.percent = (100 - (os.freemem() / os.totalmem()) * 100).toFixed(1) + "%";
        disk.error = e.message;
	}

	// Status database (readiness)
	let dbStatus = { ready: false, latencyMs: null, error: null };
	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		const start = Date.now();
		if (dbSAPPS.isReady && dbSAPPS.isReady()) {
			dbStatus.ready = true;
			if (dbSAPPS.sequelize && dbSAPPS.sequelize.authenticate) {
				await dbSAPPS.sequelize.authenticate();
				dbStatus.latencyMs = Date.now() - start;
			}
		} else {
			dbStatus.ready = false;
		}
	} catch (e) {
		dbStatus.error = e.message;
	}

	// Rekomendasi otomatis
	const rekomendasi = [];
	if (mem.heapUsed / mem.heapTotal > 0.85) rekomendasi.push("Memory usage tinggi, pertimbangkan restart service atau optimasi kode.");
	if (disk.percent && parseFloat(disk.percent) > 85) rekomendasi.push("Disk hampir penuh, hapus file tidak perlu atau tambah kapasitas.");
	if (dbStatus.latencyMs && dbStatus.latencyMs > 1000) rekomendasi.push("Database lambat (>1s), cek koneksi DB atau query lambat.");
	if (trafficStats.errorCount / (trafficStats.total || 1) > 0.05) rekomendasi.push("Tingkat error tinggi (>5%), cek log error dan perbaiki bug.");
	if (!dbStatus.ready) rekomendasi.push("Database tidak siap, cek status DB dan koneksi.");
    if (config.backendConfig.NODE_ENV !== "production") rekomendasi.push("Mode DEVELOPMENT aktif via config.json/env. Set NODE_ENV=production di Application Manager untuk performa optimal.");
	if (rekomendasi.length === 0) rekomendasi.push("Semua sistem dalam kondisi normal.");

	res.json({
		uptime,
		platform: process.platform,
		memory: mem,
		cpu: {
			count: cpus.length,
			model: cpus[0]?.model,
			speed: cpus[0]?.speed,
			load,
		},
		nodeVersion,
		errorCount: trafficStats.errorCount,
		errorRate: trafficStats.total ? trafficStats.errorCount / trafficStats.total : 0,
		recentErrors,
		disk,
		dbStatus,
		rekomendasi,
		ts: Date.now(),
	});
});

// ================= SUPER LIGHTWEIGHT STATUS ENDPOINTS (EARLY RETURN) =================
// These endpoints are placed BEFORE any heavy middleware, DB, or logging
// They do NOT hit DB, do NOT log, and are safe for health checks, monitoring, and keepalive
const statusUptime = { started: Date.now() };
app.get("/health", (req, res) => res.status(200).send("OK"));
app.head("/health", (req, res) => res.status(200).end());
app.get("/ping", (req, res) => res.status(200).json({ ok: true, ts: Date.now(), uptime: Math.floor((Date.now() - statusUptime.started) / 1000) }));
app.get("/health-keepalive", (req, res) =>
	res.status(200).json({ ok: true, ts: Date.now(), uptime: Math.floor((Date.now() - statusUptime.started) / 1000), keepalive: true }),
);
// Diagnostics/analytics endpoints (tanpa DB, hanya info statis)
app.get("/diagnostics", (req, res) =>
	res.status(200).json({ ok: true, ts: Date.now(), env: process.env.NODE_ENV, uptime: Math.floor((Date.now() - statusUptime.started) / 1000) }),
);
app.get("/analytics/isp", (req, res) => res.status(200).json({ ok: true, ts: Date.now(), note: "lightweight analytics endpoint" }));
app.get("/diagnostics/ping", (req, res) => res.status(200).json({ ok: true, ts: Date.now(), env: process.env.NODE_ENV }));
app.get("/diagnostics/statuses", (req, res) =>
	res.status(200).json({ ok: true, ts: Date.now(), env: process.env.NODE_ENV, uptime: Math.floor((Date.now() - statusUptime.started) / 1000) }),
);
// Dashboard polling interval (from config/ENV); lightweight, no DB
app.get("/api/dashboard-config", (req, res) => {
	const raw = config.dashboardConfig?.pollIntervalMs ?? 10000;
	const ms = parseInt(String(raw), 10) || 10000;
	const pollIntervalMs = Math.max(5000, Math.min(60000, ms));
	res.status(200).json({ pollIntervalMs });
});

// If you want to keep the original endpoints for deeper diagnostics, rename them (e.g. /diagnostics-full)

// =============== ISP BLOCKING MITIGATION ===============
// Track requests for ISP blocking detection
app.use(ispAnalytics.trackRequest);

// IP Connection Tracking (Enhanced)
app.use(connectionTracker);

// =============== NETWORK CONNECTIVITY MONITORING ===============
// Initialize network connectivity monitor
const { getNetworkConnectivityMonitor } = require("./utils/networkConnectivityMonitor");
const networkConnectivityMonitor = getNetworkConnectivityMonitor({
	checkInterval: 60000, // 1 minute
	historySize: 1000,
	enableISPDetection: true,
	enableLatencyTracking: true
});

// Track connection attempts with ISP detection
app.use((req, res, next) => {
	const startTime = Date.now();
	const clientIP = req.ip || req.socket?.remoteAddress || req.connection?.remoteAddress || "unknown";
	
	// Track requests (for API calls that indicate backend connectivity)
	res.on("finish", () => {
		if (req.path.startsWith("/sapps/") || req.path.startsWith("/api/")) {
			const success = res.statusCode >= 200 && res.statusCode < 400;
			const responseTime = Date.now() - startTime;
			
			// Only track if network connectivity monitor is available
			if (global.networkConnectivityMonitor) {
				global.networkConnectivityMonitor.trackConnectionAttempt(
					clientIP, 
					success, 
					responseTime, 
					success ? null : `HTTP ${res.statusCode}`
				);
			}
		}
	});
	
	next();
});

// Add request tracking metrics
app.use(healthCheck.trackRequestMetrics);

// Configure connection resilience
connectionResilience.configureConnectionResilience(app);

// Add retry headers
app.use(connectionResilience.addRetryHeaders);

// Optimize transmission
app.use(connectionResilience.optimizeTransmission);

// ================= SECURITY ===================
app.use(
	helmet({
		crossOriginEmbedderPolicy: false,
		crossOriginResourcePolicy: false,
		contentSecurityPolicy: false,
	}),
);

// ================ COMPRESSION ==================
app.use(
	compression({
		level: 6,
		threshold: 1024,
	}),
);

// =================== CORS ======================
// Enhanced CORS configuration to bypass ISP restrictions
// Allow localhost for development even in production (for local frontend dev)
const corsOptions = {
	origin: function (origin, callback) {
		// Allow requests with no origin (mobile apps, Postman, etc.)
		if (!origin) return callback(null, true);

		// Base production origins
		const productionOrigins = [
			config.frontendConfig.url,
			"https://sapps.sapharma.co.id",
			"https://sapps-uat.sapharma.co.id",
			"https://sapps-backup.sapharma.co.id",
		];

		// Localhost origins for development (allowed in both dev and production)
		const localhostOrigins = ["http://localhost:3000", "http://localhost:5173", "http://127.0.0.1:3000", "http://127.0.0.1:5173"];

		// Combine origins based on environment
		const allowedOrigins =
			config.backendConfig.NODE_ENV === "production"
				? [...productionOrigins, ...localhostOrigins] // Allow localhost in production for local dev
				: [...productionOrigins, ...localhostOrigins];

		// Normalize origin (remove trailing slash)
		const normalizedOrigin = origin.replace(/\/$/, "");

		// Check for exact match (case-sensitive for protocol/host, but allow localhost variations)
		const isAllowed = allowedOrigins.some((allowed) => {
			const normalizedAllowed = allowed.replace(/\/$/, "");
			// Exact match (case-sensitive)
			if (normalizedOrigin === normalizedAllowed) {
				return true;
			}
			// Case-insensitive match for localhost/127.0.0.1
			if (normalizedOrigin.toLowerCase().includes("localhost") || normalizedOrigin.toLowerCase().includes("127.0.0.1")) {
				return normalizedOrigin.toLowerCase() === normalizedAllowed.toLowerCase();
			}
			// StartsWith for production URLs (to handle subdomains and paths)
			return normalizedOrigin.startsWith(normalizedAllowed);
		});

		if (isAllowed) {
			callback(null, true);
		} else {
			// Log unauthorized origin for monitoring but still allow (for development flexibility)
			console.warn(`[CORS] Unauthorized origin: ${origin} - Allowed origins: ${allowedOrigins.join(", ")}`);
			callback(null, true);
		}
	},
	// Allow credentials for both development and production (needed for auth cookies)
	credentials: true,
	methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "PATCH"],
	allowedHeaders: [
		"Content-Type",
		"Authorization",
		"X-Requested-With",
		"X-Request-ID",
		"Cache-Control",
		"Pragma",
		// Concurrency and idempotency headers used by client
		"If-Match",
		"X-Record-Version",
		"X-Idempotency-Key",
	],
	exposedHeaders: ["Content-Length", "X-Request-ID", "X-Retry-After", "ETag", "X-Record-Version"],
	maxAge: 86400, // 24 hours - reduce preflight requests
	preflightContinue: false,
	optionsSuccessStatus: 200, // Use 200 instead of 204 for better compatibility
};
// Explicitly handle preflight for all routes before any other middleware
app.options("*", cors(corsOptions));
app.use(cors(corsOptions));

// ================= RATE LIMIT ===================
const limiter = rateLimit({
	windowMs: 15 * 60 * 1000,
	max: 10000000,
	standardHeaders: true,
	legacyHeaders: false,

	/* FIX ERROR:
     The 'X-Forwarded-For' header is set but trust proxy is false */
	validate: { xForwardedForHeader: false },

	/* skip static folder agar upload/download tidak dibatasi */
	skip: (req) => {
		const staticPaths = ["/avatars", "/logos", "/documents", "/images", "/videos", "/audios", "/others"];
		return staticPaths.some((p) => req.path.startsWith(p));
	},
});
app.use(limiter);

// ================= LOGGING ======================
app.use(config.backendConfig.NODE_ENV === "development" ? morgan("dev") : morgan("combined"));

// ================= BODY PARSER ==================
// FIXED: Increased limit untuk support bulk import operations
// JSON limit increased to 50mb untuk import-all-item operations
app.use(
	bodyParser.json({
		limit: "50mb", // Increased from 1mb to support bulk imports
		type: "application/json",
	}),
);
app.use(
	bodyParser.urlencoded({
		extended: true,
		limit: "50mb", // Increased from 10mb to match JSON limit
	}),
);

// ================= STATIC FILE CORS ===============
const staticCORS = (req, res, next) => {
	// Enhanced CORS for static files
	// Allow localhost in both development and production for local frontend dev
	// Use same robust matching logic as main CORS for consistency
	const allowedOrigins = [
		config.frontendConfig.url,
		"https://sapps.sapharma.co.id",
		"https://sapps-uat.sapharma.co.id",
		"https://sapps-backup.sapharma.co.id",
		"http://localhost:3000",
		"http://localhost:5173",
		"http://127.0.0.1:3000",
		"http://127.0.0.1:5173",
	];

	const origin = req.headers.origin;

	// Use same robust matching logic as main CORS
	if (origin) {
		// Normalize origin (remove trailing slash)
		const normalizedOrigin = origin.replace(/\/$/, "");

		// Check for exact match (case-sensitive for protocol/host, but allow localhost variations)
		const isAllowed = allowedOrigins.some((allowed) => {
			const normalizedAllowed = allowed.replace(/\/$/, "");
			// Exact match (case-sensitive)
			if (normalizedOrigin === normalizedAllowed) {
				return true;
			}
			// Case-insensitive match for localhost/127.0.0.1
			if (normalizedOrigin.toLowerCase().includes("localhost") || normalizedOrigin.toLowerCase().includes("127.0.0.1")) {
				return normalizedOrigin.toLowerCase() === normalizedAllowed.toLowerCase();
			}
			// StartsWith for production URLs (to handle subdomains and paths)
			return normalizedOrigin.startsWith(normalizedAllowed);
		});

		if (isAllowed) {
			res.header("Access-Control-Allow-Origin", origin);
			res.header("Access-Control-Allow-Credentials", "true");
		} else if (config.backendConfig.NODE_ENV === "development") {
			// In development, allow all origins
			res.header("Access-Control-Allow-Origin", "*");
		} else {
			// In production, log but still allow for flexibility
			console.warn(`[Static CORS] Unauthorized origin: ${origin} - Allowed origins: ${allowedOrigins.join(", ")}`);
			res.header("Access-Control-Allow-Origin", origin);
			res.header("Access-Control-Allow-Credentials", "true");
		}
	} else if (config.backendConfig.NODE_ENV === "development") {
		// No origin header (e.g., Postman, mobile apps) - allow in development
		res.header("Access-Control-Allow-Origin", "*");
	} else {
		// No origin header in production - allow but log
		res.header("Access-Control-Allow-Origin", "*");
		res.header("Access-Control-Allow-Credentials", "true");
	}

	res.header("Access-Control-Allow-Methods", "GET, OPTIONS, HEAD");
	res.header("Access-Control-Allow-Headers", "Content-Type, Range, Authorization");
	res.header("Cross-Origin-Resource-Policy", "cross-origin");
	res.header("Cross-Origin-Embedder-Policy", "unsafe-none");
	res.header("Access-Control-Expose-Headers", "Content-Length, Content-Range");
	res.header("Cache-Control", "public, max-age=31536000"); // Cache static files for 1 year
	next();
};

// Proteksi akses dashboard.html dan asset dashboard (tidak mengganggu sistem lain)
app.use("/public/dashboard.html", dashboardAuth);
app.use("/public/dashboard-enhanced.html", dashboardAuth);
app.use("/public/dashboard", dashboardAuth); // jika ada asset tambahan
const publicDataPath = path.join(__dirname, "public/data");

app.use("/avatars", staticCORS, express.static(path.join(publicDataPath, "avatars")));
app.use("/logos", staticCORS, express.static(path.join(publicDataPath, "logos")));
app.use("/documents", staticCORS, express.static(path.join(publicDataPath, "documents")));
app.use("/images", staticCORS, express.static(path.join(publicDataPath, "images")));
app.use("/videos", staticCORS, express.static(path.join(publicDataPath, "videos")));
app.use("/audios", staticCORS, express.static(path.join(publicDataPath, "audios")));
app.use("/others", staticCORS, express.static(path.join(publicDataPath, "others")));

app.use(express.static(path.join(__dirname, "public")));

// ================= API CACHE CONTROL =================
// Ensure API responses are never cached by browsers or proxies
app.use(async (req, res, next) => {
	const isApi = req.path.startsWith("/sapps/") || req.path.startsWith("/api/");
	if (isApi && req.method !== "OPTIONS") {
		res.set("Cache-Control", "no-store, no-cache, must-revalidate, proxy-revalidate");
		res.set("Pragma", "no-cache");
		res.set("Expires", "0");
		res.set("Surrogate-Control", "no-store");
		// Prevent cached responses tied to auth from leaking
		res.set("Vary", "Authorization");
	}
	next();
});

// =============== HEALTH CHECK & DIAGNOSTICS ===============
// Basic health check endpoint
app.get("/health", (req, res) => {
	res.status(200).json(healthCheck.getHealthStatus());
});

// Endpoint khusus untuk cPanel keep-alive (ping setiap 5 menit)
// Bisa dipanggil oleh cron job cPanel: curl -s https://sapps.sapharma.co.id/health-keepalive
app.get("/health-keepalive", (req, res) => {
	res.status(200).json({ ok: true, ts: Date.now(), uptime: process.uptime(), keepalive: true });
});

// Contoh cron job cPanel (setiap 5 menit):
// */5 * * * * curl -s https://sapps.sapharma.co.id/health-keepalive > /dev/null

// Fast health check for connectivity probes
app.head("/health", (req, res) => {
	res.status(200).end();
});

// Detailed diagnostics endpoint
app.get("/diagnostics", (req, res) => {
	res.status(200).json(healthCheck.getConnectionDiagnostics(req));
});

// ISP analytics endpoint
app.get("/analytics/isp", (req, res) => {
	res.status(200).json({
		summary: ispAnalytics.getAnalyticsSummary(),
		patterns: ispAnalytics.getISPPatternReport(),
	});
});

// Network connectivity monitoring endpoint
app.get("/analytics/network-connectivity", (req, res) => {
	try {
		if (!global.networkConnectivityMonitor) {
			return res.status(503).json({
				success: false,
				message: "Network connectivity monitor not initialized",
				timestamp: new Date().toISOString()
			});
		}

		const status = global.networkConnectivityMonitor.getStatus();
		const ispStats = global.networkConnectivityMonitor.getISPStatistics();
		const history = global.networkConnectivityMonitor.getHistory(100);

		res.status(200).json({
			success: true,
			status,
			ispStatistics: ispStats,
			recentHistory: history,
			timestamp: new Date().toISOString()
		});
	} catch (err) {
		res.status(500).json({
			success: false,
			error: "Failed to get network connectivity status",
			message: err.message,
			timestamp: new Date().toISOString()
		});
	}
});

// Quick diagnostics ping
app.get("/diagnostics/ping", (req, res) => {
	res.status(200).json({ ok: true, ts: Date.now(), env: process.env.NODE_ENV });
});

// Surface DB env (safe) and readiness snapshot
app.get("/diagnostics/statuses", (req, res) => {
	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		const sapps = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : undefined;
		res.status(200).json({
			ok: true,
			sapps,
			uptime: process.uptime(),
			dbEnv: {
				host: process.env.DB_HOST,
				port: process.env.DB_PORT,
				name: process.env.DB_NAME,
				user: process.env.DB_USER ? "**set**" : undefined,
				ssl: process.env.DB_SSL,
				sslRejectUnauthorized: process.env.DB_SSL_REJECT_UNAUTHORIZED,
				timezone: process.env.DB_TIMEZONE,
			},
		});
	} catch (err) {
		res.status(500).json({ ok: false, error: err.message });
	}
});

// Live DB connection diagnostic
app.get("/diagnostics/db-test", async (req, res) => {
	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		const statusBefore = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : undefined;
		if (dbSAPPS.sequelize && dbSAPPS.sequelize.authenticate) {
			await dbSAPPS.sequelize.authenticate();
			return res.status(200).json({ ok: true, mode: "existing", status: statusBefore });
		}
		if (dbSAPPS.dbConnectionManager && 
			dbSAPPS.dbConnectionManager.ensureConnection &&
			(!dbSAPPS.dbConnectionManager.isConnectionManagerClosed || !dbSAPPS.dbConnectionManager.isConnectionManagerClosed())) {
			const conn = await dbSAPPS.dbConnectionManager.ensureConnection();
			await conn.authenticate();
			return res.status(200).json({ ok: true, mode: "ensured" });
		}
		return res.status(200).json({ ok: true, note: "Connection manager not available; background retry active" });
	} catch (err) {
		res.status(500).json({ ok: false, error: err.message, code: err.code });
	}
});

// Deploy verification: 200 hanya jika DB siap. Untuk pengecekan pasca-deploy.
app.get("/ready", async (req, res) => {
	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		const ok = dbSAPPS.isReady && dbSAPPS.isReady();
		if (ok) {
			return res.status(200).json({ ready: true, ts: Date.now() });
		}
		res.status(503).json({ ready: false, ts: Date.now() });
	} catch (err) {
		res.status(503).json({ ready: false, error: err.message, ts: Date.now() });
	}
});

// Crash recovery stats endpoint (for monitoring)
app.get("/analytics/crashes", (req, res) => {
	try {
		const stats = crashRecovery.getCrashStats();
		res.status(200).json({
			status: "ok",
			crashStats: stats,
			timestamp: new Date().toISOString(),
		});
	} catch (err) {
		res.status(500).json({
			error: "Failed to get crash stats",
			message: err.message,
		});
	}
});

// Client log forwarding endpoint (async write agar tidak blocking event loop)
app.post("/analytics/client-logs", (req, res) => {
	try {
		const fs = require("fs");
		const path = require("path");
		const logDir = path.join(__dirname, "logs");
		const logFile = path.join(logDir, "client-logs.log");
		if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
		const payload = {
			timestamp: new Date().toISOString(),
			ip: healthCheck.getClientIP(req),
			body: req.body,
		};
		fs.appendFile(logFile, JSON.stringify(payload) + "\n", "utf8", (e) => {
			if (e) console.error("[client-logs] write error:", e.message);
		});
		res.status(200).json({ status: "ok" });
	} catch (err) {
		res.status(500).json({ status: "error", message: err.message });
	}
});

// Database connection stats endpoint (for monitoring)
app.get("/analytics/database", (req, res) => {
	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		if (dbSAPPS.dbConnectionManager) {
			const stats = dbSAPPS.dbConnectionManager.getConnectionStats();
			const initStatus = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : null;
			
			// Enhanced stats with initialization status
			const enhancedStats = {
				...stats,
				isReady: dbSAPPS.isReady && dbSAPPS.isReady() ? true : false,
				isInitialized: initStatus?.isInitialized || false,
				hasError: !!initStatus?.initializationError,
				error: initStatus?.initializationError ? {
					message: initStatus.initializationError.message,
					code: initStatus.initializationError.code
				} : null
			};
			
			res.status(200).json({
				status: "ok",
				databaseStats: enhancedStats,
				timestamp: new Date().toISOString(),
			});
		} else {
			res.status(503).json({
				status: "error",
				message: "Database connection manager not initialized",
				timestamp: new Date().toISOString(),
			});
		}
	} catch (err) {
		res.status(500).json({
			error: "Failed to get database stats",
			message: err.message,
			timestamp: new Date().toISOString(),
		});
	}
});

// Associate monitoring endpoint (for monitoring model associations)
app.get("/analytics/associations", (req, res) => {
	try {
		const associateStatus = healthCheck.getAssociateStatus();
		const statusCode = associateStatus.status === "healthy" ? 200 : associateStatus.status === "degraded" ? 207 : 503; // 207 = Multi-Status

		res.status(statusCode).json({
			status: associateStatus.status,
			associations: associateStatus.monitoring || associateStatus,
			verification: associateStatus.verification,
			timestamp: new Date().toISOString(),
		});
	} catch (err) {
		res.status(500).json({
			status: "error",
			error: "Failed to get associate status",
			message: err.message,
			timestamp: new Date().toISOString(),
		});
	}
});

// Database initialization status endpoint (for debugging in cPanel)
app.get("/analytics/initialization", (req, res) => {
	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		if (typeof dbSAPPS.getInitializationStatus === "function") {
			const initStatus = dbSAPPS.getInitializationStatus();
			const statusCode = initStatus.isReady ? 200 : initStatus.isInitialized ? 207 : 503;

			res.status(statusCode).json({
				status: initStatus.isReady ? "ready" : initStatus.isInitialized ? "degraded" : "not_ready",
				initialization: initStatus,
				timestamp: new Date().toISOString(),
			});
		} else {
			res.status(503).json({
				status: "error",
				message: "Initialization status function not available",
				timestamp: new Date().toISOString(),
			});
		}
	} catch (err) {
		res.status(500).json({
			status: "error",
			error: "Failed to get initialization status",
			message: err.message,
			timestamp: new Date().toISOString(),
		});
	}
});

// Enhanced database status endpoint for frontend (more user-friendly)
// Enhanced database status endpoint for frontend (more user-friendly)
app.get("/sapps/api/database-status", (req, res) => {
	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		const initStatus = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : null;
		const isReady = dbSAPPS.isReady && dbSAPPS.isReady();
		
		// Get network diagnostic info if there's an error
		let networkDiagnostic = null;
		if (initStatus?.initializationError) {
			const NetworkDiagnostics = require("./utils/networkDiagnostics");
			networkDiagnostic = NetworkDiagnostics.getNetworkErrorInfo(initStatus.initializationError);
		}

		// Get network monitor status if available
		let networkMonitorStatus = null;
		if (global.networkManager) {
			const managerStatus = global.networkManager.getStatus();
			networkMonitorStatus = {
				...managerStatus.monitor,
				diagnostic: managerStatus.diagnostic,
				autoRecovery: managerStatus.autoRecovery
			};
		} else if (global.networkMonitor) {
			const monitorStatus = global.networkMonitor.getStatus();
			networkMonitorStatus = {
				isOnline: monitorStatus.isOnline,
				isOffline: monitorStatus.isOffline,
				lastCheck: monitorStatus.lastCheck,
				lastSuccess: monitorStatus.lastSuccess,
				lastFailure: monitorStatus.lastFailure,
				consecutiveFailures: monitorStatus.consecutiveFailures,
				currentError: monitorStatus.currentError,
				diagnostic: global.networkMonitor.getDiagnostic()
			};
		}

		res.status(200).json({
			success: true,
			isReady,
			isInitialized: initStatus?.isInitialized || false,
			hasError: !!initStatus?.initializationError,
			error: initStatus?.initializationError ? {
				message: initStatus.initializationError.message,
				code: initStatus.initializationError.code
			} : null,
			networkDiagnostic,
			networkMonitor: networkMonitorStatus
		});
	} catch (err) {
		res.status(500).json({
			success: false,
			error: err.message
		});
	}
});

// Network diagnostic endpoint
app.get("/diagnostics/network", async (req, res) => {
	try {
		const NetworkDiagnostics = require("./utils/networkDiagnostics");
		const host = config.databaseConfigSAPPS.host;
		const port = config.databaseConfigSAPPS.port;
		
		const diagnostic = await NetworkDiagnostics.diagnose(host, port, 10000);
		
		// Include network monitor status if available
		if (global.networkManager) {
			diagnostic.managerStatus = global.networkManager.getStatus();
		} else if (global.networkMonitor) {
			diagnostic.monitorStatus = global.networkMonitor.getStatus();
		}
		
		const statusCode = diagnostic.success ? 200 : 503;
		res.status(statusCode).json(diagnostic);
	} catch (err) {
		res.status(500).json({
			success: false,
			error: "Failed to run network diagnostic",
			message: err.message,
			timestamp: new Date().toISOString(),
		});
	}
});

// Network status endpoint (from monitor)
app.get("/diagnostics/network-status", (req, res) => {
	try {
		if (!global.networkManager && !global.networkMonitor) {
			return res.status(503).json({
				success: false,
				message: "Network manager/monitor not initialized",
				timestamp: new Date().toISOString()
			});
		}

		let status, diagnostic;
		if (global.networkManager) {
			const managerStatus = global.networkManager.getStatus();
			status = managerStatus.monitor || managerStatus;
			diagnostic = managerStatus.diagnostic;
		} else {
			status = global.networkMonitor.getStatus();
			diagnostic = global.networkMonitor.getDiagnostic();
		}
		
		res.status(200).json({
			success: true,
			status: {
				isOnline: status.isOnline,
				isOffline: status.isOffline,
				status: status.isOnline === true ? 'online' : status.isOnline === false ? 'offline' : 'unknown',
				lastCheck: status.lastCheck,
				lastSuccess: status.lastSuccess,
				lastFailure: status.lastFailure,
				consecutiveSuccesses: status.consecutiveSuccesses,
				consecutiveFailures: status.consecutiveFailures,
				totalChecks: status.totalChecks,
				totalSuccesses: status.totalSuccesses,
				totalFailures: status.totalFailures,
				isMonitoring: status.isMonitoring
			},
			diagnostic: diagnostic || null,
			currentError: status.currentError,
			timestamp: new Date().toISOString()
		});
	} catch (err) {
		res.status(500).json({
			success: false,
			error: "Failed to get network status",
			message: err.message,
			timestamp: new Date().toISOString(),
		});
	}
});

// Force network check endpoint
app.post("/diagnostics/network-check", async (req, res) => {
	try {
		if (!global.networkManager && !global.networkMonitor) {
			return res.status(503).json({
				success: false,
				message: "Network manager/monitor not initialized",
				timestamp: new Date().toISOString()
			});
		}

		const result = global.networkManager 
			? await global.networkManager.forceCheck()
			: await global.networkMonitor.forceCheck();
		
		res.status(result.success ? 200 : 503).json({
			success: result.success,
			result,
			timestamp: new Date().toISOString()
		});
	} catch (err) {
		res.status(500).json({
			success: false,
			error: "Failed to force network check",
			message: err.message,
			timestamp: new Date().toISOString(),
		});
	}
});

// Comprehensive system health check endpoint (combines all diagnostics)
app.get("/diagnostics/system-health", async (req, res) => {
	try {
		const health = {
			timestamp: new Date().toISOString(),
			server: {
				status: "running",
				uptime: Math.floor(process.uptime()),
				environment: config.backendConfig.NODE_ENV || process.env.NODE_ENV || "development",
				port: START_PORT
			},
			database: null,
			network: null,
			overall: {
				status: "unknown",
				healthy: false,
				issues: []
			}
		};

		// Database status
		try {
			const dbSAPPS = require("./apps/sapps/helpers/db");
			const initStatus = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : null;
			const isReady = dbSAPPS.isReady && dbSAPPS.isReady();
			
			let networkDiagnostic = null;
			if (initStatus?.initializationError) {
				const NetworkDiagnostics = require("./utils/networkDiagnostics");
				networkDiagnostic = NetworkDiagnostics.getNetworkErrorInfo(initStatus.initializationError);
			}

			health.database = {
				status: isReady ? "ready" : initStatus?.isInitialized ? "initializing" : "not_ready",
				isReady,
				isInitialized: initStatus?.isInitialized || false,
				hasError: !!initStatus?.initializationError,
				error: initStatus?.initializationError ? {
					message: initStatus.initializationError.message,
					code: initStatus.initializationError.code,
					networkDiagnostic
				} : null
			};

			if (!isReady) {
				health.overall.issues.push({
					component: "database",
					severity: "critical",
					message: networkDiagnostic?.userMessage || initStatus?.initializationError?.message || "Database is not ready",
					suggestion: networkDiagnostic?.suggestion || "Please check database configuration and network connectivity"
				});
			}
		} catch (dbError) {
			health.database = {
				status: "error",
				error: dbError.message
			};
			health.overall.issues.push({
				component: "database",
				severity: "critical",
				message: `Database check failed: ${dbError.message}`
			});
		}

		// Network status
		try {
			if (global.networkManager) {
				const managerStatus = global.networkManager.getStatus();
				const monitorStatus = managerStatus.monitor || managerStatus;
				const diagnostic = managerStatus.diagnostic;
				
				health.network = {
					status: monitorStatus.isOnline === true ? "online" : monitorStatus.isOnline === false ? "offline" : "unknown",
					isOnline: monitorStatus.isOnline,
					isOffline: monitorStatus.isOffline,
					lastCheck: monitorStatus.lastCheck,
					lastSuccess: monitorStatus.lastSuccess,
					lastFailure: monitorStatus.lastFailure,
					consecutiveSuccesses: monitorStatus.consecutiveSuccesses,
					consecutiveFailures: monitorStatus.consecutiveFailures,
					totalChecks: monitorStatus.totalChecks,
					totalSuccesses: monitorStatus.totalSuccesses,
					totalFailures: monitorStatus.totalFailures,
					currentError: monitorStatus.currentError,
					diagnostic: diagnostic || null,
					isMonitoring: monitorStatus.isMonitoring,
					autoRecovery: managerStatus.autoRecovery || null
				};

				if (monitorStatus.isOnline === false) {
					health.overall.issues.push({
						component: "network",
						severity: "critical",
						message: diagnostic?.userMessage || "Network connectivity to database server is offline",
						suggestion: diagnostic?.suggestion || "Please check network connectivity and firewall rules"
					});
				} else if (monitorStatus.isOnline === null) {
					health.overall.issues.push({
						component: "network",
						severity: "warning",
						message: "Network status is unknown",
						suggestion: "Network monitoring is still initializing"
					});
				}
			} else if (global.networkMonitor) {
				// Fallback to old monitor if available
				const monitorStatus = global.networkMonitor.getStatus();
				const diagnostic = global.networkMonitor.getDiagnostic();
				
				health.network = {
					status: monitorStatus.isOnline === true ? "online" : monitorStatus.isOnline === false ? "offline" : "unknown",
					isOnline: monitorStatus.isOnline,
					isOffline: monitorStatus.isOffline,
					lastCheck: monitorStatus.lastCheck,
					diagnostic: diagnostic || null,
					isMonitoring: monitorStatus.isMonitoring
				};
			} else {
				health.network = {
					status: "not_monitored",
					message: "Network manager/monitor not initialized"
				};
				health.overall.issues.push({
					component: "network",
					severity: "warning",
					message: "Network monitoring is not active"
				});
			}
		} catch (netError) {
			health.network = {
				status: "error",
				error: netError.message
			};
			health.overall.issues.push({
				component: "network",
				severity: "warning",
				message: `Network check failed: ${netError.message}`
			});
		}

		// Determine overall health
		const hasCriticalIssues = health.overall.issues.some(issue => issue.severity === "critical");
		const hasWarnings = health.overall.issues.some(issue => issue.severity === "warning");
		
		if (hasCriticalIssues) {
			health.overall.status = "unhealthy";
			health.overall.healthy = false;
		} else if (hasWarnings) {
			health.overall.status = "degraded";
			health.overall.healthy = false;
		} else {
			health.overall.status = "healthy";
			health.overall.healthy = true;
		}

		const statusCode = health.overall.healthy ? 200 : hasCriticalIssues ? 503 : 207; // 207 = Multi-Status
		res.status(statusCode).json(health);
	} catch (err) {
		res.status(500).json({
			success: false,
			error: "Failed to get system health",
			message: err.message,
			timestamp: new Date().toISOString(),
		});
	}
});

// Simple ping endpoint for connectivity tests
app.get("/ping", (req, res) => {
	res.status(200).json({
		status: "ok",
		message: "pong",
		timestamp: new Date().toISOString(),
		yourIP: healthCheck.getClientIP(req),
	});
});

// =============== DATABASE READINESS CHECK ===============
// Middleware to ensure database is initialized before processing requests
// IMPROVED: More graceful handling with better error messages and status information
app.use(async (req, res, next) => {
	// Skip health check endpoints and statuses (uses in-memory cache)
	if (
		req.path === "/health" ||
		req.path === "/ping" ||
		req.path === "/diagnostics" ||
		req.path.startsWith("/analytics") ||
		req.path === "/sapps/api/statuses"
	) {
		return next();
	}

	// Always allow CORS preflight to pass
	if (req.method === "OPTIONS") {
		return next();
	}

	const startTime = Date.now();
	let timeoutTriggered = false;
	const TIMEOUT_MS = 8000; // Reduced to 8 seconds for faster failure detection
	const timer = setTimeout(() => {
		timeoutTriggered = true;
		console.error(`[Readiness Check][TIMEOUT] Request to ${req.method} ${req.path} waiting DB readiness > ${TIMEOUT_MS}ms`);
	}, TIMEOUT_MS);

	try {
		const dbSAPPS = require("./apps/sapps/helpers/db");
		
		// Get initialization status for better error messages
		const initStatus = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : null;
		const isReady = dbSAPPS.isReady && dbSAPPS.isReady();

		// If not ready, try a quick on-demand connection before failing
		if (!isReady) {
			try {
				if (dbSAPPS.dbConnectionManager && 
					dbSAPPS.dbConnectionManager.ensureConnection &&
					(!dbSAPPS.dbConnectionManager.isConnectionManagerClosed || !dbSAPPS.dbConnectionManager.isConnectionManagerClosed())) {
					// Attempt a fast ensure (up to ~5s inside helper)
					await dbSAPPS.dbConnectionManager.ensureConnection();
					// Re-check after ensureConnection
					if (dbSAPPS.isReady && dbSAPPS.isReady()) {
						clearTimeout(timer);
						if (!timeoutTriggered) {
							console.log(`[Readiness Check][${req.method} ${req.path}] PASSED (after ensureConnection) in ${Date.now() - startTime}ms`);
							return next();
						}
					}
				}
			} catch (e) {
				// Ignore errors if connection manager is closed
				if (e.message && !e.message.includes("closed")) {
					console.warn(`[Readiness Check] ensureConnection failed: ${e.message}`);
				}
			}

			// Still not ready after retry
			if (!dbSAPPS.isReady || !dbSAPPS.isReady()) {
				console.warn(`[Readiness Check] Database not ready for request: ${req.method} ${req.path}`);
				clearTimeout(timer);
				
				// Provide detailed error information
				const errorInfo = {
					success: false,
					message: "Database is not ready. Please check database connection and try again.",
					status: "database_not_ready",
					timestamp: new Date().toISOString(),
					details: {
						isInitialized: initStatus?.isInitialized || false,
						isReady: false,
						hasError: !!initStatus?.initializationError,
						errorMessage: initStatus?.initializationError?.message || null,
						suggestion: initStatus?.initializationError 
							? "Database connection failed. Please check database configuration and network connectivity."
							: "Database is still initializing. This may take a few moments."
					}
				};

				// Add retry-after header to help frontend with backoff
				res.set('Retry-After', '5'); // Suggest retry after 5 seconds
				res.set('X-Database-Status', 'not_ready');
				
				return res.status(503).json(errorInfo);
			}
		}

		// Database is ready, proceed
		clearTimeout(timer);
		if (!timeoutTriggered) {
			// Only log if it took significant time (>100ms)
			if (Date.now() - startTime > 100) {
				console.log(`[Readiness Check][${req.method} ${req.path}] PASSED in ${Date.now() - startTime}ms`);
			}
			next();
		} else {
			// Timeout was triggered but DB became ready - return timeout error
			res.set("Retry-After", String(config.backendConfig?.retryAfterSeconds ?? 30));
			return res.status(503).json({
				success: false,
				message: "Database readiness check timeout. Please check DB connection.",
				status: "timeout",
				timestamp: new Date().toISOString(),
			});
		}
	} catch (error) {
		clearTimeout(timer);
		console.error("[Readiness Check] Error checking database readiness:", error);
		
		// Provide detailed error information
		const errorInfo = {
			success: false,
			message: "Database initialization error. Please contact administrator.",
			status: "database_error",
			error: error.message,
			timestamp: new Date().toISOString(),
			details: {
				errorType: error.name || "UnknownError",
				errorMessage: error.message,
				suggestion: "Please check database configuration, network connectivity, and server logs."
			}
		};

		res.set("Retry-After", String(config.backendConfig?.retryAfterSeconds ?? 30));
		res.set("X-Database-Status", "error");
		return res.status(503).json(errorInfo);
	}
});

// IP Analytics Routes (setelah database readiness agar hanya dijalankan saat DB siap)
const ipAnalyticsRoutes = require("./apps/analytics/ip-analytics.routes");
app.use("/api/ip-analytics", ipAnalyticsRoutes);

// =============== API ROUTES ===============
// app.use("/api/whatsapp", require("./apps/sapps/services/Whatsapps/whatsapps.controller"));

// main
app.use("/sapps/api", require("./apps/sapps/masters/statuses/statuses.controller"));

// Logging login request untuk analisis timeout

// Logging login request hanya jika terjadi error (status >= 400)
app.use("/sapps/users/login", async (req, res, next) => {
	// Tangkap response status setelah response selesai
	const fs = require("fs");
	const path = require("path");
	const logDir = path.join(__dirname, "logs");
	const loginLogFile = path.join(logDir, "login-requests.log");
	if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
	// Simpan data request untuk log error
	const payload = {
		timestamp: new Date().toISOString(),
		ip: healthCheck.getClientIP ? healthCheck.getClientIP(req) : req.ip,
		body: req.body,
		headers: req.headers,
		path: req.path,
		method: req.method,
	};
	// Hook response finish (async write agar tidak blocking)
	res.on("finish", () => {
		if (res.statusCode >= 400) {
			fs.appendFile(loginLogFile, JSON.stringify({ ...payload, statusCode: res.statusCode }) + "\n", "utf8", (e) => {
				if (e) console.error("[login-requests] write error:", e.message);
			});
		}
	});
	next();
});

// sapps
app.use("/sapps/users", require("./apps/sapps/masters/users/users.controller"));
app.use("/sapps/user-activities", require("./apps/sapps/masters/userActivites/userActivities.controller"));
app.use("/sapps/employees", require("./apps/sapps/masters/employees/employees.controller"));
app.use("/sapps/companies", require("./apps/sapps/masters/companies/companies.controller"));
app.use("/sapps/menus", require("./apps/sapps/masters/menus/menus.controller"));
app.use("/sapps/roles", require("./apps/sapps/masters/roles/roles.controller"));
app.use("/sapps/doc-types", require("./apps/sapps/masters/docTypes/docTypes.controller"));
app.use("/sapps/permissions", require("./apps/sapps/masters/permissions/permissions.controller"));
app.use("/sapps/projects", require("./apps/sapps/masters/projects/projects.controller"));
app.use("/sapps/images", require("./apps/sapps/masters/images/images.contoller"));
app.use("/sapps/documents", require("./apps/sapps/masters/documents/documents.controller"));
app.use("/sapps/videos", require("./apps/sapps/masters/videos/videos.controller"));
app.use("/sapps/emails", require("./apps/sapps/masters/emails/emails.controller"));
app.use("/sapps/email-accounts", require("./apps/sapps/masters/emailAccounts/emailAccounts.controller"));
app.use("/sapps/developments", require("./apps/sapps/masters/developments/developments.controller"));
app.use("/sapps/apikeys", require("./apps/sapps/masters/apikeys/apikeys.controller"));
app.use("/sapps/erps", require("./apps/sapps/masters/erps/erps.controller"));
app.use("/sapps/provinces", require("./apps/sapps/masters/provinces/provinces.controller"));
app.use("/sapps/cities", require("./apps/sapps/masters/cities/cities.controller"));
app.use("/sapps/years", require("./apps/sapps/masters/years/years.controller"));
app.use("/sapps/periods", require("./apps/sapps/masters/periods/periods.controller"));
app.use("/sapps/client-types", require("./apps/sapps/masters/clientTypes/clientTypes.controller"));
app.use("/sapps/client-subtypes", require("./apps/sapps/masters/clientSubtypes/clientSubtypes.controller"));
app.use("/sapps/specialists", require("./apps/sapps/masters/specialists/specialists.controller"));
app.use("/sapps/faculties", require("./apps/sapps/masters/faculties/faculties.controller"));
app.use("/sapps/majors", require("./apps/sapps/masters/majors/majors.controller"));
app.use("/sapps/sub-specialists", require("./apps/sapps/masters/subSpecialists/subSpecialists.controller"));
app.use("/sapps/outlet-types", require("./apps/sapps/masters/outletTypes/outletTypes.controller"));
app.use("/sapps/institutions", require("./apps/sapps/masters/institutions/institutions.controller"));
app.use("/sapps/clients", require("./apps/sapps/masters/clients/clients.controller"));
app.use("/sapps/territory-versions", require("./apps/sapps/masters/territoryVersions/territoryVersions.controller"));
app.use("/sapps/territories", require("./apps/sapps/masters/territories/territories.controller"));
app.use("/sapps/outlets", require("./apps/sapps/masters/outlets/outlets.controller"));
app.use("/sapps/departments", require("./apps/sapps/masters/departments/departments.controller"));
app.use("/sapps/designations", require("./apps/sapps/masters/designations/designations.controller"));
app.use("/sapps/holiday-groups", require("./apps/sapps/masters/holidayGroups/holidayGroups.controller"));
app.use("/sapps/holidays", require("./apps/sapps/masters/holidays/holidays.controller"));
app.use("/sapps/item-groups", require("./apps/sapps/masters/itemGroups/itemGroups.controller"));
app.use("/sapps/items", require("./apps/sapps/masters/items/items.controller"));
app.use("/sapps/item-competitors", require("./apps/sapps/masters/itemCompetitors/itemCompetitors.controller"));
app.use("/sapps/question-banks", require("./apps/sapps/masters/questionBanks/questionBanks.controller"));
app.use("/sapps/delivery-trips", require("./apps/sapps/masters/deliveryTrips/deliveryTrips.controller"));
app.use("/sapps/dms-legals", require("./apps/sapps/masters/dmsLegals/dmsLegals.controller"));
app.use("/sapps/courses", require("./apps/sapps/masters/courses/courses.controller"));
app.use("/sapps/questions", require("./apps/sapps/masters/questions/questions.controller"));
app.use("/sapps/project-managements", require("./apps/sapps/masters/projectManagements/projectManagements.controller"));
app.use("/sapps/vehicles", require("./apps/sapps/masters/vehicles/vehicles.controller"));
app.use("/sapps/workplan-monitors", require("./apps/sapps/masters/workplanMonitors/workplanMonitors.controller"));
app.use("/sapps/project-management", require("./apps/sapps/masters/projectManagements/projectManagements.controller"));
app.use("/sapps/call-plans", require("./apps/sapps/masters/callPlans/callPlans.controller"));
app.use("/sapps/moms", require("./apps/sapps/masters/moms/moms.controller"));
app.use("/sapps/designation-versions", require("./apps/sapps/masters/designationVersions/designationVersions.controller"));
app.use("/sapps/workflow-settings", require("./apps/sapps/masters/workflowSettings/workflowSettings.controller"));
app.use("/sapps/workflows", require("./apps/sapps/masters/workflows/workflows.controller"));
app.use("/sapps/employee-checkins", require("./apps/sapps/masters/employeeCheckins/employeeCheckins.controller"));
app.use("/sapps/attendances", require("./apps/sapps/masters/attendances/attendances.controller"));
app.use("/sapps/attendance-requests", require("./apps/sapps/masters/attendanceRequests/attendanceRequests.controller"));
app.use("/sapps/leave-types", require("./apps/sapps/masters/leaveTypes/leaveTypes.controller"));
app.use("/sapps/leave-applications", require("./apps/sapps/masters/leaveApplications/leaveApplications.controller"));
app.use("/sapps/shift-types", require("./apps/sapps/masters/shiftTypes/shiftTypes.controller"));
app.use("/sapps/distributors", require("./apps/sapps/masters/distributors/distributors.controller"));
app.use("/sapps/customer-lists", require("./apps/sapps/masters/customerLists/customerLists.controller"));
app.use("/sapps/udks", require("./apps/sapps/masters/udks/udks.controller"));
app.use("/sapps/applicants", require("./apps/sapps/masters/applicants/applicants.controller"));
app.use("/sapps/employee-requests", require("./apps/sapps/masters/employeeRequests/employeeRequests.controller"));
app.use("/sapps/absence-clarifications", require("./apps/sapps/masters/absenceClarifications/absenceClarifications.controller"));
app.use("/sapps/sales-distributors", require("./apps/sapps/masters/salesDistributors/salesDistributor.controller"));
app.use("/sapps/permit-eases", require("./apps/sapps/masters/permitEases/permitEases.controller"));
app.use("/sapps/leave-allocations", require("./apps/sapps/masters/leaveAllocations/leaveAllocations.controller"));
app.use("/sapps/client-collabs", require("./apps/sapps/masters/clientCollabs/clientCollabs.controller"));
app.use("/sapps/surveys", require("./apps/sapps/masters/surveys/surveys.controller"));
app.use("/sapps/account-entries", require("./apps/sapps/masters/accountEntries/accountEntries.controller"));
app.use("/sapps/receipt-of-goods", require("./apps/sapps/masters/receiptOfGoods/receiptOfGoods.controller"));
app.use("/sapps/udps", require("./apps/sapps/masters/udps/udps.controller"));
app.use("/sapps/sales-targets", require("./apps/sapps/masters/salesTargets/salesTargets.controller"));
app.use("/sapps/internal-audits", require("./apps/sapps/masters/internalAudits/internalAudits.controller"));
app.use("/sapps/work-instructions", require("./apps/sapps/masters/workInstructions/workInstructions.controller"));
app.use("/sapps/job-postings", require("./apps/sapps/masters/jobPostings/jobPostings.controller"));
app.use("/sapps/nc-products", require("./apps/sapps/masters/ncProducts/ncProducts.controller"));
app.use("/sapps/product-returns", require("./apps/sapps/masters/productReturns/productReturns.controller"));
app.use("/sapps/comments", require("./apps/sapps/masters/comments/comments.controller"));
app.use("/sapps/sales-plans", require("./apps/sapps/masters/salesPlans/salesPlans.controller"));
app.use("/sapps/invoice-receipts", require("./apps/sapps/masters/invoiceReceipts/invoiceReceipts.controller"));
app.use("/sapps/fses", require("./apps/sapps/masters/fses/fses.controller"));
app.use("/sapps/gimmick-req-types", require("./apps/sapps/masters/gimmickReqTypes/gimmickReqTypes.controller"));
app.use("/sapps/employee-statuses", require("./apps/sapps/masters/employeeStatuses/employeeStatuses.controller"));
app.use("/sapps/genders", require("./apps/sapps/masters/genders/genders.controller"));
app.use("/sapps/religions", require("./apps/sapps/masters/religions/religions.controller"));
app.use("/sapps/bank-accounts", require("./apps/sapps/masters/bankAccounts/bankAccounts.controller"));
app.use("/sapps/marital-statuses", require("./apps/sapps/masters/maritalStatuses/maritalStatuses.controller"));
app.use("/sapps/gimmick-requests", require("./apps/sapps/masters/gimmickRequests/gimmickRequests.controller"));
app.use("/sapps/banks", require("./apps/sapps/masters/banks/banks.controller"));
app.use("/sapps/employee-grades", require("./apps/sapps/masters/employeeGrades/employeeGrades.controller"));
app.use("/sapps/employee-groups", require("./apps/sapps/masters/employeeGroups/employeeGroups.controller"));
app.use("/sapps/employment-types", require("./apps/sapps/masters/employmentTypes/employmentTypes.controller"));
app.use("/sapps/branches", require("./apps/sapps/masters/branches/branches.controller"));
app.use("/sapps/leave-periods", require("./apps/sapps/masters/leavePeriods/leavePeriods.controller"));
app.use("/sapps/holiday-lists", require("./apps/sapps/masters/holidayLists/holidayLists.controller"));
app.use("/sapps/reg-categories", require("./apps/sapps/masters/regulatoryCategories/regulatoryCategories.controller"));
app.use("/sapps/document-groups", require("./apps/sapps/masters/documentGroups/documentGroups.controller"));
app.use("/sapps/travel-cost-rates", require("./apps/sapps/masters/travelCostRates/travelCostRates.controller"));
app.use("/sapps/dms-regulatories", require("./apps/sapps/masters/dmsRegulatories/dmsRegulatories.controller"));
app.use("/sapps/tracking-locations", require("./apps/sapps/masters/trackingLocations/trackingLocations.controller"));
app.use("/sapps/uc-settings", require("./apps/sapps/masters/ucSettings/ucSettings.controller"));
app.use("/sapps/blood-groups", require("./apps/sapps/masters/bloodGroups/bloodGroups.controller"));
app.use("/sapps/nationalities", require("./apps/sapps/masters/nationalities/nationalities.controller"));
app.use("/sapps/occupations", require("./apps/sapps/masters/occupations/occupations.controller"));
app.use("/sapps/up-countries", require("./apps/sapps/masters/upCountries/upCountries.controller"));
app.use("/sapps/uc-cost-realizations", require("./apps/sapps/masters/ucCostRealizations/ucCostRealizations.controller"));
app.use("/sapps/release-notes", require("./apps/sapps/masters/releaseNotes/releaseNotes.controller"));
app.use("/sapps/good-receipts", require("./apps/sapps/masters/goodReceipts/goodReceipts.controller")); // yang mana yang dipakai
app.use("/sapps/goods-receipts", require("./apps/sapps/masters/goodsReceipts/goodsReceipts.controller")); // yang mana yang dipakai
app.use("/sapps/budgetings", require("./apps/sapps/masters/budgetings/budgetings.controller"));
app.use("/sapps/procurement-trackers", require("./apps/sapps/masters/procurementTrackers/procurementTrackers.controller"));
app.use("/sapps/work-activities", require("./apps/sapps/masters/workActivities/workActivities.controller"));
app.use("/sapps/accounts", require("./apps/sapps/masters/accounts/accounts.controller"));
app.use("/sapps/budget-categories", require("./apps/sapps/masters/budgetCategories/budgetCategories.controller"));
app.use("/sapps/budget-classifications", require("./apps/sapps/masters/budgetClassifications/budgetClassifications.controller"));
app.use("/sapps/budget-departments", require("./apps/sapps/masters/budgetDepartments/budgetDepartments.controller"));
app.use("/sapps/budget-items", require("./apps/sapps/masters/budgetItems/budgetItems.controller"));
app.use("/sapps/budget-accounts", require("./apps/sapps/masters/budgetAccounts/budgetAccounts.controller"));
app.use("/sapps/budget-unit-of-measurements", require("./apps/sapps/masters/budgetUnitOfMeasurements/budgetUnitOfMeasurements.controller"));
app.use("/sapps/budget-plans", require("./apps/sapps/masters/budgetPlans/budgetPlans.controller"));
app.use("/sapps/budget-approvers", require("./apps/sapps/masters/budgetApprovers/budgetApprovers.controller"));
app.use("/sapps/saldo-awal-clients", require("./apps/sapps/masters/saldoAwalClients/saldoAwalClients.controller"));
app.use("/sapps/tenders", require("./apps/sapps/masters/tenders/tenders.controller"));
app.use("/sapps/tender-categories", require("./apps/sapps/masters/tenderCategories/tenderCategories.controller"));


app.use("/sapps/tenders", require("./apps/sapps/masters/tenders/tenders.controller"));
app.use("/sapps/tender-categories", require("./apps/sapps/masters/tenderCategories/tenderCategories.controller"));

// dashboard
app.use("/sapps/dashboards-employee-checkins", require("./apps/sapps/dashboards/employeeCheckins/employeeCheckins.controller"));
app.use("/sapps/dashboards-accountings", require("./apps/sapps/dashboards/accountings/accountings.controller"));
app.use("/sapps/dashboards-leave-allocations", require("./apps/sapps/dashboards/leaveAllocations/leaveAllocations.controller"));
app.use("/sapps/dashboards-leave-applications", require("./apps/sapps/dashboards/leaveApplications/leaveApplications.controller"));
app.use("/sapps/dashboards-attendance-requests", require("./apps/sapps/dashboards/attendanceRequests/attendanceRequests.controller"));

// junction
app.use("/sapps/role-menu", require("./apps/sapps/junctions/roleMenus/roleMenus.controller"));
app.use("/sapps/user-company", require("./apps/sapps/junctions/userCompanies/userCompanies.controller"));
app.use("/sapps/employee-territory", require("./apps/sapps/junctions/employeeTerritories/employeeTerritories.controller"));
app.use("/sapps/territory-itemgroup", require("./apps/sapps/junctions/territoryItemGroups/territoryItemGroups.controller"));
app.use("/sapps/outlet-client", require("./apps/sapps/junctions/outletClients/outletClients.controller"));
app.use("/sapps/territory-outlet", require("./apps/sapps/junctions/territoryOutlet/territoryOutlets.controller"));
app.use("/sapps/master-lists", require("./apps/sapps/junctions/masterLists/masterLists.controller"));

// reports
app.use("/sapps/reports", require("./apps/sapps/reports/reports.controller"));

// imports
app.use("/sapps/imports", require("./apps/sapps/imports/imports.controller"));

// additional
app.use("/sapps/additional-designations", require("./apps/sapps/additionals/designations/designations.controller"));
app.use("/sapps/additional-territories", require("./apps/sapps/additionals/territories/territories.controller"));

// integration
// erpnext
app.use("/sapps/integrations/erpnext/accounts", require("./apps/sapps/integrations/erpnext/accounts/accounts.controller"));
app.use("/sapps/integrations/erpnext/items", require("./apps/sapps/integrations/erpnext/items/items.controller"));
app.use("/sapps/integrations/erpnext/departments", require("./apps/sapps/integrations/erpnext/departments/departments.controller"));

// =============== ERROR HANDLER ====================
// Handle connection errors first
app.use(connectionResilience.handleConnectionErrors);

// Global error handler
app.use(errorHandler);

// =============== SERVER LISTEN ====================
// =============== SERVER LISTEN WITH FALLBACK ====================
let serverInstance = null;

// PORT handling for cPanel deployment
// Priority: process.env.PORT (set by cPanel Application Manager) > config.json > default 3000
const START_PORT = process.env.PORT ? parseInt(process.env.PORT, 10) : config.backendConfig.port ? parseInt(config.backendConfig.port, 10) : 3000;

// Log port configuration for debugging in cPanel
console.log(`[Server Config] NODE_ENV: ${config.backendConfig.NODE_ENV || process.env.NODE_ENV || "development"}`);
console.log(`[Server Config] PORT: ${START_PORT} (from ${process.env.PORT ? "environment" : "config.json"})`);

async function startServer(port, attemptsLeft = 10) {
	try {
		// Wait for database initialization to complete before starting server
		console.log("========================================");
		console.log("Starting SAPPS Backend Server...");
		console.log("========================================");
		console.log(`Environment: ${config.backendConfig.NODE_ENV || process.env.NODE_ENV || "development"}`);
		console.log(`Port: ${port}`);
		console.log(`Database Host: ${config.databaseConfigSAPPS.host}:${config.databaseConfigSAPPS.port}`);
		console.log(`Database Name: ${config.databaseConfigSAPPS.database}`);
		console.log("========================================");
		
		// Pre-start network connectivity test with retry strategies
		console.log("Testing network connectivity to database server...");
		console.log("========================================");
		const NetworkDiagnostics = require("./utils/networkDiagnostics");
		
		// Try with retry strategies first (more robust)
		let networkTest = await NetworkDiagnostics.testTCPConnectionWithStrategies(
			config.databaseConfigSAPPS.host,
			config.databaseConfigSAPPS.port
		);
		
		// If all strategies failed, try with retry mechanism
		if (!networkTest.success) {
			console.log("Quick strategies failed, trying with retry mechanism...");
			networkTest = await NetworkDiagnostics.testTCPConnection(
				config.databaseConfigSAPPS.host,
				config.databaseConfigSAPPS.port,
				10000, // 10 second timeout
				{ maxRetries: 2, retryDelay: 2000 } // 2 retries with 2s delay
			);
		}
		
		if (!networkTest.success) {
			console.error("========================================");
			console.error("NETWORK CONNECTIVITY TEST FAILED");
			console.error("========================================");
			console.error(`Error: ${networkTest.error || 'Unknown'}`);
			console.error(`Error Type: ${networkTest.errorType || 'UNKNOWN'}`);
			console.error(`Message: ${networkTest.message}`);
			console.error(`Duration: ${networkTest.duration}ms`);
			console.error("========================================");
			console.error("DIAGNOSTIC INFORMATION:");
			console.error(`  - Host: ${config.databaseConfigSAPPS.host}`);
			console.error(`  - Port: ${config.databaseConfigSAPPS.port}`);
			console.error(`  - Suggestion: ${networkTest.suggestion || 'Periksa koneksi network'}`);
			console.error("========================================");
			console.error("TROUBLESHOOTING STEPS:");
			console.error("1. Gunakan VPN kantor untuk mengakses database server");
			console.error("2. Periksa apakah IP Anda dalam whitelist database server");
			console.error("3. Periksa firewall rules di database server");
			console.error("4. Pastikan port 3307 accessible dari network Anda");
			console.error("5. Test koneksi dengan: telnet " + config.databaseConfigSAPPS.host + " " + config.databaseConfigSAPPS.port);
			console.error("========================================");
			console.error("Server will NOT start without network connectivity to database.");
			console.error("Please resolve network connectivity issues before starting server.");
			console.error("========================================");
			
			// Log to file for debugging
			const fs = require("fs");
			const path = require("path");
			const logDir = path.join(__dirname, "logs");
			const networkLogFile = path.join(logDir, "network-connectivity-errors.log");
			try {
				if (!fs.existsSync(logDir)) {
					fs.mkdirSync(logDir, { recursive: true });
				}
				fs.appendFileSync(
					networkLogFile,
					JSON.stringify({
						timestamp: new Date().toISOString(),
						error: "Network connectivity test failed - server not started",
						networkTest,
						config: {
							host: config.databaseConfigSAPPS.host,
							port: config.databaseConfigSAPPS.port
						}
					}) + "\n",
					"utf8"
				);
			} catch (logError) {
				console.error("Failed to write network error log:", logError);
			}
			
			// Fail fast - do not start server if network connectivity fails
			throw new Error(`Network connectivity test failed: ${networkTest.error || 'Unknown error'}. ${networkTest.suggestion || ''}`);
		}
		
		console.log("========================================");
		console.log("Network connectivity test: PASSED");
		console.log(`Duration: ${networkTest.duration}ms`);
		console.log("========================================");
		
		// Initialize Robust Network Manager for comprehensive network management
		console.log("Initializing robust network manager...");
		const RobustNetworkManager = require("./utils/robustNetworkManager");
		const networkManager = new RobustNetworkManager(config.databaseConfigSAPPS, {
			testTimeout: 10000,
			testRetries: 2,
			testRetryDelay: 2000,
			monitoringEnabled: true,
			checkInterval: 30000,
			successThreshold: 2,
			failureThreshold: 3,
			autoRecoveryEnabled: true,
			recoveryAttempts: 5,
			recoveryDelay: 5000
		});

		// Setup network manager event handlers
		networkManager.on('online', (data) => {
			console.log("========================================");
			console.log("NETWORK STATUS: ONLINE");
			console.log("========================================");
			console.log(`Network recovered after ${data.wasOffline ? 'being offline' : 'initial check'}`);
			console.log(`Consecutive successes: ${data.consecutiveSuccesses}`);
			console.log("========================================");
		});

		networkManager.on('offline', (data) => {
			console.error("========================================");
			console.error("NETWORK STATUS: OFFLINE");
			console.error("========================================");
			console.error(`Error: ${data.error}`);
			console.error(`Error Type: ${data.errorType}`);
			console.error(`Message: ${data.message}`);
			console.error(`Suggestion: ${data.suggestion}`);
			console.error(`Consecutive failures: ${data.consecutiveFailures}`);
			console.error("========================================");
		});

		networkManager.on('recovery-success', (data) => {
			console.log("========================================");
			console.log("AUTO-RECOVERY: SUCCESS");
			console.log("========================================");
			console.log(`Recovery attempt ${data.attempt} successful`);
			console.log(`Timestamp: ${data.timestamp}`);
			console.log("========================================");
		});

		networkManager.on('recovery-failure', (data) => {
			console.error("========================================");
			console.error("AUTO-RECOVERY: FAILED");
			console.error("========================================");
			console.error(`Recovery attempt ${data.attempt} failed`);
			console.error(`Error: ${data.error}`);
			console.error(`Timestamp: ${data.timestamp}`);
			console.error("========================================");
		});

		// Initialize network manager (performs initial test and starts monitoring)
		await networkManager.initialize();
		
		// Store network manager globally for access in endpoints
		global.networkMonitor = networkManager.monitor; // Keep for backward compatibility
		global.networkManager = networkManager; // New robust manager
		
		// Start network connectivity monitoring
		networkConnectivityMonitor.startMonitoring();
		global.networkConnectivityMonitor = networkConnectivityMonitor;
		
		console.log("Robust network manager initialized and monitoring started");
		console.log("Network connectivity monitoring started");
		console.log("========================================");
		console.log("Waiting for database initialization...");
		console.log("========================================");

		const dbSAPPS = require("./apps/sapps/helpers/db");

		// Increase timeout for production/cPanel (timeout 0 means no timeout, so initialization can take longer)
		const isProduction = process.env.NODE_ENV === "production" || config.backendConfig.NODE_ENV === "production";
		const initTimeout = isProduction ? 180000 : 90000; // 180 seconds for production, 90 seconds for development

		console.log(`Waiting for database initialization (timeout: ${initTimeout / 1000}s)...`);
		console.log(`Database connection details:`);
		console.log(`  - Host: ${config.databaseConfigSAPPS.host}`);
		console.log(`  - Port: ${config.databaseConfigSAPPS.port}`);
		console.log(`  - Database: ${config.databaseConfigSAPPS.database}`);
		console.log(`  - User: ${config.databaseConfigSAPPS.user}`);

		// Check initial status
		const initialStatus = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : null;
		if (initialStatus) {
			console.log(`Initial status:`, {
				isInitialized: initialStatus.isInitialized,
				isReady: initialStatus.isReady,
				hasError: !!initialStatus.initializationError
			});
			if (initialStatus.initializationError) {
				console.warn(`Initial error detected:`, initialStatus.initializationError.message);
			}
		}

		try {
			await dbSAPPS.waitForInitialization(initTimeout);

			// Verify database is truly ready before starting server
			if (!dbSAPPS.isReady || !dbSAPPS.isReady()) {
				// Wait a bit more and check again (in case of race condition)
				console.log("Database not ready yet, waiting additional 2 seconds...");
				await new Promise((resolve) => setTimeout(resolve, 2000));

				if (!dbSAPPS.isReady || !dbSAPPS.isReady()) {
					const initStatus = dbSAPPS.getInitializationStatus ? dbSAPPS.getInitializationStatus() : {};
					console.error("========================================");
					console.error("CRITICAL: Database initialization incomplete");
					console.error("Server will NOT start without database connection");
					console.error("========================================");
					console.error("Initialization Status:");
					console.error(`  - isInitialized: ${initStatus.isInitialized || false}`);
					console.error(`  - isReady: ${initStatus.isReady || false}`);
					if (initStatus.initializationError) {
						console.error(`  - Error: ${initStatus.initializationError.message || 'Unknown error'}`);
						console.error(`  - Error Type: ${initStatus.initializationError.name || 'Unknown'}`);
					}
					console.error("========================================");
					console.error("Troubleshooting Steps:");
					console.error("1. Check database server is running");
					console.error(`2. Verify database connection details (host: ${config.databaseConfigSAPPS.host}, port: ${config.databaseConfigSAPPS.port})`);
					console.error(`3. Verify database credentials (user: ${config.databaseConfigSAPPS.user})`);
					console.error(`4. Check network connectivity to database server`);
					console.error(`5. Check database logs for connection errors`);
					console.error("6. Review logs/initialization-errors.log for detailed error information");
					console.error("========================================");

					// Log to file for debugging in cPanel
					const fs = require("fs");
					const path = require("path");
					const logDir = path.join(__dirname, "logs");
					const initLogFile = path.join(logDir, "initialization-errors.log");

					try {
						if (!fs.existsSync(logDir)) {
							fs.mkdirSync(logDir, { recursive: true });
						}
						fs.appendFileSync(
							initLogFile,
							JSON.stringify({
								timestamp: new Date().toISOString(),
								error: "Database initialization incomplete - server not started",
								initStatus,
							}) + "\n",
							"utf8",
						);
					} catch (logError) {
						console.error("Failed to write initialization error log:", logError);
					}

					// Fail fast - do not start server if database is not ready
					throw new Error("Database initialization incomplete. Server cannot start without database connection.");
				}
			}

			console.log("========================================");
			console.log("Database initialization: COMPLETE");
			console.log("Associations: VERIFIED");
			console.log("Status: READY");
			console.log("Server will start now...");
			console.log("========================================");
		} catch (initError) {
			console.error("========================================");
			console.error("CRITICAL: Database initialization failed");
			console.error("========================================");
			console.error("Error Details:");
			console.error(`  - Message: ${initError.message}`);
			console.error(`  - Type: ${initError.name || 'Unknown'}`);
			if (initError.code) {
				console.error(`  - Code: ${initError.code}`);
			}
			if (initError.networkDiagnostic) {
				console.error(`  - Network Error Type: ${initError.networkDiagnostic.errorType}`);
				console.error(`  - User Message: ${initError.networkDiagnostic.userMessage}`);
				console.error(`  - Suggestion: ${initError.networkDiagnostic.suggestion}`);
			}
			if (initError.stack) {
				console.error(`  - Stack: ${initError.stack.split('\n').slice(0, 5).join('\n')}`);
			}
			console.error("========================================");
			console.error("Server will NOT start without database connection");
			console.error("========================================");
			console.error("Troubleshooting Steps:");
			console.error(`1. Database Host: ${config.databaseConfigSAPPS.host}`);
			console.error(`2. Database Port: ${config.databaseConfigSAPPS.port}`);
			console.error(`3. Database Name: ${config.databaseConfigSAPPS.database}`);
			console.error(`4. Database User: ${config.databaseConfigSAPPS.user}`);
			console.error("========================================");
			
			// Network-specific troubleshooting
			if (initError.networkDiagnostic) {
				const netDiag = initError.networkDiagnostic;
				if (netDiag.errorType === 'FIREWALL_BLOCKED' || netDiag.errorType === 'TIMEOUT') {
					console.error("NETWORK CONNECTIVITY ISSUE DETECTED:");
					console.error("========================================");
					console.error("Kemungkinan besar masalah network/firewall:");
					console.error("1. Gunakan VPN kantor untuk mengakses database server");
					console.error("2. Periksa apakah IP Anda dalam whitelist database server");
					console.error("3. Periksa firewall rules di database server");
					console.error("4. Test koneksi dengan: telnet " + config.databaseConfigSAPPS.host + " " + config.databaseConfigSAPPS.port);
					console.error("5. Jika menggunakan IP publik, pastikan firewall allow connection");
					console.error("========================================");
				} else if (netDiag.errorType === 'NETWORK_UNREACHABLE') {
					console.error("NETWORK ROUTING ISSUE DETECTED:");
					console.error("========================================");
					console.error("Network tidak dapat mencapai database server:");
					console.error("1. Periksa network routing configuration");
					console.error("2. Coba gunakan VPN kantor");
					console.error("3. Periksa network connectivity ke database server");
					console.error("========================================");
				}
			} else {
				console.error("5. Verify database server is accessible");
				console.error("6. Check firewall rules");
				console.error("7. Verify database credentials");
				console.error("8. Check database server logs");
				console.error("9. If using public IP, may need VPN to access database");
			}
			console.error("========================================");

			// Log to file for debugging in cPanel
			const fs = require("fs");
			const path = require("path");
			const logDir = path.join(__dirname, "logs");
			const initLogFile = path.join(logDir, "initialization-errors.log");

			try {
				if (!fs.existsSync(logDir)) {
					fs.mkdirSync(logDir, { recursive: true });
				}
				fs.appendFileSync(
					initLogFile,
					JSON.stringify({
						timestamp: new Date().toISOString(),
						error: initError.message,
						stack: initError.stack,
						critical: true,
					}) + "\n",
					"utf8",
				);
			} catch (logError) {
				console.error("Failed to write initialization error log:", logError);
			}

			// Fail fast - do not start server if database initialization fails
			throw initError;
		}

		serverInstance = app.listen(port, () => {
			console.log("========================================");
			console.log(`Server running on port ${port} in ${config.databaseConfigSAPPS.NODE_ENV} mode`);
			console.log(`PID: ${process.pid}`);
			console.log(`Uptime: ${process.uptime()}s`);
			console.log("Static routes enabled with CORS: /avatars, /logos, /documents, /images, /videos, /audios, /others");
			console.log("Crash recovery system: ACTIVE");
			console.log("Database readiness check: ACTIVE");
			console.log("========================================");
			// Start background DB initialization retries
			try {
				const dbSAPPS = require("./apps/sapps/helpers/db");
				if (!dbSAPPS.isReady || !dbSAPPS.isReady()) {
					dbSAPPS.retryInitialization();
				}
			} catch (bgErr) {
				console.error("Failed to start background DB retry:", bgErr.message);
			}

			// Reset restart count after successful startup (if uptime > 5 minutes)
			// This will be checked periodically
			if (process.uptime() > 300) {
				crashRecovery.resetRestartCount();
			}
		});

		serverInstance.on("error", (err) => {
			if (err && err.code === "EADDRINUSE" && attemptsLeft > 0) {
				const nextPort = port + 1;
				console.warn(`Port ${port} in use. Retrying on ${nextPort}...`);
				startServer(nextPort, attemptsLeft - 1);
			} else {
				console.error("Server failed to start:", err);
				// Log crash before exiting
				crashRecovery.logCrash(err, {
					type: "serverStartError",
					port,
					attemptsLeft,
				});
				// Don't restart for port errors - these are fatal
				process.exit(1);
			}
		});

		// Handle server close event
		serverInstance.on("close", () => {
			console.log("Server connection closed");
		});
	} catch (err) {
		console.error("Unexpected error while starting server:", err);
		// Log crash before exiting
		crashRecovery.logCrash(err, {
			type: "serverStartException",
		});
		process.exit(1);
	}
}

// Start server - wrap in async IIFE to handle async startServer
(async () => {
	try {
		await startServer(START_PORT);
	} catch (error) {
		console.error("Failed to start server:", error);
		process.exit(1);
	}
})();

// Periodic check to reset restart count after stable uptime
// FIXED: Store interval ID for proper cleanup
const restartCountResetInterval = setInterval(() => {
	try {
		crashRecovery.resetRestartCount();
	} catch (error) {
		console.error("Error resetting restart count:", error);
	}
}, 60000); // Check every minute

// Cleanup interval on shutdown
process.on("SIGINT", () => {
	if (restartCountResetInterval) {
		clearInterval(restartCountResetInterval);
	}
});
process.on("SIGTERM", () => {
	if (restartCountResetInterval) {
		clearInterval(restartCountResetInterval);
	}
});

function gracefulShutdown(signal) {
	try {
		// Mark as shutting down to prevent auto-restart
		enhancedErrorHandler.setShuttingDown(true);

		console.log(`Received ${signal}. Closing server...`);
		console.log(`Uptime before shutdown: ${Math.floor(process.uptime())}s`);

		// Stop network monitoring
		if (global.networkManager) {
			console.log("Stopping network manager...");
			global.networkManager.stopMonitoring();
		} else if (global.networkMonitor) {
			console.log("Stopping network status monitor...");
			global.networkMonitor.stop();
		}
		if (global.networkConnectivityMonitor) {
			console.log("Stopping network connectivity monitor...");
			global.networkConnectivityMonitor.stopMonitoring();
		}

		// Close database connection manager
		const dbSAPPS = require("./apps/sapps/helpers/db");
		if (dbSAPPS.dbConnectionManager && dbSAPPS.dbConnectionManager.close) {
			console.log("Closing database connection manager...");
			dbSAPPS.dbConnectionManager.close().catch((err) => {
				console.error("Error closing database connection manager:", err.message);
			});
		}

		if (serverInstance) {
			serverInstance.close(() => {
				console.log("Server closed gracefully.");
				console.log("========================================");
				process.exit(0);
			});

			// Fallback: force exit if not closed within 5 seconds
			setTimeout(() => {
				console.warn("Forced shutdown due to timeout.");
				process.exit(1);
			}, 5000);
		} else {
			process.exit(0);
		}
	} catch (err) {
		console.error("Error during graceful shutdown:", err);
		process.exit(1);
	}
}

// Handle shutdown signals
process.on("SIGINT", () => gracefulShutdown("SIGINT"));
process.on("SIGTERM", () => gracefulShutdown("SIGTERM"));
