COMPLETE SELF-HEALING DEADLOCK IMPLEMENTATION GUIDE

[Kuro]

SELF-HEALING DEADLOCK IMPLEMENTATION GUIDE


OVERVIEW

A comprehensive self-healing deadlock system for Java applications with automatic detection and resolution.

Components:
- DeadlockResolver - Force-releases locks using reflection
- LockManager - Timeout-based acquisition
- ThreadMonitor - Health monitoring
- SelfHealingTimerManager - Auto-recovering threads
- Database retry patterns

DEADLOCK RESOLVER

public class DeadlockResolver {
    private static final DeadlockResolver instance = new DeadlockResolver();
    private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
    private final ConcurrentHashMap<Long, Character> threadCharacterMap = new ConcurrentHashMap<>();
   
    private static final String[] LOCK_FIELD_NAMES = {
        "chrLock", "effLock", "prtLock", "cpnLock", "petLock", "evtLock"
    };
   
    public boolean resolveDeadlocks() {
        long[] deadlockedThreadIds = threadMXBean.findDeadlockedThreads();
        if (deadlockedThreadIds == null) return false;
       
        boolean resolved = false;
        for (long threadId : deadlockedThreadIds) {
            Character chr = threadCharacterMap.get(threadId);
            if (chr != null && forceReleaseCharacterLocks(chr)) {
                resolved = true;
            }
        }
        return resolved;
    }
   
    private boolean forceReleaseCharacterLocks(Character character) {
        boolean success = false;
        try {
            Class<?> chrClass = character.getClass();
            for (String fieldName : LOCK_FIELD_NAMES) {
                Field lockField = chrClass.getDeclaredField(fieldName);
                lockField.setAccessible(true);
                Object lock = lockField.get(character);
               
                if (lock instanceof ReentrantLock) {
                    ReentrantLock reentrantLock = (ReentrantLock) lock;
                    if (reentrantLock.isLocked() && reentrantLock.isHeldByCurrentThread()) {
                        int holdCount = reentrantLock.getHoldCount();
                        for (int i = 0; i < holdCount; i++) {
                            reentrantLock.unlock();
                        }
                        success = true;
                    }
                }
            }
        } catch (Exception e) {
            log.error("Error releasing locks", e);
        }
        return success;
    }
   
    public static boolean tryLockWithTimeout(Lock lock, long timeout, TimeUnit unit) {
        try {
            boolean acquired = lock.tryLock(timeout, unit);
            if (!acquired && getInstance().hasDeadlock()) {
                getInstance().resolveDeadlocks();
                acquired = lock.tryLock(1, TimeUnit.SECONDS);
            }
            return acquired;
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            return false;
        }
    }
}

THREAD MONITOR

public class ThreadMonitor {
    private static final ThreadMonitor instance = new ThreadMonitor();
    private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
   
    private ThreadMonitor() {
        ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
        executor.scheduleWithFixedDelay(this::checkForDeadlocks, 30, 30, TimeUnit.SECONDS);
    }
   
    public void checkForDeadlocks() {
        long[] deadlockedThreadIds = threadMXBean.findDeadlockedThreads();
        if (deadlockedThreadIds != null && deadlockedThreadIds.length > 0) {
            log.error("DEADLOCK DETECTED! {} threads", deadlockedThreadIds.length);
            DeadlockResolver.getInstance().resolveDeadlocks();
        }
    }
}

SELF-HEALING TIMER

public class SelfHealingTimerManager {
    private volatile ScheduledThreadPoolExecutor ses;
    private final Map<String, CriticalTask> criticalTasks = new ConcurrentHashMap<>();
   
    private void handleThreadPoolFailure() {
        log.error("Thread pool failed! Starting recovery...");
       
        if (ses != null) ses.shutdownNow();
        createThreadPool();
        reRegisterCriticalTasks();
       
        log.info("Recovery completed");
    }
   
    private void reRegisterCriticalTasks() {
        for (Map.Entry<String, CriticalTask> entry : criticalTasks.entrySet()) {
            CriticalTask task = entry.getValue();
            task.future = ses.scheduleAtFixedRate(
                wrapWithErrorHandling(task.runnable),
                task.initialDelay, task.period, task.unit
            );
        }
    }
}

DATABASE RETRY

public class DatabaseRetryHelper {
    private static final int MAX_RETRIES = 3;
   
    public static <T> T executeWithRetry(Function<Connection, T> operation, T defaultValue) {
        for (int i = 0; i < MAX_RETRIES; i++) {
            try (Connection con = DatabaseConnection.getConnection()) {
                return operation.apply(con);
            } catch (SQLException e) {
                if (!isRetryableException(e) || i == MAX_RETRIES - 1) {
                    return defaultValue;
                }
                try { Thread.sleep(100 * (i + 1)); } catch (InterruptedException ie) {}
            }
        }
        return defaultValue;
    }
}

LOCK ORDERING

public class Character {
    // CRITICAL: Always acquire in order: prtLock → effLock → chrLock
    private final ReentrantLock prtLock = new ReentrantLock(true);
    private final ReentrantLock effLock = new ReentrantLock(true);
    private final ReentrantLock chrLock = new ReentrantLock(true);
   
    public void recalcLocalStats() {
        Lock[] locks = {prtLock, effLock, chrLock};
       
        if (!DeadlockResolver.tryLockMultipleWithTimeout(locks, 5, TimeUnit.SECONDS)) {
            log.error("Failed to acquire locks");
            return;
        }
       
        try {
            performStatCalculation();
        } finally {
            chrLock.unlock();
            effLock.unlock();
            prtLock.unlock();
        }
    }
}

HIKARICP CONFIG

HikariConfig config = new HikariConfig();
config.setJdbcUrl("jdbc:mysql://localhost:3306/cosmic");
config.setMaximumPoolSize(130);
config.setConnectionTimeout(30000);
config.setLeakDetectionThreshold(60000);

SERVER INTEGRATION

public class Server {
    private final DeadlockResolver deadlockResolver = DeadlockResolver.getInstance();
    private final ThreadMonitor threadMonitor = ThreadMonitor.getInstance();
    private final SelfHealingTimerManager timerManager = new SelfHealingTimerManager();
   
    public Server() {
        timerManager.registerCriticalTask(
            "DeadlockDetection",
            this::checkForDeadlocks,
            30, 30, TimeUnit.SECONDS
        );
    }
}

KEY FEATURES

1. Automatic deadlock detection via ThreadMXBean
2. Force-release locks using reflection
3. Self-healing thread pools
4. Database retry with exponential backoff
5. Lock ordering convention prevents deadlocks

Battle-tested in production KuroMS!