java watchservice - running multiple job on cluster

Question

I am using java watchservice to watch a specific folder when a specific file is written from another program.

I am running multiple jobs using this code and each watching each a specific folder that is different from each other.

This seems to work for the first job. But for the subsequent jobs, the watch service doesn't seem to be able get the write event even after the file it is looking for is written. It just kept idle as if the file has not yet been written.

Does anyone has experience with this?

Here is part of the code:

import java.util.*;
import java.nio.file.*;
import java.io.File;

private void run_code() {

    WatchEvent.Kind watchType = StandardWatchEventKinds.ENTRY_CREATE;
    
    while (i <= max_loop)
    {
        
        // Look out for output from python
        String py_java_file = "Py2java_" + Integer.toString(i) + ".txt";
        lookOutForFile(work_dir, py_java_file, watchType);
            
        // Write finish file
        try {
            Files.write(Paths.get(work_dir+"java2Py_" + Integer.toString(i) + ".txt"), "run completed".getBytes());
        }
        catch (Exception e) {
            print("Error writing file. ");
        }
        
        // Increment counter
        i = i + 1;
    }

}
  
private void lookOutForFile(String watchPath, String watchFile, WatchEvent.Kind watchType){

        Path path = Paths.get(watchPath);
    
        try (WatchService watchService = FileSystems.getDefault().newWatchService()) {
            WatchKey key = path.register(watchService,
                                StandardWatchEventKinds.ENTRY_CREATE,
                                StandardWatchEventKinds.ENTRY_MODIFY,
                                StandardWatchEventKinds.ENTRY_DELETE);
            
            startListening(watchService, watchPath, watchFile, watchType);
            print("Found watch file");
            
        } catch (Exception e) {
            e.printStackTrace();
        }

}
  
private void startListening (WatchService watchService, String watchPath, String watchFile, WatchEvent.Kind watchType)
                        throws InterruptedException {

        boolean foundEvent = false;
        
        while (!foundEvent) {
            WatchKey queuedKey = watchService.take();
            for (WatchEvent<?> watchEvent : queuedKey.pollEvents()) {
                String sf1=String.format("kind=%s, count=%d, context=%s Context type=%s%n ", watchEvent.kind(), watchEvent.count(), watchEvent.context(), ((Path) watchEvent.context()).getClass());
    
                if (watchEvent.kind() == watchType) {               
                    String fileCreated = String.format("%s", watchEvent.context()); 
                    if (fileCreated.equals(watchFile))
                    {
                        foundEvent = true;
                    }
                }
                if (!queuedKey.reset()) {
                    break;
                }
                
                
            }
        }

}