Recursively and continously watch for file change (#35)

* chore: find hpi/jpi recursively

* docker: trigger build

* fix: import

* recursively and continously watch for changes in plugin dir

* updated to add recursvie flag and to revert some formatting issues and how loop logic worked

* added recursive option to readme

* change forEach loop to java8 notation

* removing unnecessary code

* fixing formating issues

* fixing formating issues

* fixing formating issues

* fixing formating issues

* fixing formating issues

* removed string format, fixed getter for recursivewatch

README.md

@@ -117,6 +117,9 @@ Complete list of vars can be found after `juseppe env` command.
 - `JUSEPPE_BIND_PORT` (`juseppe.jetty.port`)   
   port for juseppe file server. Defaults to `8080`
 
+- `JUSEPPE_RECURSIVE_WATCH` (`juseppe.recursive.watch`)
+  watch for file changes recursively Defaults to `true`
+
 Example: 
 
 `java -jar -Djuseppe.saveto.dir=/tmp/update/ juseppe.jar -w serve` or `JUSEPPE_SAVE_TO_DIR=/tmp/update/ java -jar juseppe.jar -w serve`
@@ -128,4 +131,3 @@ Properties are overridden in order: *default value* -> *env vars* -> *system pro
 Site can be added with help of: 
 
 - [UpdateSites Manager plugin](https://wiki.jenkins-ci.org/display/JENKINS/UpdateSites+Manager+plugin)
-

juseppe-cli/src/main/java/ru/lanwen/jenkins/juseppe/files/WatchFiles.java

@@ -2,14 +2,22 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import ru.lanwen.jenkins.juseppe.gen.UpdateSiteGen;
-import ru.lanwen.jenkins.juseppe.props.Props;
 
 import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.WatchEvent;
 import java.nio.file.WatchKey;
 import java.nio.file.WatchService;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashMap;
+import java.util.Map;
+
+import ru.lanwen.jenkins.juseppe.gen.UpdateSiteGen;
+import ru.lanwen.jenkins.juseppe.props.Props;
 
 import static java.lang.String.format;
 import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
@@ -29,6 +37,7 @@ public class WatchFiles extends Thread {
     private WatchService watcher;
     private Path path;
     private Props props;
+    private Map<WatchKey, Path> keys;
 
     private WatchFiles() {
         setDaemon(true);
@@ -37,36 +46,99 @@ private WatchFiles() {
     public WatchFiles configureFor(Props props) throws IOException {
         this.props = props;
         path = Paths.get(props.getPluginsDir());
+        this.keys = new HashMap<>();
         setName(format("file-watcher-%s", path.getFileName()));
 
-
         watcher = this.path.getFileSystem().newWatchService();
-        path.register(watcher,
-                ENTRY_CREATE,
-                ENTRY_DELETE,
-                ENTRY_MODIFY
-        );
+        walkAndRegisterDirectories(path);
+
         return this;
     }
 
     public static WatchFiles watchFor(Props props) throws IOException {
         return new WatchFiles().configureFor(props);
     }
 
+    /**
+     * Register the given directory with the WatchService;
+     * This function will be called by FileVisitor
+     */
+    private void registerDirectory(Path dir) throws IOException {
+        WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
+        keys.put(key, dir);
+    }
+
+    /**
+     * Register the given directory, and all its sub-directories,
+     * with the WatchService.
+     */
+    private void walkAndRegisterDirectories(final Path start) throws IOException {
+        // register directory and sub-directories
+        Files.walkFileTree(start, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                registerDirectory(dir);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
     @Override
+    @SuppressWarnings("unchecked")
     public void run() {
         LOG.info("Start to watch for changes: {}", path);
         try {
             // get the first event before looping
             WatchKey key = watcher.take();
             while (key != null) {
 
-                if (key.pollEvents().stream().anyMatch(hasExt(".hpi").or(hasExt(".jpi")))) {
-                    LOG.trace("HPI (JPI) list modify found!");
-                    UpdateSiteGen.updateSite(props).withDefaults().toSave().saveAll();
+                Path dir = keys.get(key);
+
+                if (dir == null) {
+                    LOG.error("{}: WatchKey: {} is not recognized!", getClass(), key.toString());
+                    continue;
                 }
 
-                key.reset();
+                key.pollEvents().forEach(event -> {
+                    WatchEvent.Kind kind = event.kind();
+
+                    // Context for directory entry event is the file name of entry
+                    Path name = ((WatchEvent<Path>) event).context();
+                    Path child = dir.resolve(name);
+                    String fileName = child.getFileName().toString();
+
+                    if (fileName.endsWith(".hpi") || fileName.endsWith(".jpi")) {
+                        LOG.trace("{}: HPI (JPI) list modify found!", getClass());
+                        UpdateSiteGen.updateSite(props).withDefaults().toSave().saveAll();
+                    }
+
+                    // print out event
+                    LOG.trace("{}: {}: {}\n", getClass(), event.kind().name(), child);
+
+                    // if directory is created, and watching recursively, then register it and its sub-directories
+                    if (kind == ENTRY_CREATE) {
+                        try {
+                            if (Files.isDirectory(child)) {
+                                walkAndRegisterDirectories(child);
+                            }
+                        } catch (IOException x) {
+                            LOG.debug("{}: Unable to access {}", getClass(), child);
+                        }
+                    }
+                });
+
+                // reset key and remove from set if directory is no longer accessible
+                boolean valid = key.reset();
+
+                if (!valid) {
+                    keys.remove(key);
+
+                    // all directories are inaccessible
+                    if (keys.isEmpty()) {
+                        LOG.error("{} WatchKey map is empty. All directories are inaccessible!", getClass());
+                        break;
+                    }
+                }
                 key = watcher.take();
             }
         } catch (InterruptedException e) {

juseppe-core/src/main/java/ru/lanwen/jenkins/juseppe/gen/UpdateSiteGen.java

@@ -54,7 +54,7 @@ public UpdateSiteGen withDefaults() {
                 site -> site.withUpdateCenterVersion(Props.UPDATE_CENTER_VERSION)
                         .withId(props.getUcId())
         ).register(
-                site -> Collections.singleton(new PathPluginSource(Paths.get(props.getPluginsDir())))
+                site -> Collections.singleton(new PathPluginSource(Paths.get(props.getPluginsDir()), props.getRecursiveWatch()))
                         .forEach(source -> site.getPlugins().addAll(source.plugins()))
         ).register(
                 site -> site.getPlugins()

juseppe-core/src/main/java/ru/lanwen/jenkins/juseppe/gen/source/PathPluginSource.java

@@ -2,8 +2,6 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import ru.lanwen.jenkins.juseppe.beans.Plugin;
-import ru.lanwen.jenkins.juseppe.gen.HPI;
 
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
@@ -12,8 +10,12 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
+import ru.lanwen.jenkins.juseppe.beans.Plugin;
+import ru.lanwen.jenkins.juseppe.gen.HPI;
+
 import static java.lang.String.format;
 
 /**
@@ -23,15 +25,19 @@ public class PathPluginSource implements PluginSource {
 
     private static final Logger LOG = LoggerFactory.getLogger(PathPluginSource.class);
     private final Path pluginsDir;
+    private final boolean recursiveWatch;
 
-    public PathPluginSource(Path pluginsDir) {
+    public PathPluginSource(Path pluginsDir, boolean recursiveWatch) {
         this.pluginsDir = pluginsDir;
+        this.recursiveWatch = recursiveWatch;
     }
 
     @Override
     public List<Plugin> plugins() {
-        try (DirectoryStream<Path> paths = Files.newDirectoryStream(pluginsDir, "*.{hpi,jpi}")) {
-            return StreamSupport.stream(paths.spliterator(), false).map(path -> {
+        try (Stream<Path> paths = (recursiveWatch) ? Files.walk(pluginsDir) : Files.list(pluginsDir)) {
+            return paths
+                .filter(path -> path.toString().endsWith(".hpi") || path.toString().endsWith(".jpi"))
+                .map(path -> {
                 try {
                     LOG.trace("Process file {}", path);
 

juseppe-core/src/main/java/ru/lanwen/jenkins/juseppe/props/JuseppeEnvVars.java

@@ -17,6 +17,7 @@ public final class JuseppeEnvVars {
     static final String JUSEPPE_BASE_URI = "juseppe.baseurl";
     static final String JUSEPPE_UPDATE_CENTER_ID = "juseppe.update.center.id";
     static final String JUSEPPE_BIND_PORT = "juseppe.jetty.port";
+    static final String JUSEPPE_RECURSIVE_WATCH = "juseppe.recursive.watch";
 
     private JuseppeEnvVars() {
         throw new IllegalAccessError();
@@ -111,6 +112,16 @@ public String resolved() {
             public String resolved() {
                 return String.valueOf(populated().getPort());
             }
+        },
+
+        JUSEPPE_RECURSIVE_WATCH(
+                 JuseppeEnvVars.JUSEPPE_RECURSIVE_WATCH,
+                 "watch for file changes recursively. Defaults to `true`"
+        ) {
+            @Override
+            public String resolved() {
+                return String.valueOf(populated().getRecursiveWatch());
+            }
         };
 
         private String mapping;

juseppe-core/src/main/java/ru/lanwen/jenkins/juseppe/props/Props.java

@@ -67,6 +67,9 @@ public static Props populated() {
     @Property(JuseppeEnvVars.JUSEPPE_UPDATE_CENTER_ID)
     private String ucId = "juseppe";
 
+    @Property(JuseppeEnvVars.JUSEPPE_RECURSIVE_WATCH)
+    private boolean recursiveWatch = true;
+
     public String getUcId() {
         return ucId;
     }
@@ -103,6 +106,9 @@ public String getReleaseHistoryJsonName() {
         return releaseHistoryJsonName;
     }
 
+    public boolean getRecursiveWatch() {
+        return recursiveWatch;
+    }
 
     public Props withPluginsDir(String plugins) {
         this.pluginsDir = plugins;
@@ -149,6 +155,11 @@ public Props withUcId(String ucId) {
         return this;
     }
 
+    public Props withRecursiveWatch(boolean recursiveWatch) {
+        this.recursiveWatch = recursiveWatch;
+        return this;
+    }
+
     public void setPluginsDir(String pluginsDir) {
         this.pluginsDir = pluginsDir;
     }
@@ -184,4 +195,8 @@ public void setBaseurl(URI baseurl) {
     public void setUcId(String ucId) {
         this.ucId = ucId;
     }
+
+    public void setRecursiveWatch(boolean recursiveWatch) {
+        this.recursiveWatch = recursiveWatch;
+    }
 }

juseppe-core/src/test/java/ru/lanwen/jenkins/juseppe/HPICreationTest.java

@@ -22,6 +22,7 @@
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.everyItem;
+import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.junit.Assert.assertThat;
 import static ru.lanwen.jenkins.juseppe.gen.UpdateSiteGen.updateSite;
@@ -84,6 +85,7 @@ public void shouldContainPlugin() throws IOException {
                 .collect(toList());
 
         assertThat(contents, everyItem(containsString("clang-scanbuild-plugin")));
-        assertThat(contents, everyItem(containsString(Props.populated().getBaseurl() + "/clang-scanbuild-plugin.hpi")));
+        assertThat(contents, hasItem(containsString(Props.populated().getBaseurl() + "/clang-scanbuild-plugin.hpi")));
+        assertThat(contents, hasItem(containsString(Props.populated().getBaseurl() + "/plugins2/clang-scanbuild-plugin.hpi")));
     }
 }

juseppe-core/src/test/java/ru/lanwen/jenkins/juseppe/gen/source/PathPluginSourceTest.java

@@ -20,11 +20,25 @@ public class PathPluginSourceTest {
 
     @Test
     public void shouldFindAllPlugins() throws Exception {
-        List<Plugin> plugins = new PathPluginSource(Paths.get(getResource(PLUGINS_DIR_CLASSPATH).getFile()))
+        boolean recursiveWatch = false;
+        List<Plugin> plugins = new PathPluginSource(Paths.get(getResource(PLUGINS_DIR_CLASSPATH).getFile()),
+                                                    recursiveWatch)
                 .plugins();
 
         assertThat("plugins", plugins, hasSize(2));
         assertThat(plugins.stream().map(Plugin::getName).collect(toList()),
                 hasItems("clang-scanbuild-plugin", "jucies-sample-pipeline-dsl-extension"));
     }
-}
+
+    @Test
+    public void shouldFindAllPluginsRecursively() throws Exception {
+        boolean recursiveWatch = true;
+        List<Plugin> plugins = new PathPluginSource(Paths.get(getResource(PLUGINS_DIR_CLASSPATH).getFile()),
+                                                    recursiveWatch)
+                 .plugins();
+
+        assertThat("plugins", plugins, hasSize(4));
+        assertThat(plugins.stream().map(Plugin::getName).collect(toList()),
+                   hasItems("clang-scanbuild-plugin", "jucies-sample-pipeline-dsl-extension"));
+    }
+}