424168 - Module [ext] should load libraries recursively from lib/ext/

+ Added support for [LIB] definitions that use regex.
+ Using new expression for [LIB] sections in ext.mod
    regex:lib/ext/.*\.jar$

jetty-server/src/main/config/modules/ext.mod

@@ -3,7 +3,7 @@
 #
 
 [lib]
-lib/ext/*.jar
+regex:lib/ext/.*\.jar$
 
 [files]
 lib/

jetty-start/src/main/java/org/eclipse/jetty/start/BaseHome.java

@@ -32,6 +32,8 @@ import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.eclipse.jetty.start.FS.RelativeRegexFilter;
+
 /**
  * File access for <code>${jetty.home}</code>, <code>${jetty.base}</code>, directories.
  * <p>
@@ -288,7 +290,83 @@ public class BaseHome
     }
     
     /**
-     * Collect the list of files in both <code>${jetty.base}</code> and <code>${jetty.home}</code>, with , even if the same file shows up in both places.
+     * Get all of the files that are in a specific relative directory, with applied regex.
+     * <p>
+     * If the same found path exists in both <code>${jetty.base}</code> and <code>${jetty.home}</code>, then the one in <code>${jetty.base}</code> is returned
+     * (it overrides the one in ${jetty.home})
+     * <p>
+     * All regex paths are assumed to be in unix notation (use of <code>"/"</code> to separate paths, as <code>"\"</code> is used to escape in regex)
+     * 
+     * @param regex
+     *            the regex to use to match against the found files.
+     * @return the list of files found.
+     */
+    public List<File> listFilesRegex(String regex)
+    {
+        Objects.requireNonNull(regex,"Glob cannot be null");
+
+        Pattern pattern = Pattern.compile(regex);
+
+        List<File> homeFiles = new ArrayList<>();
+        if (FS.canReadDirectory(homeDir))
+        {
+            StartLog.debug("Finding files in ${jetty.home} that match: %s",regex);
+            recurseDir(homeFiles,homeDir,new FS.RelativeRegexFilter(homeDir,pattern));
+            StartLog.debug("Found %,d files",homeFiles.size());
+        }
+
+        if (isBaseDifferent())
+        {
+            // merge
+            List<File> ret = new ArrayList<>();
+            if (FS.canReadDirectory(baseDir))
+            {
+                List<File> baseFiles = new ArrayList<>();
+                StartLog.debug("Finding files in ${jetty.base} that match: %s",regex);
+                recurseDir(baseFiles,baseDir,new FS.RelativeRegexFilter(baseDir,pattern));
+                StartLog.debug("Found %,d files",baseFiles.size());
+
+                for (File base : baseFiles)
+                {
+                    String relpath = toRelativePath(baseDir,base);
+                    File home = new File(homeDir,FS.separators(relpath));
+                    if (home.exists())
+                    {
+                        homeFiles.remove(home);
+                    }
+                    ret.add(base);
+                }
+            }
+
+            // add any remaining home files.
+            ret.addAll(homeFiles);
+            StartLog.debug("Merged Files: %,d files%n",ret.size());
+
+            Collections.sort(ret,new NaturalSort.Files());
+            return ret;
+        }
+        else
+        {
+            // simple return
+            Collections.sort(homeFiles,new NaturalSort.Files());
+            return homeFiles;
+        }
+    }
+
+    private void recurseDir(List<File> files, File dir, RelativeRegexFilter filter)
+    {
+        // find matches first
+        files.addAll(Arrays.asList(dir.listFiles(filter)));
+
+        // now dive down into sub-directories
+        for (File subdir : dir.listFiles(FS.DirFilter.INSTANCE))
+        {
+            recurseDir(files,subdir,filter);
+        }
+    }
+
+    /**
+     * Collect the list of files in both <code>${jetty.base}</code> and <code>${jetty.home}</code>, even if the same file shows up in both places.
      */
     public List<File> rawListFiles(String relPathToDirectory, FileFilter filter)
     {
@@ -338,9 +416,10 @@ public class BaseHome
         }
     }
 
+    // TODO - inline
     private String toRelativePath(File dir, File path)
     {
-        return dir.toURI().relativize(path.toURI()).toASCIIString();
+        return FS.toRelativePath(dir,path);
     }
 
     /**

jetty-start/src/main/java/org/eclipse/jetty/start/Classpath.java

@@ -75,6 +75,7 @@ public class Classpath implements Iterable<File>
 
     public boolean addComponent(File path)
     {
+        StartLog.debug("Adding classpath component: %s",path);
         if ((path == null) || (!path.exists()))
         {
             // not a valid component

jetty-start/src/main/java/org/eclipse/jetty/start/FS.java

@@ -38,6 +38,39 @@ public class FS
         }
     }
     
+    public static class DirFilter implements FileFilter
+    {
+        public static final DirFilter INSTANCE = new DirFilter();
+
+        @Override
+        public boolean accept(File path)
+        {
+            return path.isDirectory();
+        }
+    }
+    
+    public static class RelativeRegexFilter implements FileFilter
+    {
+        private final File baseDir;
+        private final Pattern pattern;
+
+        public RelativeRegexFilter(File baseDir, Pattern pattern)
+        {
+            this.baseDir = baseDir;
+            this.pattern = pattern;
+        }
+
+        @Override
+        public boolean accept(File path)
+        {
+            // get relative path
+            String relativePath = FS.toRelativePath(baseDir,path);
+            
+            // see if it matches
+            return (pattern.matcher(relativePath).matches());
+        }
+    }
+
     public static class FilenameRegexFilter implements FileFilter
     {
         private final Pattern pattern;
@@ -150,6 +183,11 @@ public class FS
         return filename.toLowerCase(Locale.ENGLISH).endsWith(".xml");
     }
     
+    public static String toRelativePath(File baseDir, File path)
+    {
+        return baseDir.toURI().relativize(path.toURI()).toASCIIString();
+    }
+
     public static String separators(String path)
     {
         StringBuilder ret = new StringBuilder();

jetty-start/src/main/java/org/eclipse/jetty/start/Module.java

@@ -306,7 +306,7 @@ public class Module
                     }
                     else
                     {
-                        // blank lines and comments are valid for initialize section
+                        // blank lines and comments are valid for ini-template section
                         if ((line.length() == 0) || line.startsWith("#"))
                         {
                             if ("INI-TEMPLATE".equals(sectionType))

jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java

@@ -323,8 +323,20 @@ public class StartArgs
             for (String rawlibref : module.getLibs())
             {
                 String libref = properties.expand(rawlibref);
+
+                if (libref.startsWith("regex:"))
+                {
+                    String regex = libref.substring("regex:".length());
+                    for (File libfile : baseHome.listFilesRegex(regex))
+                    {
+                        classpath.addComponent(libfile);
+                    }
+                    continue;
+                }
+
                 libref = FS.separators(libref);
 
+                // Any globs here?
                 if (libref.contains("*"))
                 {
                     // Glob Reference

jetty-start/src/test/java/org/eclipse/jetty/start/ConfigurationAssert.java

@@ -119,11 +119,32 @@ public class ConfigurationAssert
         }
         List<String> actualDownloads = new ArrayList<>();
         for (FileArg darg : args.getFiles())
+        {
+            if (darg.uri != null)
             {
                 actualDownloads.add(String.format("%s:%s",darg.uri,darg.location));
             }
+        }
         assertContainsUnordered("Downloads",expectedDownloads,actualDownloads);
         
+        // Validate Files/Dirs creation
+        List<String> expectedFiles = new ArrayList<>();
+        for(String line: textFile)
+        {
+            if(line.startsWith("FILE|"))
+            {
+                expectedFiles.add(getValue(line));
+            }
+        }
+        List<String> actualFiles = new ArrayList<>();
+        for(FileArg farg: args.getFiles())
+        {
+            if(farg.uri == null)
+            {
+                actualFiles.add(farg.location);
+            }
+        }
+        assertContainsUnordered("Files/Dirs",expectedFiles,actualFiles);
     }
 
     private static String shorten(BaseHome baseHome, File path, File testResourcesDir)

jetty-start/src/test/java/org/eclipse/jetty/start/ModulesTest.java

@@ -44,7 +44,7 @@ public class ModulesTest
 
         Modules modules = new Modules();
         modules.registerAll(basehome, DEFAULT_ARGS);
-        Assert.assertThat("Module count",modules.count(),is(29));
+        Assert.assertThat("Module count",modules.count(),is(30));
     }
 
     @Test

jetty-start/src/test/java/org/eclipse/jetty/start/TestUseCases.java

@@ -73,6 +73,12 @@ public class TestUseCases
         assertUseCase("home","base.with.db","assert-with-db.txt");
     }
 
+    @Test
+    public void testWithDeepExt() throws Exception
+    {
+        assertUseCase("home","base.with.ext","assert-with.ext.txt");
+    }
+    
     @Test
     public void testWithPropsBasic() throws Exception
     {

jetty-start/src/test/resources/usecases/assert-with.ext.txt

@@ -0,0 +1,26 @@
+# The XMLs we expect (order is important)
+XML|${jetty.home}/etc/jetty.xml
+XML|${jetty.home}/etc/jetty-http.xml
+
+# The LIBs we expect (order is irrelevant)
+LIB|${jetty.home}/lib/jetty-continuation-TEST.jar
+LIB|${jetty.home}/lib/jetty-http-TEST.jar
+LIB|${jetty.home}/lib/jetty-io-TEST.jar
+LIB|${jetty.home}/lib/jetty-schemas-3.1.jar
+LIB|${jetty.home}/lib/jetty-server-TEST.jar
+LIB|${jetty.home}/lib/jetty-util-TEST.jar
+LIB|${jetty.home}/lib/jetty-xml-TEST.jar
+LIB|${jetty.home}/lib/servlet-api-3.1.jar
+LIB|${jetty.base}/lib/ext/agent.jar
+LIB|${jetty.base}/lib/ext/jdbc/mariadb-jdbc.jar
+LIB|${jetty.base}/lib/ext/logging/slf4j-api.jar
+LIB|${jetty.base}/lib/ext/logging/jul-to-slf4j.jar
+LIB|${jetty.base}/lib/ext/logging/logback-core.jar
+LIB|${jetty.base}/lib/ext/logging/logback-classic.jar
+
+# The Properties we expect (order is irrelevant)
+PROP|jetty.port=9090
+
+# Files / Directories to create
+FILE|lib/
+FILE|lib/ext/

jetty-start/src/test/resources/usecases/base.with.ext/start.ini

@@ -0,0 +1,6 @@
+
+--module=server
+--module=http
+--module=ext
+
+jetty.port=9090

jetty-start/src/test/resources/usecases/home/modules/ext.mod

@@ -0,0 +1,10 @@
+#
+# ext module
+#
+
+[lib]
+regex:lib/ext/.*\.jar$
+
+[files]
+lib/
+lib/ext/

jetty-start/src/test/resources/usecases/home/modules/server.mod

@@ -2,6 +2,9 @@
 # Base server
 #
 
+[optional]
+ext
+
 [depend]
 base
 xml