From f8e44a1ae7c8031d46d911fb61f493aa29d64523 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 27 Dec 2013 10:30:08 -0700 Subject: [PATCH] 424168 - Module [ext] should load libraries recursively from lib/ext/ + Added support for [LIB] definitions that use regex. + Using new expression for [LIB] sections in ext.mod regex:lib/ext/.*\.jar$ --- jetty-server/src/main/config/modules/ext.mod | 2 +- .../org/eclipse/jetty/start/BaseHome.java | 83 ++++++++++++++++++- .../org/eclipse/jetty/start/Classpath.java | 1 + .../main/java/org/eclipse/jetty/start/FS.java | 38 +++++++++ .../java/org/eclipse/jetty/start/Module.java | 2 +- .../org/eclipse/jetty/start/StartArgs.java | 14 +++- .../jetty/start/ConfigurationAssert.java | 27 +++++- .../org/eclipse/jetty/start/ModulesTest.java | 2 +- .../org/eclipse/jetty/start/TestUseCases.java | 6 ++ .../resources/usecases/assert-with.ext.txt | 26 ++++++ .../usecases/base.with.ext/lib/ext/agent.jar | 0 .../lib/ext/jdbc/mariadb-jdbc.jar | 0 .../lib/ext/logging/jul-to-slf4j.jar | 0 .../lib/ext/logging/logback-classic.jar | 0 .../lib/ext/logging/logback-core.jar | 0 .../lib/ext/logging/slf4j-api.jar | 0 .../base.with.ext/lib/jetty-util-alt.jar | 0 .../usecases/base.with.ext/start.ini | 6 ++ .../resources/usecases/home/modules/ext.mod | 10 +++ .../usecases/home/modules/server.mod | 3 + 20 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 jetty-start/src/test/resources/usecases/assert-with.ext.txt create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/agent.jar create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/jdbc/mariadb-jdbc.jar create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/jul-to-slf4j.jar create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/logback-classic.jar create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/logback-core.jar create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/slf4j-api.jar create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/lib/jetty-util-alt.jar create mode 100644 jetty-start/src/test/resources/usecases/base.with.ext/start.ini create mode 100644 jetty-start/src/test/resources/usecases/home/modules/ext.mod diff --git a/jetty-server/src/main/config/modules/ext.mod b/jetty-server/src/main/config/modules/ext.mod index 329a833d442..1207bddc391 100644 --- a/jetty-server/src/main/config/modules/ext.mod +++ b/jetty-server/src/main/config/modules/ext.mod @@ -3,7 +3,7 @@ # [lib] -lib/ext/*.jar +regex:lib/ext/.*\.jar$ [files] lib/ diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/BaseHome.java b/jetty-start/src/main/java/org/eclipse/jetty/start/BaseHome.java index 2986c195579..d00b1e06949 100644 --- a/jetty-start/src/main/java/org/eclipse/jetty/start/BaseHome.java +++ b/jetty-start/src/main/java/org/eclipse/jetty/start/BaseHome.java @@ -32,6 +32,8 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.eclipse.jetty.start.FS.RelativeRegexFilter; + /** * File access for ${jetty.home}, ${jetty.base}, directories. *

@@ -286,9 +288,85 @@ public class BaseHome return homeFiles; } } + + /** + * Get all of the files that are in a specific relative directory, with applied regex. + *

+ * If the same found path exists in both ${jetty.base} and ${jetty.home}, then the one in ${jetty.base} is returned + * (it overrides the one in ${jetty.home}) + *

+ * All regex paths are assumed to be in unix notation (use of "/" to separate paths, as "\" is used to escape in regex) + * + * @param regex + * the regex to use to match against the found files. + * @return the list of files found. + */ + public List listFilesRegex(String regex) + { + Objects.requireNonNull(regex,"Glob cannot be null"); + + Pattern pattern = Pattern.compile(regex); + + List homeFiles = new ArrayList<>(); + if (FS.canReadDirectory(homeDir)) + { + StartLog.debug("Finding files in ${jetty.home} that match: %s",regex); + recurseDir(homeFiles,homeDir,new FS.RelativeRegexFilter(homeDir,pattern)); + StartLog.debug("Found %,d files",homeFiles.size()); + } + + if (isBaseDifferent()) + { + // merge + List ret = new ArrayList<>(); + if (FS.canReadDirectory(baseDir)) + { + List baseFiles = new ArrayList<>(); + StartLog.debug("Finding files in ${jetty.base} that match: %s",regex); + recurseDir(baseFiles,baseDir,new FS.RelativeRegexFilter(baseDir,pattern)); + StartLog.debug("Found %,d files",baseFiles.size()); + + for (File base : baseFiles) + { + String relpath = toRelativePath(baseDir,base); + File home = new File(homeDir,FS.separators(relpath)); + if (home.exists()) + { + homeFiles.remove(home); + } + ret.add(base); + } + } + + // add any remaining home files. + ret.addAll(homeFiles); + StartLog.debug("Merged Files: %,d files%n",ret.size()); + + Collections.sort(ret,new NaturalSort.Files()); + return ret; + } + else + { + // simple return + Collections.sort(homeFiles,new NaturalSort.Files()); + return homeFiles; + } + } + + private void recurseDir(List files, File dir, RelativeRegexFilter filter) + { + // find matches first + files.addAll(Arrays.asList(dir.listFiles(filter))); + + // now dive down into sub-directories + for (File subdir : dir.listFiles(FS.DirFilter.INSTANCE)) + { + recurseDir(files,subdir,filter); + } + } /** - * Collect the list of files in both ${jetty.base} and ${jetty.home}, with , even if the same file shows up in both places. + * Collect the list of files in both ${jetty.base} and ${jetty.home}, even if the same file shows up in both places. */ public List rawListFiles(String relPathToDirectory, FileFilter filter) { @@ -338,9 +416,10 @@ public class BaseHome } } + // TODO - inline private String toRelativePath(File dir, File path) { - return dir.toURI().relativize(path.toURI()).toASCIIString(); + return FS.toRelativePath(dir,path); } /** diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/Classpath.java b/jetty-start/src/main/java/org/eclipse/jetty/start/Classpath.java index 3333b52d613..fb18fa080e4 100644 --- a/jetty-start/src/main/java/org/eclipse/jetty/start/Classpath.java +++ b/jetty-start/src/main/java/org/eclipse/jetty/start/Classpath.java @@ -75,6 +75,7 @@ public class Classpath implements Iterable public boolean addComponent(File path) { + StartLog.debug("Adding classpath component: %s",path); if ((path == null) || (!path.exists())) { // not a valid component diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/FS.java b/jetty-start/src/main/java/org/eclipse/jetty/start/FS.java index 977985ccdb4..e1119d4ce65 100644 --- a/jetty-start/src/main/java/org/eclipse/jetty/start/FS.java +++ b/jetty-start/src/main/java/org/eclipse/jetty/start/FS.java @@ -37,6 +37,39 @@ public class FS return true; } } + + public static class DirFilter implements FileFilter + { + public static final DirFilter INSTANCE = new DirFilter(); + + @Override + public boolean accept(File path) + { + return path.isDirectory(); + } + } + + public static class RelativeRegexFilter implements FileFilter + { + private final File baseDir; + private final Pattern pattern; + + public RelativeRegexFilter(File baseDir, Pattern pattern) + { + this.baseDir = baseDir; + this.pattern = pattern; + } + + @Override + public boolean accept(File path) + { + // get relative path + String relativePath = FS.toRelativePath(baseDir,path); + + // see if it matches + return (pattern.matcher(relativePath).matches()); + } + } public static class FilenameRegexFilter implements FileFilter { @@ -149,6 +182,11 @@ public class FS { return filename.toLowerCase(Locale.ENGLISH).endsWith(".xml"); } + + public static String toRelativePath(File baseDir, File path) + { + return baseDir.toURI().relativize(path.toURI()).toASCIIString(); + } public static String separators(String path) { diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/Module.java b/jetty-start/src/main/java/org/eclipse/jetty/start/Module.java index bc1359772ab..487d9ffbad3 100644 --- a/jetty-start/src/main/java/org/eclipse/jetty/start/Module.java +++ b/jetty-start/src/main/java/org/eclipse/jetty/start/Module.java @@ -306,7 +306,7 @@ public class Module } else { - // blank lines and comments are valid for initialize section + // blank lines and comments are valid for ini-template section if ((line.length() == 0) || line.startsWith("#")) { if ("INI-TEMPLATE".equals(sectionType)) diff --git a/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java b/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java index 97875637e6d..0b140e1d9f9 100644 --- a/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java +++ b/jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java @@ -323,8 +323,20 @@ public class StartArgs for (String rawlibref : module.getLibs()) { String libref = properties.expand(rawlibref); + + if (libref.startsWith("regex:")) + { + String regex = libref.substring("regex:".length()); + for (File libfile : baseHome.listFilesRegex(regex)) + { + classpath.addComponent(libfile); + } + continue; + } + libref = FS.separators(libref); + // Any globs here? if (libref.contains("*")) { // Glob Reference @@ -895,7 +907,7 @@ public class StartArgs { parse(line,StartArgs.CMD_LINE_SOURCE); } - + return this; } diff --git a/jetty-start/src/test/java/org/eclipse/jetty/start/ConfigurationAssert.java b/jetty-start/src/test/java/org/eclipse/jetty/start/ConfigurationAssert.java index 0f739369f8c..382611b7467 100644 --- a/jetty-start/src/test/java/org/eclipse/jetty/start/ConfigurationAssert.java +++ b/jetty-start/src/test/java/org/eclipse/jetty/start/ConfigurationAssert.java @@ -96,7 +96,7 @@ public class ConfigurationAssert } } List actualProperties = new ArrayList<>(); - for(Prop prop: args.getProperties()) + for (Prop prop : args.getProperties()) { String name = prop.key; if ("jetty.home".equals(name) || "jetty.base".equals(name) || prop.origin.equals(Props.ORIGIN_SYSPROP)) @@ -120,10 +120,31 @@ public class ConfigurationAssert List actualDownloads = new ArrayList<>(); for (FileArg darg : args.getFiles()) { - actualDownloads.add(String.format("%s:%s",darg.uri,darg.location)); + if (darg.uri != null) + { + actualDownloads.add(String.format("%s:%s",darg.uri,darg.location)); + } } assertContainsUnordered("Downloads",expectedDownloads,actualDownloads); - + + // Validate Files/Dirs creation + List expectedFiles = new ArrayList<>(); + for(String line: textFile) + { + if(line.startsWith("FILE|")) + { + expectedFiles.add(getValue(line)); + } + } + List actualFiles = new ArrayList<>(); + for(FileArg farg: args.getFiles()) + { + if(farg.uri == null) + { + actualFiles.add(farg.location); + } + } + assertContainsUnordered("Files/Dirs",expectedFiles,actualFiles); } private static String shorten(BaseHome baseHome, File path, File testResourcesDir) diff --git a/jetty-start/src/test/java/org/eclipse/jetty/start/ModulesTest.java b/jetty-start/src/test/java/org/eclipse/jetty/start/ModulesTest.java index ddf005e587e..1e3348ade19 100644 --- a/jetty-start/src/test/java/org/eclipse/jetty/start/ModulesTest.java +++ b/jetty-start/src/test/java/org/eclipse/jetty/start/ModulesTest.java @@ -44,7 +44,7 @@ public class ModulesTest Modules modules = new Modules(); modules.registerAll(basehome, DEFAULT_ARGS); - Assert.assertThat("Module count",modules.count(),is(29)); + Assert.assertThat("Module count",modules.count(),is(30)); } @Test diff --git a/jetty-start/src/test/java/org/eclipse/jetty/start/TestUseCases.java b/jetty-start/src/test/java/org/eclipse/jetty/start/TestUseCases.java index 5173afd72c8..4790eff1f6f 100644 --- a/jetty-start/src/test/java/org/eclipse/jetty/start/TestUseCases.java +++ b/jetty-start/src/test/java/org/eclipse/jetty/start/TestUseCases.java @@ -73,6 +73,12 @@ public class TestUseCases assertUseCase("home","base.with.db","assert-with-db.txt"); } + @Test + public void testWithDeepExt() throws Exception + { + assertUseCase("home","base.with.ext","assert-with.ext.txt"); + } + @Test public void testWithPropsBasic() throws Exception { diff --git a/jetty-start/src/test/resources/usecases/assert-with.ext.txt b/jetty-start/src/test/resources/usecases/assert-with.ext.txt new file mode 100644 index 00000000000..7759a419943 --- /dev/null +++ b/jetty-start/src/test/resources/usecases/assert-with.ext.txt @@ -0,0 +1,26 @@ +# The XMLs we expect (order is important) +XML|${jetty.home}/etc/jetty.xml +XML|${jetty.home}/etc/jetty-http.xml + +# The LIBs we expect (order is irrelevant) +LIB|${jetty.home}/lib/jetty-continuation-TEST.jar +LIB|${jetty.home}/lib/jetty-http-TEST.jar +LIB|${jetty.home}/lib/jetty-io-TEST.jar +LIB|${jetty.home}/lib/jetty-schemas-3.1.jar +LIB|${jetty.home}/lib/jetty-server-TEST.jar +LIB|${jetty.home}/lib/jetty-util-TEST.jar +LIB|${jetty.home}/lib/jetty-xml-TEST.jar +LIB|${jetty.home}/lib/servlet-api-3.1.jar +LIB|${jetty.base}/lib/ext/agent.jar +LIB|${jetty.base}/lib/ext/jdbc/mariadb-jdbc.jar +LIB|${jetty.base}/lib/ext/logging/slf4j-api.jar +LIB|${jetty.base}/lib/ext/logging/jul-to-slf4j.jar +LIB|${jetty.base}/lib/ext/logging/logback-core.jar +LIB|${jetty.base}/lib/ext/logging/logback-classic.jar + +# The Properties we expect (order is irrelevant) +PROP|jetty.port=9090 + +# Files / Directories to create +FILE|lib/ +FILE|lib/ext/ diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/agent.jar b/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/agent.jar new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/jdbc/mariadb-jdbc.jar b/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/jdbc/mariadb-jdbc.jar new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/jul-to-slf4j.jar b/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/jul-to-slf4j.jar new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/logback-classic.jar b/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/logback-classic.jar new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/logback-core.jar b/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/logback-core.jar new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/slf4j-api.jar b/jetty-start/src/test/resources/usecases/base.with.ext/lib/ext/logging/slf4j-api.jar new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/lib/jetty-util-alt.jar b/jetty-start/src/test/resources/usecases/base.with.ext/lib/jetty-util-alt.jar new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jetty-start/src/test/resources/usecases/base.with.ext/start.ini b/jetty-start/src/test/resources/usecases/base.with.ext/start.ini new file mode 100644 index 00000000000..c8fa93065c4 --- /dev/null +++ b/jetty-start/src/test/resources/usecases/base.with.ext/start.ini @@ -0,0 +1,6 @@ + +--module=server +--module=http +--module=ext + +jetty.port=9090 diff --git a/jetty-start/src/test/resources/usecases/home/modules/ext.mod b/jetty-start/src/test/resources/usecases/home/modules/ext.mod new file mode 100644 index 00000000000..c84697a5631 --- /dev/null +++ b/jetty-start/src/test/resources/usecases/home/modules/ext.mod @@ -0,0 +1,10 @@ +# +# ext module +# + +[lib] +regex:lib/ext/.*\.jar$ + +[files] +lib/ +lib/ext/ diff --git a/jetty-start/src/test/resources/usecases/home/modules/server.mod b/jetty-start/src/test/resources/usecases/home/modules/server.mod index d0c2de35d74..b1e9f336738 100644 --- a/jetty-start/src/test/resources/usecases/home/modules/server.mod +++ b/jetty-start/src/test/resources/usecases/home/modules/server.mod @@ -2,6 +2,9 @@ # Base server # +[optional] +ext + [depend] base xml