diff --git a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java index 05c50ea9a..84fa8eb5a 100644 --- a/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java +++ b/engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java @@ -247,6 +247,10 @@ public class NBCLIScenarioParser { public static List getWorkloadsWithScenarioScripts() { + List> activities = NBIO.all().prefix("activities").regex().name(".*\\.yaml").list(); + + broken here + String dir = "activities/"; NBPathsAPI.ForContentSource content = NBIO.all(dir).prefix("activities").exact().(".yaml"); diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java index 6a47526ae..e68333940 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java @@ -14,8 +14,10 @@ public interface Content { T getLocation(); URI getURI(); CharBuffer getCharBuffer(); + public default String asString() { return getCharBuffer().toString(); } + Path asPath(); } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java index 99cba676c..8d4c6f436 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java @@ -2,8 +2,12 @@ package io.nosqlbench.nb.api.content; import io.nosqlbench.nb.api.content.fluent.NBPathsAPI; +import java.io.IOException; +import java.nio.file.DirectoryStream; import java.nio.file.Path; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * NBIO is a helper utility packaged as a search builder and fluent API. @@ -167,7 +171,7 @@ public class NBIO implements NBPathsAPI.Facets { LinkedHashSet specificPathsToSearch = expandSearches(); for (String candidatePath : specificPathsToSearch) { Content content = resolver.resolve(candidatePath); - if (content!=null) { + if (content != null) { return Optional.of(content); } } @@ -182,7 +186,7 @@ public class NBIO implements NBPathsAPI.Facets { Content content = null; for (String slotSearchPath : slotSearchPaths) { content = resolver.resolve(slotSearchPath); - if (content!=null) { + if (content != null) { break; } } @@ -192,9 +196,10 @@ public class NBIO implements NBPathsAPI.Facets { return resolved; } + // for testing public LinkedHashSet expandSearches() { - LinkedHashSet searchSet = new LinkedHashSet<>(extensions.size()*names.size()*searchPaths.size()); + LinkedHashSet searchSet = new LinkedHashSet<>(extensions.size() * names.size() * searchPaths.size()); for (String name : names) { searchSet.addAll(expandSearches(name)); } @@ -217,13 +222,80 @@ public class NBIO implements NBPathsAPI.Facets { for (String searchPath : searchPathsToTry) { for (String extension : extensionsToTry) { if (!name.endsWith(extension)) { - name = name+extension; + name = name + extension; } - searchSet.add(Path.of(searchPath,name).toString()); + searchSet.add(Path.of(searchPath, name).toString()); } } return searchSet; } + @Override + public List> list() { + List> foundFiles = new ArrayList<>(); + + for (String searchPath : searchPaths) { + Optional opath = resolver.resolveDirectory(searchPath); + Path path = opath.orElseThrow(); + + FileCapture capture = new FileCapture(); + for (String name : names) { + + for (String extension : extensions) { + if (!extension.startsWith(".")) { + extension = "." + extension; + } + String pattern = name.endsWith(extension) ? name : name + Pattern.quote(extension); + RegexPathFilter filter = new RegexPathFilter(pattern); + + NBIOWalker.walk(path, capture, filter); + } + + } + + for (Path foundPath : capture) { + Path fullPath = path.resolve(foundPath); + foundFiles.add(new PathContent(fullPath)); + } + + } + return foundFiles; + } + + private static class RegexPathFilter implements DirectoryStream.Filter { + + private final Pattern regex; + + public RegexPathFilter(String pattern) { + this.regex = Pattern.compile(pattern); + } + + @Override + public boolean accept(Path entry) throws IOException { + String input = entry.toString(); + Matcher matcher = regex.matcher(input); + boolean matches = matcher.matches(); + return matches; + } + + public String toString() { + return regex.toString(); + } + } + + private static class FileCapture implements NBIOWalker.PathVisitor, Iterable { + List found = new ArrayList<>(); + + @Override + public void visit(Path foundPath) { + found.add(foundPath); + } + + @Override + public Iterator iterator() { + return found.iterator(); + } + } + } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java new file mode 100644 index 000000000..075340bf4 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java @@ -0,0 +1,95 @@ +package io.nosqlbench.nb.api.content; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.spi.FileSystemProvider; +import java.util.ArrayList; +import java.util.List; + +public class NBIOWalker { + private final static Logger logger = LogManager.getLogger(NBIOWalker.class); + public static void walk(Path p, PathVisitor v) { + walk(p,v,NBIOWalker.WALK_ALL); + } + + public static List findAll(Path p) { + Collect fileCollector = new Collect(true, false); + walk(p, fileCollector); + return fileCollector.get(); + + } + + public static void walk(Path p, PathVisitor v, DirectoryStream.Filter filter) { + try { + FileSystemProvider provider = p.getFileSystem().provider(); + DirectoryStream paths = provider.newDirectoryStream(p, (Path r) -> true); + List pathlist = new ArrayList<>(); + for (Path path : paths) { + pathlist.add(path); + } + for (Path path : pathlist) { + if (path.getFileSystem().provider().readAttributes(path, BasicFileAttributes.class).isDirectory()) { + v.preVisitDir(path); + walk(path, v, filter); + v.postVisitDir(path); + } else if (filter.accept(path)) { + v.preVisitFile(path); + v.visit(path); + v.postVisitFile(path); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public interface PathVisitor { + void visit(Path p); + default void preVisitFile(Path path) {} + default void postVisitFile(Path path) {} + default void preVisitDir(Path path) {} + default void postVisitDir(Path path) {} + } + + public static DirectoryStream.Filter WALK_ALL = entry -> true; + + public static class Collect implements PathVisitor { + private final List listing = new ArrayList<>(); + private final boolean collectFiles; + private final boolean collectDirectories; + + public Collect(boolean collectFiles, boolean collectDirectories) { + + this.collectFiles = collectFiles; + this.collectDirectories = collectDirectories; + } + + public List get() { + return listing; + } + + @Override + public void visit(Path p) { + } + + @Override + public void preVisitFile(Path path) { + if (this.collectFiles) { + listing.add(path); + } + } + + @Override + public void preVisitDir(Path path) { + if (this.collectDirectories) { + listing.add(path); + } + } + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java index 2b3c03b28..66bff468c 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java @@ -33,8 +33,10 @@ public class ResolverForClasspath implements ContentResolver { } URI resolved = URI.create(systemResource.toExternalForm()); if (resolved.getScheme().equals("file")) { - return Path.of(uri.getPath()); -// return Path.of(resolved.getPath()); + Path current = Paths.get("").toAbsolutePath(); + Path logical = Path.of(resolved.getPath()); + Path relativePath = current.relativize(logical); + return relativePath; } FileSystem fs; diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java index 1c0a6d854..0911e2790 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java @@ -113,6 +113,8 @@ public interface NBPathsAPI { * @return A list of optional {@code Content} elements. */ List>> resolveEach(); + + List> list(); } } diff --git a/nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java b/nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java index 847d153ab..7a4ad4810 100644 --- a/nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java +++ b/nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java @@ -65,4 +65,37 @@ public class NBIOTest { assertThat(content).isNotNull(); } + @Test + public void testPathSearchForExtension() { + List> list = NBIO.classpath() + .prefix("nesteddir1") + .regex() + .name(".*\\.csv") + .extension("csv") + .list(); + assertThat(list).hasSize(2); + } + + @Test + public void testPathSearchForExtensionMissing() { + List> list = NBIO.classpath() + .prefix("nesteddir1") + .regex() + .name(".*") + .extension("csv") + .list(); + assertThat(list).hasSize(2); + } + + @Test + public void testPathSearchForMultipleExtensions() { + List> list = NBIO.classpath() + .prefix("nesteddir1") + .regex() + .name(".*") + .extension("csv","txt") + .list(); + assertThat(list).hasSize(3); + } + } diff --git a/nb-api/src/test/resources/nesteddir1/nesteddir2/testdata12.txt b/nb-api/src/test/resources/nesteddir1/nesteddir2/testdata12.txt new file mode 100644 index 000000000..52ea935c7 --- /dev/null +++ b/nb-api/src/test/resources/nesteddir1/nesteddir2/testdata12.txt @@ -0,0 +1,3 @@ +heading1, heading2 +row1col1, row1col2 +"row2col1",row2col2