diff --git a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java index dfaa787cc..a6225d0d2 100644 --- a/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java +++ b/engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java @@ -4,8 +4,9 @@ import io.nosqlbench.docsys.core.PathWalker; import io.nosqlbench.engine.api.activityconfig.StatementsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; -import io.nosqlbench.engine.api.exceptions.BasicError; -import io.nosqlbench.nb.api.pathutil.NBPaths; +import io.nosqlbench.nb.api.content.Content; +import io.nosqlbench.nb.api.content.NBIO; +import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.nb.api.pathutil.NBPaths; import io.nosqlbench.engine.api.util.StrInterpolator; import org.slf4j.Logger; @@ -28,8 +29,10 @@ public class NBCLIScenarioParser { private final static Logger logger = LoggerFactory.getLogger(NBCLIScenarioParser.class); public static boolean isFoundWorkload(String word) { - Optional workloadPath = NBPaths.findOptionalPath(word, "yaml", false, "activities"); - return workloadPath.isPresent(); + Optional> found = NBIO.all().prefix("activities").exact().name(word).extension("yaml").first(); + return found.isPresent(); +// Optional workloadPath = NBPathOldUtil.findOptionalPathIn(word, "yaml", false, "activities"); +// return workloadPath.isPresent(); } public static void parseScenarioCommand(LinkedList arglist) { diff --git a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java index ee44dec50..675f0369d 100644 --- a/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java +++ b/engine-cli/src/test/java/io/nosqlbench/engine/cli/NBCLIScenarioParserTest.java @@ -1,12 +1,11 @@ package io.nosqlbench.engine.cli; -import io.nosqlbench.engine.api.exceptions.BasicError; +import io.nosqlbench.nb.api.errors.BasicError; import org.junit.Test; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.*; public class NBCLIScenarioParserTest { diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java new file mode 100644 index 000000000..6a47526ae --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java @@ -0,0 +1,21 @@ +package io.nosqlbench.nb.api.content; + +import java.net.URI; +import java.nio.CharBuffer; +import java.nio.file.Path; + +/** + * A generic content wrapper for anything that can be given to a NoSQLBench runtime + * using a specific type of locator. + * @param + */ +public interface Content { + + T getLocation(); + URI getURI(); + CharBuffer getCharBuffer(); + public default String asString() { + return getCharBuffer().toString(); + } + Path asPath(); +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/ContentResolver.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ContentResolver.java new file mode 100644 index 000000000..ec6ec931e --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ContentResolver.java @@ -0,0 +1,40 @@ +package io.nosqlbench.nb.api.content; + +import java.net.URI; +import java.nio.file.Path; +import java.util.Optional; + +public interface ContentResolver { + + /** + * Locate the content referenced by the specified name. Content is any + * URL or file path which contains data to be loaded. + *

+ * Implementors should take care to ensure the following conditions are met: + * + *

    + *
  • For URL style content, resolution is only successful if a stream to download the content + * * is acquired.
    • + *
  • For file paths, resolution is only successful if the filesystem does a standard access + * * check for readability of a file that is present.
    • + *
+ * + * A content resolver may be given a path which is fundamentally the scheme. It is + * required that the resolver return null for such URI values. + * + * @param uri The URI of a content location, like a file name or URL. + * @return A content element which may then be used to access the content + */ + Content resolve(URI uri); + + default Content resolve(String uri) { + return resolve(URI.create(uri)); + } + + Optional resolveDirectory(URI uri); + + default Optional resolveDirectory(String uri) { + return resolveDirectory(URI.create(uri)); + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java new file mode 100644 index 000000000..99cba676c --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java @@ -0,0 +1,229 @@ +package io.nosqlbench.nb.api.content; + +import io.nosqlbench.nb.api.content.fluent.NBPathsAPI; + +import java.nio.file.Path; +import java.util.*; + +/** + * NBIO is a helper utility packaged as a search builder and fluent API. + * It uses value semantics internally, so it is safe to re-use as a + * stateful configuration builder for finding files in various ways. + *

+ * Since this is meant to ease development around a usually over-complicated + * surface area in the JVM (Files, Paths, URIs, accessing data, knowing where it comes + * from, searching for it, etc), more emphasis was put on ease of use and + * clarity than efficiency. This set of classes is not expected to be used + * much in NoSqlBench after initialization. + */ +public class NBIO implements NBPathsAPI.Facets { + + private URIResolver resolver; + private MatchType matchas = MatchType.exact; + + private List names = new ArrayList<>(); + private List extensions = new ArrayList<>(); + private List searchPaths = new ArrayList<>(); + + + private enum MatchType { + exact, + suffix, + pattern + } + + private NBIO() { + } + + private NBIO(URIResolver resolver, + MatchType matchas, + List searchPaths, + List names, + List extensions) { + this.resolver = resolver; + this.matchas = matchas; + this.searchPaths = searchPaths; + this.names = names; + this.extensions = extensions; + } + + @Override + public NBPathsAPI.ForContentSource localContent() { + this.resolver = URIResolvers.inFS().inCP(); + return this; + } + + @Override + public NBPathsAPI.ForContentSource remoteContent() { + this.resolver = URIResolvers.inURLs(); + return this; + } + + @Override + public NBPathsAPI.ForContentSource internalContent() { + this.resolver = URIResolvers.inClasspath(); + return this; + } + + @Override + public NBPathsAPI.ForContentSource fileContent() { + this.resolver = URIResolvers.inFS(); + return this; + } + + @Override + public NBPathsAPI.ForContentSource allContent() { + this.resolver = URIResolvers.inFS().inCP().inURLs(); + return this; + } + + @Override + public NBPathsAPI.WantsContentName exact() { + return new NBIO(resolver, MatchType.exact, searchPaths, names, extensions); + } + + @Override + public NBPathsAPI.WantsContentName matchtail() { + return new NBIO(resolver, MatchType.suffix, searchPaths, names, extensions); + } + + @Override + public NBPathsAPI.WantsContentName regex() { + return new NBIO(resolver, MatchType.pattern, searchPaths, names, extensions); + } + + @Override + public NBPathsAPI.ForPrefix prefix(String... searchPaths) { + ArrayList addingPaths = new ArrayList<>(this.searchPaths); + addingPaths.addAll(Arrays.asList(searchPaths)); + return new NBIO(resolver, matchas, addingPaths, names, extensions); + } + + @Override + public NBPathsAPI.ForName name(String... searchNames) { + ArrayList addingNames = new ArrayList<>(this.names); + addingNames.addAll(Arrays.asList(searchNames)); + return new NBIO(resolver, matchas, searchPaths, addingNames, extensions); + } + + @Override + public NBPathsAPI.ForExtension extension(String... extensions) { + ArrayList addingExtensions = new ArrayList<>(this.extensions); + for (String addingExtension : extensions) { + addingExtensions.add(addingExtension.startsWith(".") ? addingExtension : "." + addingExtension); + } + return new NBIO(resolver, matchas, searchPaths, names, addingExtensions); + } + + /** + * Search for named resources everywhere: URLs, filesystem, classpath + * + * @return a builder + */ + public static NBPathsAPI.ForContentSource all() { + return new NBIO().allContent(); + } + + /** + * Search for named resources in the classpath + * + * @return a builder + */ + public static NBPathsAPI.ForContentSource classpath() { + return new NBIO().internalContent(); + } + + /** + * Search for named resources on the filesystem + * + * @return a builder + */ + public static NBPathsAPI.ForContentSource fs() { + return new NBIO().fileContent(); + } + + /** + * Search for named resources locally: filesystem, classpath + * + * @return a builder + */ + public static NBPathsAPI.ForContentSource local() { + return new NBIO().localContent(); + } + + /** + * Search for named resources only in URLs + * + * @return a builder + */ + public static NBPathsAPI.ForContentSource remote() { + return new NBIO().remoteContent(); + } + + + @Override + public Optional> first() { + Content found = null; + LinkedHashSet specificPathsToSearch = expandSearches(); + for (String candidatePath : specificPathsToSearch) { + Content content = resolver.resolve(candidatePath); + if (content!=null) { + return Optional.of(content); + } + } + return Optional.empty(); + } + + @Override + public List>> resolveEach() { + List>> resolved = new ArrayList<>(); + for (String name : names) { + LinkedHashSet slotSearchPaths = expandSearches(name); + Content content = null; + for (String slotSearchPath : slotSearchPaths) { + content = resolver.resolve(slotSearchPath); + if (content!=null) { + break; + } + } + resolved.add(Optional.ofNullable(content)); + } + + return resolved; + } + + // for testing + public LinkedHashSet expandSearches() { + LinkedHashSet searchSet = new LinkedHashSet<>(extensions.size()*names.size()*searchPaths.size()); + for (String name : names) { + searchSet.addAll(expandSearches(name)); + } + return searchSet; + } + + // for testing + public LinkedHashSet expandSearches(String name) { + + LinkedHashSet searchSet = new LinkedHashSet<>(); + + List searchPathsToTry = new ArrayList<>(); + searchPathsToTry.add(""); + searchPathsToTry.addAll(searchPaths); + + List extensionsToTry = new ArrayList<>(); + extensionsToTry.add(""); + extensionsToTry.addAll(extensions); + + for (String searchPath : searchPathsToTry) { + for (String extension : extensionsToTry) { + if (!name.endsWith(extension)) { + name = name+extension; + } + searchSet.add(Path.of(searchPath,name).toString()); + } + } + return searchSet; + } + + +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathContent.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathContent.java new file mode 100644 index 000000000..1db93d9e8 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathContent.java @@ -0,0 +1,49 @@ +package io.nosqlbench.nb.api.content; + +import java.io.IOException; +import java.net.URI; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; + +/** + * PathContent provides the Path-centric way of accessing + * resolved content from the URIs API. + */ +public class PathContent implements Content { + + private final Path path; + + public PathContent(Path path) { + this.path = path; + } + + @Override + public Path getLocation() { + return path; + } + + + @Override + public URI getURI() { + return path.toUri(); + } + + @Override + public CharBuffer getCharBuffer() { + try { + String content = Files.readString(path, StandardCharsets.UTF_8); + return CharBuffer.wrap(content); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Path asPath() { + return this.path; + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathFinder.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathFinder.java new file mode 100644 index 000000000..1e231d718 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/PathFinder.java @@ -0,0 +1,142 @@ +package io.nosqlbench.nb.api.content; + +import io.nosqlbench.nb.api.errors.BasicError; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.spi.FileSystemProvider; +import java.util.*; +import java.util.stream.Collectors; + +/** + * A central API for finding and accessing Paths which are either in + * the classpath or in the file system, or both. + */ +public class PathFinder { + + private final static Logger logger = LoggerFactory.getLogger(PathFinder.class); + + /** + * Find exactly zero or one matching Paths, and return an {@link Optional} of {@link Path}. + * Or, if more than one are found, throw a basic error. + * + * @param names The names of paths to find, with tailing slashes signifying directories + * @return An optional Path + * @throws BasicError if there is more than one matching path found. + */ + public static Optional find(String... names) { + List paths = findAll(names); + if (paths.size() == 0) { + return Optional.empty(); + } + if (paths.size() > 1) { + throw new BasicError("Found " + paths.size() + " paths, when only one is allowed:" + + paths.stream() + .map(p -> p.toString() + " on " + p.getFileSystem()) + .collect(Collectors.joining(",")) + ); + } + return Optional.of(paths.get(0)); + } + + /** + * Find one or more matching path, and return a list. + * + * @param pathspecs The names of paths to search for. + * @return A list of matching paths, possibly empty. + */ + static List findAll(String... pathspecs) { + List founds = new ArrayList<>(); + + for (String pathspec : pathspecs) { + + boolean wantsADirectory = pathspec.endsWith(FileSystems.getDefault().getSeparator()); + String candidatePath = wantsADirectory ? pathspec.substring(0, pathspec.length() - 1) : pathspec; + Path candidate = Path.of(candidatePath); + + findPathOnRemoteURL(pathspec).ifPresent(founds::add); + findPathOnFilesystem(candidatePath, wantsADirectory).ifPresent(founds::add); + findPathOnClasspath(candidatePath, wantsADirectory).ifPresent(founds::add); + } + return founds; + } + + private static Optional findPathOnRemoteURL(String pathspec) { + if (pathspec.toLowerCase().startsWith("http:") || + pathspec.toLowerCase().startsWith("https:")) { + Optional inputStreamForUrl = getInputStreamForUrl(pathspec); + if (inputStreamForUrl.isPresent()) { + Path found = Path.of(URI.create(pathspec)); + logger.debug("Found accessible remote file at " + found.toString()); + return Optional.of(found); + } + } + return Optional.empty(); + } + + private static Optional findPathOnClasspath(String candidatePath, boolean wantsADirectory) { + + try { + URL url = ClassLoader.getSystemResource(candidatePath); + if (url != null) { + URI uri = URI.create(url.toExternalForm()); + + FileSystem fileSystem = null; + try { + fileSystem = FileSystems.getFileSystem(uri); + } catch (FileSystemNotFoundException retried) { + try { + fileSystem = FileSystems.newFileSystem(uri, new HashMap<>()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + logger.debug("Found path in classpath: " + candidatePath + ": " + candidatePath.toString()); + return Optional.of(Path.of(uri)); + } + return Optional.empty(); + } catch (Exception e) { + logger.trace("Error while looking in classpath for " + e.getMessage(), e); + return Optional.empty(); + } + } + + private static Optional findPathOnFilesystem(String pathName, boolean wantsADirectory) { + try { + Path candidatePath = Path.of(pathName); + FileSystemProvider provider = candidatePath.getFileSystem().provider(); + provider.checkAccess(candidatePath, AccessMode.READ); + BasicFileAttributes attrs = provider.readAttributes(candidatePath, BasicFileAttributes.class); + boolean foundADirectory = attrs.isDirectory(); + if (wantsADirectory != foundADirectory) { + throw new RuntimeException("for path " + candidatePath + ", user wanted a " + + (wantsADirectory ? "directory" : "file") + ", but found a " + + (foundADirectory ? "directory" : "file") + " while searching for " + + pathName); + } + return Optional.of(candidatePath); + } catch (Exception ignored) { + return Optional.empty(); + } + } + + + private static Optional getInputStreamForUrl(String path) { + URL url; + try { + url = new URL(path); + InputStream inputStream = url.openStream(); + return Optional.ofNullable(inputStream); + } catch (Exception e) { + return Optional.empty(); + } + } + +} + diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java new file mode 100644 index 000000000..de1167d13 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java @@ -0,0 +1,68 @@ +package io.nosqlbench.nb.api.content; + +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.file.*; +import java.nio.file.spi.FileSystemProvider; +import java.util.Collections; +import java.util.Optional; + +/** + * Resolves resources which can be found via the class loader. + */ +public class ResolverForClasspath implements ContentResolver { + + public static final ContentResolver INSTANCE = new ResolverForClasspath(); + + private Path resolvePath(URI uri) { + if (uri.getScheme() != null && !uri.getScheme().isEmpty()) { + return null; + } + Path fspath = Path.of(uri.getPath()); + URL systemResource = ClassLoader.getSystemResource(uri.getPath()); + if (systemResource == null) { + return null; + } + if (fspath.getFileSystem() == null || fspath.getFileSystem() == FileSystems.getDefault()) { + return fspath; + } + + URI externalUri = URI.create(systemResource.toExternalForm()); + FileSystem fs; + try { + fs = FileSystems.getFileSystem(externalUri); + } catch (FileSystemNotFoundException notfound) { + try { + fs = FileSystems.newFileSystem(externalUri, Collections.EMPTY_MAP); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return fspath; + } + + @Override + public Content resolve(URI uri) { + Path path = resolvePath(uri); + if (path==null) { + return null; + } + return new PathContent(path); + } + + @Override + public Optional resolveDirectory(URI uri) { + Path path = resolvePath(uri); + if (path == null) { + return Optional.empty(); + } + if (Files.isDirectory(path)) { + return Optional.of(path); + } else { + return Optional.empty(); + } + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForFilesystem.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForFilesystem.java new file mode 100644 index 000000000..3090dc6d7 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForFilesystem.java @@ -0,0 +1,46 @@ +package io.nosqlbench.nb.api.content; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +public class ResolverForFilesystem implements ContentResolver { + + public static ResolverForFilesystem INSTANCE = new ResolverForFilesystem(); + + private Path resolvePath(URI uri) { + if (uri.getScheme()!=null&&!uri.getScheme().isEmpty()&&!uri.getScheme().equals("file")) { + return null; + } + Path pathFromUri = Path.of(uri.getPath()); + + if (Files.isReadable(pathFromUri)) { + return pathFromUri; + } + return null; + } + + @Override + public Content resolve(URI uri) { + Path path = resolvePath(uri); + if (path==null) { + return null; + } + return new PathContent(path); + } + + @Override + public Optional resolveDirectory(URI uri) { + Path path = resolvePath(uri); + if (path == null) { + return Optional.empty(); + } + if (Files.isDirectory(path)) { + return Optional.of(path); + } else { + return Optional.empty(); + } + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForURL.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForURL.java new file mode 100644 index 000000000..6a976778f --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForURL.java @@ -0,0 +1,41 @@ +package io.nosqlbench.nb.api.content; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.nio.file.Path; +import java.util.Optional; + +public class ResolverForURL implements ContentResolver { + + public static final ContentResolver INSTANCE = new ResolverForURL(); + private final static Logger logger = LoggerFactory.getLogger(ResolverForURL.class); + + @Override + public Content resolve(URI uri) { + if (uri.getScheme()==null) { + return null; + } + if (uri.getScheme().equals("http") + || uri.getScheme().equals("https")) { + try { + URL url = uri.toURL(); + InputStream inputStream = url.openStream(); + logger.debug("Found accessible remote file at " + url.toString()); + return new URLContent(url, inputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return null; + } + + @Override + public Optional resolveDirectory(URI uri) { + return Optional.empty(); + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolver.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolver.java new file mode 100644 index 000000000..b06e8b95a --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolver.java @@ -0,0 +1,126 @@ +package io.nosqlbench.nb.api.content; + +import org.apache.commons.math3.FieldElement; + +import java.net.URI; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * This is a stateful search object for resources like Paths or URLs. + * It provides the abilitiy to look for URIs in any form, with simple + * pluggable search back-ends, in some preferential order. + */ +public class URIResolver implements ContentResolver { + + private List loaders = new ArrayList<>(); + + private static List EVERYWHERE = List.of( + ResolverForURL.INSTANCE, + ResolverForFilesystem.INSTANCE, + ResolverForClasspath.INSTANCE + ); + + private List extensions; + private List extraPaths; + + public URIResolver() { + } + + /** + * Include resources from all known places, including remote URLs, + * the local default filesystem, and the classpath, which includes + * the jars that hold the current runtime application. + * @return this URISearch + */ + public URIResolver all() { + loaders = EVERYWHERE; + return this; + } + + /** + * Includ resources in the default filesystem + * @return this URISearch + */ + public URIResolver inFS() { + loaders.add(ResolverForFilesystem.INSTANCE); + return this; + } + + /** + * Include resources in remote URLs + * @return this URISearch + */ + public URIResolver inURLs() { + loaders.add(ResolverForURL.INSTANCE); + return this; + } + + /** + * Include resources within the classpath. + * @return this URISearch + */ + public URIResolver inCP() { + loaders.add(ResolverForClasspath.INSTANCE); + return this; + } + + public Optional> resolveOptional(String uri) { + return Optional.ofNullable(resolve(uri)); + } + + public Content resolve(String uri) { + return resolve(URI.create(uri)); + } + + @Override + public Optional resolveDirectory(URI uri) { + for (ContentResolver loader : loaders) { + Optional path = loader.resolveDirectory(uri); + if (path.isPresent()) { + return path; + } + } + return Optional.empty(); + } + + public Content resolve(URI uri) { + Content resolved = null; + for (ContentResolver loader : loaders) { + resolved = loader.resolve(uri); + if (resolved!=null) { + break; + } + } + return resolved; + } + + public List> resolveAll(String uri) { + return resolveAll(URI.create(uri)); + } + + public List> resolveAll(URI uri) { + List> allFound = new ArrayList<>(); + for (ContentResolver loader : loaders) { + Content found = loader.resolve(uri); + if (found!=null) { + allFound.add(found); + } + } + return allFound; + } + + public URIResolver extension(String extension) { + this.extensions = this.extensions==null ? new ArrayList<>() : this.extensions; + this.extensions.add(extension); + return this; + } + + public URIResolver extraPaths(String extraPath) { + this.extraPaths = this.extraPaths==null ? new ArrayList<>() : this.extraPaths; + this.extraPaths.add(Path.of(extraPath)); + return this; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolvers.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolvers.java new file mode 100644 index 000000000..0e60ed1fc --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolvers.java @@ -0,0 +1,39 @@ +package io.nosqlbench.nb.api.content; + +/** + * This is a URI-centric content locator for URLs and Paths. This central + * API is meant to make dealing with content bundling and loading easy, + * given how convoluted using the standard APIs can be. + * + *

Selecting Content Source

+ *

+ * You can load content from all sources, or any individual source, from + * URLs, the (default) filesystem, and the classpath. If you use the {@link #lookEverywhere()} + * method go get your instance, then they are all included, in the order mentioned above. + * However, it is possible to search within only one, or in a specific order by calling + * the {@link #inURLs()}, the {@link #inFS()}, or the {@link #inClasspath()}, methods respectively. + * For example, to search only in the local filesystem use {@link URIResolvers#inFS()} only.

+ * + *

Reading Content

+ *

+ * All of the + *

+ */ +public class URIResolvers { + + public static URIResolver lookEverywhere() { + return new URIResolver().all(); + } + + public static URIResolver inFS() { + return new URIResolver().inFS(); + } + + public static URIResolver inURLs() { + return new URIResolver().inURLs(); + } + + public static URIResolver inClasspath() { + return new URIResolver().inCP(); + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/URLContent.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/URLContent.java new file mode 100644 index 000000000..aaacf5c3c --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/URLContent.java @@ -0,0 +1,65 @@ +package io.nosqlbench.nb.api.content; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * StreamContent is meant for short-lived use as an efficient way to + * find a read URL content. If a caller has already acquired an + * input stream, it can be passed to the stream content holder + * to avoid double fetch or other unintuitive and inefficient + * behavior. + */ +public class URLContent implements Content { + + private final URL url; + private CharBuffer buffer; + private InputStream inputStream; + + public URLContent(URL url, InputStream inputStream) { + this.url = url; + this.inputStream = inputStream; + } + + @Override + public URL getLocation() { + return url; + } + + @Override + public URI getURI() { + try { + return url.toURI(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override + public CharBuffer getCharBuffer() { + if (buffer==null) { + InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader); + Stream lines = bufferedReader.lines(); + String buffdata = lines.collect(Collectors.joining()); + this.buffer = ByteBuffer.wrap(buffdata.getBytes(StandardCharsets.UTF_8)).asCharBuffer().asReadOnlyBuffer(); + } + + return buffer; + } + + @Override + public Path asPath() { + return null; + } +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java b/nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java new file mode 100644 index 000000000..1c0a6d854 --- /dev/null +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java @@ -0,0 +1,118 @@ +package io.nosqlbench.nb.api.content.fluent; + +import io.nosqlbench.nb.api.content.Content; + +import java.util.List; +import java.util.Optional; + +public interface NBPathsAPI { + + public static interface Facets extends + WantsSpaces, ForContentSource, ForPrefix, WantsContentName, ForName, ForExtension {} + + public static interface WantsSpaces { + /** + * Only provide content from the class path and the local filesystem. + * @return this builder + */ + ForContentSource localContent(); + + /** + * Only return content from remote URLs. If the user is providing non-URL content + * in this context, it is an error. Throw an error in that case. + * @return this builder + */ + ForContentSource remoteContent(); + + /** + * Only return content from the runtime classpath, internal resources that are bundled, + * and do not return content on the file system. + * @return this builder + */ + ForContentSource internalContent(); + + /** + * Only return content from the filesystem, but not remote URLs nor internal bundled resources. + * @return this builder + */ + ForContentSource fileContent(); + + /** + * Return content from everywhere, from remote URls, or from the file system and then the internal + * bundled content if not found in the file system first. + * @return this builder + */ + ForContentSource allContent(); + } + + public static interface ForContentSource extends ForPrefix { + /** + * Each of the prefix paths will be searched if the resource is not found with the exact + * path given. + * @param prefixPaths A list of paths to include in the search + * @return this builder + */ + ForPrefix prefix(String... prefixPaths); + } + + public static interface ForPrefix extends WantsContentName { + /** + * Only look at exact matches of the names as given, and if not found, look for exact matches + * of the path directly within each given search directory. + * @return this builders + */ + WantsContentName exact(); + + /** + * Attempt {@link #exact()} matching, and if not found, also attempt to look within any + * provided search directories recursively for a path which matches the provided path at + * the end. For example "baz.csv" will be found under search directory "foo" if it is at + * "foo/bar/baz.csv", as will "bar/baz.csv", so long as "foo" is specified as a search directory. + * @return this builder + */ + WantsContentName matchtail(); + + /** + * Attempt {@link #exact()} matching, and if not found, search in each provided search directory + * for a path name that matches the provided name as a regex pattern. + * @return this builder + */ + WantsContentName regex(); + } + + public static interface WantsContentName { + /** + * Provide the names of the resources to be resolved. More than one resource may be provided. + * @param name The name of the resource to load + * @return this builder + */ + ForName name(String... name); + } + + public static interface ForName extends ForExtension { + /** + * provide a list of optional file extensions which should be considered. If the content is + * not found under the provided name, then each of the extensios is tried in order. + * @param extensions The extension names to try + * @return this builder + */ + ForExtension extension(String... extensions); + + } + + public static interface ForExtension { + /** + * Return the result of resolving the resource. + * @return an optional {@code Content} element. + */ + Optional> first(); + + /** + * Return the result of resolving each of the resource names given. This has the same semantics + * of {@link #first()}, except that it returns a result pair-wise for each name given. + * @return A list of optional {@code Content} elements. + */ + List>> resolveEach(); + } + +} diff --git a/nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPaths.java b/nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPaths.java index 424c4b128..5c456b695 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPaths.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPaths.java @@ -235,7 +235,6 @@ public class NBPaths { }}; for (String path : paths) { - Optional stream = getInputStream(path); if (stream.isPresent()) { return Optional.of(Path.of(path)); diff --git a/nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java b/nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java new file mode 100644 index 000000000..847d153ab --- /dev/null +++ b/nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java @@ -0,0 +1,68 @@ +package io.nosqlbench.nb.api.content; + +import io.nosqlbench.nb.api.content.fluent.NBPathsAPI; +import org.junit.Test; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +public class NBIOTest { + + @Test + public void testSimpleSearch() { + NBIO extensions = (NBIO) NBIO.all().exact().name("foo.bar"); + LinkedHashSet searches = extensions.expandSearches(); + assertThat(searches).containsExactly("foo.bar"); + } + + @Test + public void testSearchPaths() { + NBIO extensions = (NBIO) NBIO.all().prefix("act1","act2").exact().name("foo.bar"); + LinkedHashSet searches = extensions.expandSearches(); + assertThat(searches).containsExactly("foo.bar","act1/foo.bar","act2/foo.bar"); + } + + @Test + public void testDefaultExtensionOn() { + NBIO extensions = (NBIO) NBIO.all().exact().name("foo.bar").extension("bar"); + LinkedHashSet searches = extensions.expandSearches(); + assertThat(searches).containsExactly("foo.bar"); + NBIO extensionsDot = (NBIO) NBIO.all().exact().name("foo.bar").extension(".bar"); + LinkedHashSet searchesDot = extensionsDot.expandSearches(); + assertThat(searchesDot).containsExactly("foo.bar"); + } + + @Test + public void testDefaultExtensionOff() { + NBIO extensions = (NBIO) NBIO.all().exact().name("foo").extension("bar"); + LinkedHashSet searches = extensions.expandSearches(); + assertThat(searches).containsExactly("foo","foo.bar"); + NBIO extensionsDot = (NBIO) NBIO.all().exact().name("foo").extension(".bar"); + LinkedHashSet searchesDot = extensionsDot.expandSearches(); + assertThat(searchesDot).containsExactly("foo","foo.bar"); + } + + @Test + public void testLoadCsv1() { + NBPathsAPI.ForContentSource forSourceType = NBIO.fs(); + NBPathsAPI.ForPrefix forPrefix = forSourceType.prefix("nesteddir1"); + NBPathsAPI.ForName forName = forPrefix.name("nesteddir2/testcsv1"); + NBPathsAPI.ForExtension forCsvExtension = forName.extension(".csv"); + Optional> testcsv1 = forCsvExtension.first(); + + assertThat(testcsv1).isNotPresent(); + } + + @Test + public void testClasspathTestResource() { + List>> optionals = + NBIO.classpath().name("nesteddir1/nesteddir2/testcsv12.csv").resolveEach(); + assertThat(optionals).hasSize(1); + Content content = optionals.get(0).get(); + assertThat(content).isNotNull(); + } + +} diff --git a/nb-api/src/test/java/io/nosqlbench/nb/api/pathutil/ResolverForURLTest.java b/nb-api/src/test/java/io/nosqlbench/nb/api/pathutil/ResolverForURLTest.java new file mode 100644 index 000000000..1f04fdc09 --- /dev/null +++ b/nb-api/src/test/java/io/nosqlbench/nb/api/pathutil/ResolverForURLTest.java @@ -0,0 +1,56 @@ +package io.nosqlbench.nb.api.pathutil; + +import io.nosqlbench.nb.api.content.Content; +import io.nosqlbench.nb.api.content.ResolverForClasspath; +import io.nosqlbench.nb.api.content.ResolverForFilesystem; +import io.nosqlbench.nb.api.content.ResolverForURL; +import org.junit.Test; + +import java.net.URL; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ResolverForURLTest { + + @Test + public void testUrlResource() { + ResolverForURL r = new ResolverForURL(); + Content c = r.resolve("http://google.com"); + assertThat(c).isNotNull(); + Object location = c.getLocation(); + assertThat(location).isInstanceOf(URL.class); + assertThat(location.toString()).isEqualTo("http://google.com"); + } + + @Test + public void testFileResource() { + String p = "src/test/resources/nesteddir1/nesteddir2/testcsv12.csv"; + ResolverForFilesystem r = new ResolverForFilesystem(); + Content c = r.resolve(p); + assertThat(c).isNotNull(); + Object location = c.getLocation(); + assertThat(location).isInstanceOf(Path.class); + assertThat(location.toString()).isEqualTo(p); + + String q = "nesteddir1/nesteddir2/testcsv12.csv"; + Content notfound = r.resolve(q); + assertThat(notfound).isNull(); + + } + + @Test + public void testCPResource() { + String p = "nesteddir1/nesteddir2/testcsv12.csv"; + ResolverForClasspath r = new ResolverForClasspath(); + Content c = r.resolve(p); + assertThat(c).isNotNull(); + Object location = c.getLocation(); + assertThat(location.toString()).isEqualTo(p); + + String q = "src/test/resources/nesteddir1/nesteddir2/testcsv12.csv"; + Content notfound = r.resolve(q); + assertThat(notfound).isNull(); + } + +} diff --git a/nb-api/src/test/resources/nesteddir1/nesteddir2/testcsv12.csv b/nb-api/src/test/resources/nesteddir1/nesteddir2/testcsv12.csv new file mode 100644 index 000000000..52ea935c7 --- /dev/null +++ b/nb-api/src/test/resources/nesteddir1/nesteddir2/testcsv12.csv @@ -0,0 +1,3 @@ +heading1, heading2 +row1col1, row1col2 +"row2col1",row2col2 diff --git a/nb-api/src/test/resources/nesteddir1/testcsv1.csv b/nb-api/src/test/resources/nesteddir1/testcsv1.csv new file mode 100644 index 000000000..52ea935c7 --- /dev/null +++ b/nb-api/src/test/resources/nesteddir1/testcsv1.csv @@ -0,0 +1,3 @@ +heading1, heading2 +row1col1, row1col2 +"row2col1",row2col2 diff --git a/nb-api/src/test/resources/testdir1/testcsv3.csv b/nb-api/src/test/resources/testdir1/testcsv3.csv new file mode 100644 index 000000000..52ea935c7 --- /dev/null +++ b/nb-api/src/test/resources/testdir1/testcsv3.csv @@ -0,0 +1,3 @@ +heading1, heading2 +row1col1, row1col2 +"row2col1",row2col2