resource loading API improvements

This commit is contained in:
Jonathan Shook 2020-04-06 03:04:37 -05:00
parent c21f41694c
commit d887d3b8b4
20 changed files with 1125 additions and 7 deletions

View File

@ -4,8 +4,9 @@ import io.nosqlbench.docsys.core.PathWalker;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.exceptions.BasicError;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.engine.api.util.StrInterpolator;
import org.slf4j.Logger;
@ -28,8 +29,10 @@ public class NBCLIScenarioParser {
private final static Logger logger = LoggerFactory.getLogger(NBCLIScenarioParser.class);
public static boolean isFoundWorkload(String word) {
Optional<Path> workloadPath = NBPaths.findOptionalPath(word, "yaml", false, "activities");
return workloadPath.isPresent();
Optional<Content<?>> found = NBIO.all().prefix("activities").exact().name(word).extension("yaml").first();
return found.isPresent();
// Optional<Path> workloadPath = NBPathOldUtil.findOptionalPathIn(word, "yaml", false, "activities");
// return workloadPath.isPresent();
}
public static void parseScenarioCommand(LinkedList<String> arglist) {

View File

@ -1,12 +1,11 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.exceptions.BasicError;
import io.nosqlbench.nb.api.errors.BasicError;
import org.junit.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.*;
public class NBCLIScenarioParserTest {

View File

@ -0,0 +1,21 @@
package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.CharBuffer;
import java.nio.file.Path;
/**
* A generic content wrapper for anything that can be given to a NoSQLBench runtime
* using a specific type of locator.
* @param <T>
*/
public interface Content<T> {
T getLocation();
URI getURI();
CharBuffer getCharBuffer();
public default String asString() {
return getCharBuffer().toString();
}
Path asPath();
}

View File

@ -0,0 +1,40 @@
package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.file.Path;
import java.util.Optional;
public interface ContentResolver {
/**
* Locate the content referenced by the specified name. Content is any
* URL or file path which contains data to be loaded.
* <p>
* Implementors should take care to ensure the following conditions are met:
*
* <UL>
* <LI>For URL style content, resolution is only successful if a stream to download the content
* * is acquired.</LI>
* <LI>For file paths, resolution is only successful if the filesystem does a standard access
* * check for readability of a file that is present.</LI>
* </UL>
*
* A content resolver may be given a path which is fundamentally the scheme. It is
* required that the resolver return null for such URI values.
*
* @param uri The URI of a content location, like a file name or URL.
* @return A content element which may then be used to access the content
*/
Content<?> resolve(URI uri);
default Content<?> resolve(String uri) {
return resolve(URI.create(uri));
}
Optional<Path> resolveDirectory(URI uri);
default Optional<Path> resolveDirectory(String uri) {
return resolveDirectory(URI.create(uri));
}
}

View File

@ -0,0 +1,229 @@
package io.nosqlbench.nb.api.content;
import io.nosqlbench.nb.api.content.fluent.NBPathsAPI;
import java.nio.file.Path;
import java.util.*;
/**
* NBIO is a helper utility packaged as a search builder and fluent API.
* It uses value semantics internally, so it is safe to re-use as a
* stateful configuration builder for finding files in various ways.
* <p>
* Since this is meant to ease development around a usually over-complicated
* surface area in the JVM (Files, Paths, URIs, accessing data, knowing where it comes
* from, searching for it, etc), more emphasis was put on ease of use and
* clarity than efficiency. This set of classes is not expected to be used
* much in NoSqlBench after initialization.
*/
public class NBIO implements NBPathsAPI.Facets {
private URIResolver resolver;
private MatchType matchas = MatchType.exact;
private List<String> names = new ArrayList<>();
private List<String> extensions = new ArrayList<>();
private List<String> searchPaths = new ArrayList<>();
private enum MatchType {
exact,
suffix,
pattern
}
private NBIO() {
}
private NBIO(URIResolver resolver,
MatchType matchas,
List<String> searchPaths,
List<String> names,
List<String> extensions) {
this.resolver = resolver;
this.matchas = matchas;
this.searchPaths = searchPaths;
this.names = names;
this.extensions = extensions;
}
@Override
public NBPathsAPI.ForContentSource localContent() {
this.resolver = URIResolvers.inFS().inCP();
return this;
}
@Override
public NBPathsAPI.ForContentSource remoteContent() {
this.resolver = URIResolvers.inURLs();
return this;
}
@Override
public NBPathsAPI.ForContentSource internalContent() {
this.resolver = URIResolvers.inClasspath();
return this;
}
@Override
public NBPathsAPI.ForContentSource fileContent() {
this.resolver = URIResolvers.inFS();
return this;
}
@Override
public NBPathsAPI.ForContentSource allContent() {
this.resolver = URIResolvers.inFS().inCP().inURLs();
return this;
}
@Override
public NBPathsAPI.WantsContentName exact() {
return new NBIO(resolver, MatchType.exact, searchPaths, names, extensions);
}
@Override
public NBPathsAPI.WantsContentName matchtail() {
return new NBIO(resolver, MatchType.suffix, searchPaths, names, extensions);
}
@Override
public NBPathsAPI.WantsContentName regex() {
return new NBIO(resolver, MatchType.pattern, searchPaths, names, extensions);
}
@Override
public NBPathsAPI.ForPrefix prefix(String... searchPaths) {
ArrayList<String> addingPaths = new ArrayList<>(this.searchPaths);
addingPaths.addAll(Arrays.asList(searchPaths));
return new NBIO(resolver, matchas, addingPaths, names, extensions);
}
@Override
public NBPathsAPI.ForName name(String... searchNames) {
ArrayList<String> addingNames = new ArrayList<>(this.names);
addingNames.addAll(Arrays.asList(searchNames));
return new NBIO(resolver, matchas, searchPaths, addingNames, extensions);
}
@Override
public NBPathsAPI.ForExtension extension(String... extensions) {
ArrayList<String> addingExtensions = new ArrayList<>(this.extensions);
for (String addingExtension : extensions) {
addingExtensions.add(addingExtension.startsWith(".") ? addingExtension : "." + addingExtension);
}
return new NBIO(resolver, matchas, searchPaths, names, addingExtensions);
}
/**
* Search for named resources everywhere: URLs, filesystem, classpath
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource all() {
return new NBIO().allContent();
}
/**
* Search for named resources in the classpath
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource classpath() {
return new NBIO().internalContent();
}
/**
* Search for named resources on the filesystem
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource fs() {
return new NBIO().fileContent();
}
/**
* Search for named resources locally: filesystem, classpath
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource local() {
return new NBIO().localContent();
}
/**
* Search for named resources only in URLs
*
* @return a builder
*/
public static NBPathsAPI.ForContentSource remote() {
return new NBIO().remoteContent();
}
@Override
public Optional<Content<?>> first() {
Content<?> found = null;
LinkedHashSet<String> specificPathsToSearch = expandSearches();
for (String candidatePath : specificPathsToSearch) {
Content<?> content = resolver.resolve(candidatePath);
if (content!=null) {
return Optional.of(content);
}
}
return Optional.empty();
}
@Override
public List<Optional<Content<?>>> resolveEach() {
List<Optional<Content<?>>> resolved = new ArrayList<>();
for (String name : names) {
LinkedHashSet<String> slotSearchPaths = expandSearches(name);
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
content = resolver.resolve(slotSearchPath);
if (content!=null) {
break;
}
}
resolved.add(Optional.ofNullable(content));
}
return resolved;
}
// for testing
public LinkedHashSet<String> expandSearches() {
LinkedHashSet<String> searchSet = new LinkedHashSet<>(extensions.size()*names.size()*searchPaths.size());
for (String name : names) {
searchSet.addAll(expandSearches(name));
}
return searchSet;
}
// for testing
public LinkedHashSet<String> expandSearches(String name) {
LinkedHashSet<String> searchSet = new LinkedHashSet<>();
List<String> searchPathsToTry = new ArrayList<>();
searchPathsToTry.add("");
searchPathsToTry.addAll(searchPaths);
List<String> extensionsToTry = new ArrayList<>();
extensionsToTry.add("");
extensionsToTry.addAll(extensions);
for (String searchPath : searchPathsToTry) {
for (String extension : extensionsToTry) {
if (!name.endsWith(extension)) {
name = name+extension;
}
searchSet.add(Path.of(searchPath,name).toString());
}
}
return searchSet;
}
}

View File

@ -0,0 +1,49 @@
package io.nosqlbench.nb.api.content;
import java.io.IOException;
import java.net.URI;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
/**
* PathContent provides the Path-centric way of accessing
* resolved content from the URIs API.
*/
public class PathContent implements Content<Path> {
private final Path path;
public PathContent(Path path) {
this.path = path;
}
@Override
public Path getLocation() {
return path;
}
@Override
public URI getURI() {
return path.toUri();
}
@Override
public CharBuffer getCharBuffer() {
try {
String content = Files.readString(path, StandardCharsets.UTF_8);
return CharBuffer.wrap(content);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public Path asPath() {
return this.path;
}
}

View File

@ -0,0 +1,142 @@
package io.nosqlbench.nb.api.content;
import io.nosqlbench.nb.api.errors.BasicError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.*;
import java.util.stream.Collectors;
/**
* A central API for finding and accessing Paths which are either in
* the classpath or in the file system, or both.
*/
public class PathFinder {
private final static Logger logger = LoggerFactory.getLogger(PathFinder.class);
/**
* Find exactly zero or one matching Paths, and return an {@link Optional} of {@link Path}.
* Or, if more than one are found, throw a basic error.
*
* @param names The names of paths to find, with tailing slashes signifying directories
* @return An optional Path
* @throws BasicError if there is more than one matching path found.
*/
public static Optional<Path> find(String... names) {
List<Path> paths = findAll(names);
if (paths.size() == 0) {
return Optional.empty();
}
if (paths.size() > 1) {
throw new BasicError("Found " + paths.size() + " paths, when only one is allowed:" +
paths.stream()
.map(p -> p.toString() + " on " + p.getFileSystem())
.collect(Collectors.joining(","))
);
}
return Optional.of(paths.get(0));
}
/**
* Find one or more matching path, and return a list.
*
* @param pathspecs The names of paths to search for.
* @return A list of matching paths, possibly empty.
*/
static List<Path> findAll(String... pathspecs) {
List<Path> founds = new ArrayList<>();
for (String pathspec : pathspecs) {
boolean wantsADirectory = pathspec.endsWith(FileSystems.getDefault().getSeparator());
String candidatePath = wantsADirectory ? pathspec.substring(0, pathspec.length() - 1) : pathspec;
Path candidate = Path.of(candidatePath);
findPathOnRemoteURL(pathspec).ifPresent(founds::add);
findPathOnFilesystem(candidatePath, wantsADirectory).ifPresent(founds::add);
findPathOnClasspath(candidatePath, wantsADirectory).ifPresent(founds::add);
}
return founds;
}
private static Optional<Path> findPathOnRemoteURL(String pathspec) {
if (pathspec.toLowerCase().startsWith("http:") ||
pathspec.toLowerCase().startsWith("https:")) {
Optional<InputStream> inputStreamForUrl = getInputStreamForUrl(pathspec);
if (inputStreamForUrl.isPresent()) {
Path found = Path.of(URI.create(pathspec));
logger.debug("Found accessible remote file at " + found.toString());
return Optional.of(found);
}
}
return Optional.empty();
}
private static Optional<Path> findPathOnClasspath(String candidatePath, boolean wantsADirectory) {
try {
URL url = ClassLoader.getSystemResource(candidatePath);
if (url != null) {
URI uri = URI.create(url.toExternalForm());
FileSystem fileSystem = null;
try {
fileSystem = FileSystems.getFileSystem(uri);
} catch (FileSystemNotFoundException retried) {
try {
fileSystem = FileSystems.newFileSystem(uri, new HashMap<>());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
logger.debug("Found path in classpath: " + candidatePath + ": " + candidatePath.toString());
return Optional.of(Path.of(uri));
}
return Optional.empty();
} catch (Exception e) {
logger.trace("Error while looking in classpath for " + e.getMessage(), e);
return Optional.empty();
}
}
private static Optional<Path> findPathOnFilesystem(String pathName, boolean wantsADirectory) {
try {
Path candidatePath = Path.of(pathName);
FileSystemProvider provider = candidatePath.getFileSystem().provider();
provider.checkAccess(candidatePath, AccessMode.READ);
BasicFileAttributes attrs = provider.readAttributes(candidatePath, BasicFileAttributes.class);
boolean foundADirectory = attrs.isDirectory();
if (wantsADirectory != foundADirectory) {
throw new RuntimeException("for path " + candidatePath + ", user wanted a " +
(wantsADirectory ? "directory" : "file") + ", but found a " +
(foundADirectory ? "directory" : "file") + " while searching for " +
pathName);
}
return Optional.of(candidatePath);
} catch (Exception ignored) {
return Optional.empty();
}
}
private static Optional<InputStream> getInputStreamForUrl(String path) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
return Optional.ofNullable(inputStream);
} catch (Exception e) {
return Optional.empty();
}
}
}

View File

@ -0,0 +1,68 @@
package io.nosqlbench.nb.api.content;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.file.*;
import java.nio.file.spi.FileSystemProvider;
import java.util.Collections;
import java.util.Optional;
/**
* Resolves resources which can be found via the class loader.
*/
public class ResolverForClasspath implements ContentResolver {
public static final ContentResolver INSTANCE = new ResolverForClasspath();
private Path resolvePath(URI uri) {
if (uri.getScheme() != null && !uri.getScheme().isEmpty()) {
return null;
}
Path fspath = Path.of(uri.getPath());
URL systemResource = ClassLoader.getSystemResource(uri.getPath());
if (systemResource == null) {
return null;
}
if (fspath.getFileSystem() == null || fspath.getFileSystem() == FileSystems.getDefault()) {
return fspath;
}
URI externalUri = URI.create(systemResource.toExternalForm());
FileSystem fs;
try {
fs = FileSystems.getFileSystem(externalUri);
} catch (FileSystemNotFoundException notfound) {
try {
fs = FileSystems.newFileSystem(externalUri, Collections.EMPTY_MAP);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return fspath;
}
@Override
public Content<?> resolve(URI uri) {
Path path = resolvePath(uri);
if (path==null) {
return null;
}
return new PathContent(path);
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
Path path = resolvePath(uri);
if (path == null) {
return Optional.empty();
}
if (Files.isDirectory(path)) {
return Optional.of(path);
} else {
return Optional.empty();
}
}
}

View File

@ -0,0 +1,46 @@
package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
public class ResolverForFilesystem implements ContentResolver {
public static ResolverForFilesystem INSTANCE = new ResolverForFilesystem();
private Path resolvePath(URI uri) {
if (uri.getScheme()!=null&&!uri.getScheme().isEmpty()&&!uri.getScheme().equals("file")) {
return null;
}
Path pathFromUri = Path.of(uri.getPath());
if (Files.isReadable(pathFromUri)) {
return pathFromUri;
}
return null;
}
@Override
public Content<?> resolve(URI uri) {
Path path = resolvePath(uri);
if (path==null) {
return null;
}
return new PathContent(path);
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
Path path = resolvePath(uri);
if (path == null) {
return Optional.empty();
}
if (Files.isDirectory(path)) {
return Optional.of(path);
} else {
return Optional.empty();
}
}
}

View File

@ -0,0 +1,41 @@
package io.nosqlbench.nb.api.content;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import java.util.Optional;
public class ResolverForURL implements ContentResolver {
public static final ContentResolver INSTANCE = new ResolverForURL();
private final static Logger logger = LoggerFactory.getLogger(ResolverForURL.class);
@Override
public Content<?> resolve(URI uri) {
if (uri.getScheme()==null) {
return null;
}
if (uri.getScheme().equals("http")
|| uri.getScheme().equals("https")) {
try {
URL url = uri.toURL();
InputStream inputStream = url.openStream();
logger.debug("Found accessible remote file at " + url.toString());
return new URLContent(url, inputStream);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return null;
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
return Optional.empty();
}
}

View File

@ -0,0 +1,126 @@
package io.nosqlbench.nb.api.content;
import org.apache.commons.math3.FieldElement;
import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
/**
* This is a stateful search object for resources like Paths or URLs.
* It provides the abilitiy to look for URIs in any form, with simple
* pluggable search back-ends, in some preferential order.
*/
public class URIResolver implements ContentResolver {
private List<ContentResolver> loaders = new ArrayList<>();
private static List<ContentResolver> EVERYWHERE = List.of(
ResolverForURL.INSTANCE,
ResolverForFilesystem.INSTANCE,
ResolverForClasspath.INSTANCE
);
private List<String> extensions;
private List<Path> extraPaths;
public URIResolver() {
}
/**
* Include resources from all known places, including remote URLs,
* the local default filesystem, and the classpath, which includes
* the jars that hold the current runtime application.
* @return this URISearch
*/
public URIResolver all() {
loaders = EVERYWHERE;
return this;
}
/**
* Includ resources in the default filesystem
* @return this URISearch
*/
public URIResolver inFS() {
loaders.add(ResolverForFilesystem.INSTANCE);
return this;
}
/**
* Include resources in remote URLs
* @return this URISearch
*/
public URIResolver inURLs() {
loaders.add(ResolverForURL.INSTANCE);
return this;
}
/**
* Include resources within the classpath.
* @return this URISearch
*/
public URIResolver inCP() {
loaders.add(ResolverForClasspath.INSTANCE);
return this;
}
public Optional<Content<?>> resolveOptional(String uri) {
return Optional.ofNullable(resolve(uri));
}
public Content<?> resolve(String uri) {
return resolve(URI.create(uri));
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
for (ContentResolver loader : loaders) {
Optional<Path> path = loader.resolveDirectory(uri);
if (path.isPresent()) {
return path;
}
}
return Optional.empty();
}
public Content<?> resolve(URI uri) {
Content<?> resolved = null;
for (ContentResolver loader : loaders) {
resolved = loader.resolve(uri);
if (resolved!=null) {
break;
}
}
return resolved;
}
public List<Content<?>> resolveAll(String uri) {
return resolveAll(URI.create(uri));
}
public List<Content<?>> resolveAll(URI uri) {
List<Content<?>> allFound = new ArrayList<>();
for (ContentResolver loader : loaders) {
Content<?> found = loader.resolve(uri);
if (found!=null) {
allFound.add(found);
}
}
return allFound;
}
public URIResolver extension(String extension) {
this.extensions = this.extensions==null ? new ArrayList<>() : this.extensions;
this.extensions.add(extension);
return this;
}
public URIResolver extraPaths(String extraPath) {
this.extraPaths = this.extraPaths==null ? new ArrayList<>() : this.extraPaths;
this.extraPaths.add(Path.of(extraPath));
return this;
}
}

View File

@ -0,0 +1,39 @@
package io.nosqlbench.nb.api.content;
/**
* This is a URI-centric content locator for URLs and Paths. This central
* API is meant to make dealing with content bundling and loading easy,
* given how convoluted using the standard APIs can be.
*
* <H2>Selecting Content Source</H2>
* <p>
* You can load content from all sources, or any individual source, from
* URLs, the (default) filesystem, and the classpath. If you use the {@link #lookEverywhere()}
* method go get your instance, then they are all included, in the order mentioned above.
* However, it is possible to search within only one, or in a specific order by calling
* the {@link #inURLs()}, the {@link #inFS()}, or the {@link #inClasspath()}, methods respectively.
* For example, to search only in the local filesystem use {@link URIResolvers#inFS()} only.</p>
*
* <H2>Reading Content</H2>
* <p>
* All of the
* </p>
*/
public class URIResolvers {
public static URIResolver lookEverywhere() {
return new URIResolver().all();
}
public static URIResolver inFS() {
return new URIResolver().inFS();
}
public static URIResolver inURLs() {
return new URIResolver().inURLs();
}
public static URIResolver inClasspath() {
return new URIResolver().inCP();
}
}

View File

@ -0,0 +1,65 @@
package io.nosqlbench.nb.api.content;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* StreamContent is meant for short-lived use as an efficient way to
* find a read URL content. If a caller has already acquired an
* input stream, it can be passed to the stream content holder
* to avoid double fetch or other unintuitive and inefficient
* behavior.
*/
public class URLContent implements Content<URL> {
private final URL url;
private CharBuffer buffer;
private InputStream inputStream;
public URLContent(URL url, InputStream inputStream) {
this.url = url;
this.inputStream = inputStream;
}
@Override
public URL getLocation() {
return url;
}
@Override
public URI getURI() {
try {
return url.toURI();
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
@Override
public CharBuffer getCharBuffer() {
if (buffer==null) {
InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
Stream<String> lines = bufferedReader.lines();
String buffdata = lines.collect(Collectors.joining());
this.buffer = ByteBuffer.wrap(buffdata.getBytes(StandardCharsets.UTF_8)).asCharBuffer().asReadOnlyBuffer();
}
return buffer;
}
@Override
public Path asPath() {
return null;
}
}

View File

@ -0,0 +1,118 @@
package io.nosqlbench.nb.api.content.fluent;
import io.nosqlbench.nb.api.content.Content;
import java.util.List;
import java.util.Optional;
public interface NBPathsAPI {
public static interface Facets extends
WantsSpaces, ForContentSource, ForPrefix, WantsContentName, ForName, ForExtension {}
public static interface WantsSpaces {
/**
* Only provide content from the class path and the local filesystem.
* @return this builder
*/
ForContentSource localContent();
/**
* Only return content from remote URLs. If the user is providing non-URL content
* in this context, it is an error. Throw an error in that case.
* @return this builder
*/
ForContentSource remoteContent();
/**
* Only return content from the runtime classpath, internal resources that are bundled,
* and do not return content on the file system.
* @return this builder
*/
ForContentSource internalContent();
/**
* Only return content from the filesystem, but not remote URLs nor internal bundled resources.
* @return this builder
*/
ForContentSource fileContent();
/**
* Return content from everywhere, from remote URls, or from the file system and then the internal
* bundled content if not found in the file system first.
* @return this builder
*/
ForContentSource allContent();
}
public static interface ForContentSource extends ForPrefix {
/**
* Each of the prefix paths will be searched if the resource is not found with the exact
* path given.
* @param prefixPaths A list of paths to include in the search
* @return this builder
*/
ForPrefix prefix(String... prefixPaths);
}
public static interface ForPrefix extends WantsContentName {
/**
* Only look at exact matches of the names as given, and if not found, look for exact matches
* of the path directly within each given search directory.
* @return this builders
*/
WantsContentName exact();
/**
* Attempt {@link #exact()} matching, and if not found, also attempt to look within any
* provided search directories recursively for a path which matches the provided path at
* the end. For example "baz.csv" will be found under search directory "foo" if it is at
* "foo/bar/baz.csv", as will "bar/baz.csv", so long as "foo" is specified as a search directory.
* @return this builder
*/
WantsContentName matchtail();
/**
* Attempt {@link #exact()} matching, and if not found, search in each provided search directory
* for a path name that matches the provided name as a regex pattern.
* @return this builder
*/
WantsContentName regex();
}
public static interface WantsContentName {
/**
* Provide the names of the resources to be resolved. More than one resource may be provided.
* @param name The name of the resource to load
* @return this builder
*/
ForName name(String... name);
}
public static interface ForName extends ForExtension {
/**
* provide a list of optional file extensions which should be considered. If the content is
* not found under the provided name, then each of the extensios is tried in order.
* @param extensions The extension names to try
* @return this builder
*/
ForExtension extension(String... extensions);
}
public static interface ForExtension {
/**
* Return the result of resolving the resource.
* @return an optional {@code Content<?>} element.
*/
Optional<Content<?>> first();
/**
* Return the result of resolving each of the resource names given. This has the same semantics
* of {@link #first()}, except that it returns a result pair-wise for each name given.
* @return A list of optional {@code Content<?>} elements.
*/
List<Optional<Content<?>>> resolveEach();
}
}

View File

@ -235,7 +235,6 @@ public class NBPaths {
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return Optional.of(Path.of(path));

View File

@ -0,0 +1,68 @@
package io.nosqlbench.nb.api.content;
import io.nosqlbench.nb.api.content.fluent.NBPathsAPI;
import org.junit.Test;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
public class NBIOTest {
@Test
public void testSimpleSearch() {
NBIO extensions = (NBIO) NBIO.all().exact().name("foo.bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testSearchPaths() {
NBIO extensions = (NBIO) NBIO.all().prefix("act1","act2").exact().name("foo.bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar","act1/foo.bar","act2/foo.bar");
}
@Test
public void testDefaultExtensionOn() {
NBIO extensions = (NBIO) NBIO.all().exact().name("foo.bar").extension("bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().exact().name("foo.bar").extension(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandSearches();
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testDefaultExtensionOff() {
NBIO extensions = (NBIO) NBIO.all().exact().name("foo").extension("bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo","foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().exact().name("foo").extension(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandSearches();
assertThat(searchesDot).containsExactly("foo","foo.bar");
}
@Test
public void testLoadCsv1() {
NBPathsAPI.ForContentSource forSourceType = NBIO.fs();
NBPathsAPI.ForPrefix forPrefix = forSourceType.prefix("nesteddir1");
NBPathsAPI.ForName forName = forPrefix.name("nesteddir2/testcsv1");
NBPathsAPI.ForExtension forCsvExtension = forName.extension(".csv");
Optional<Content<?>> testcsv1 = forCsvExtension.first();
assertThat(testcsv1).isNotPresent();
}
@Test
public void testClasspathTestResource() {
List<Optional<Content<?>>> optionals =
NBIO.classpath().name("nesteddir1/nesteddir2/testcsv12.csv").resolveEach();
assertThat(optionals).hasSize(1);
Content<?> content = optionals.get(0).get();
assertThat(content).isNotNull();
}
}

View File

@ -0,0 +1,56 @@
package io.nosqlbench.nb.api.pathutil;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.ResolverForClasspath;
import io.nosqlbench.nb.api.content.ResolverForFilesystem;
import io.nosqlbench.nb.api.content.ResolverForURL;
import org.junit.Test;
import java.net.URL;
import java.nio.file.Path;
import static org.assertj.core.api.Assertions.assertThat;
public class ResolverForURLTest {
@Test
public void testUrlResource() {
ResolverForURL r = new ResolverForURL();
Content<?> c = r.resolve("http://google.com");
assertThat(c).isNotNull();
Object location = c.getLocation();
assertThat(location).isInstanceOf(URL.class);
assertThat(location.toString()).isEqualTo("http://google.com");
}
@Test
public void testFileResource() {
String p = "src/test/resources/nesteddir1/nesteddir2/testcsv12.csv";
ResolverForFilesystem r = new ResolverForFilesystem();
Content<?> c = r.resolve(p);
assertThat(c).isNotNull();
Object location = c.getLocation();
assertThat(location).isInstanceOf(Path.class);
assertThat(location.toString()).isEqualTo(p);
String q = "nesteddir1/nesteddir2/testcsv12.csv";
Content<?> notfound = r.resolve(q);
assertThat(notfound).isNull();
}
@Test
public void testCPResource() {
String p = "nesteddir1/nesteddir2/testcsv12.csv";
ResolverForClasspath r = new ResolverForClasspath();
Content<?> c = r.resolve(p);
assertThat(c).isNotNull();
Object location = c.getLocation();
assertThat(location.toString()).isEqualTo(p);
String q = "src/test/resources/nesteddir1/nesteddir2/testcsv12.csv";
Content<?> notfound = r.resolve(q);
assertThat(notfound).isNull();
}
}

View File

@ -0,0 +1,3 @@
heading1, heading2
row1col1, row1col2
"row2col1",row2col2
1 heading1 heading2
2 row1col1 row1col2
3 row2col1 row2col2

View File

@ -0,0 +1,3 @@
heading1, heading2
row1col1, row1col2
"row2col1",row2col2
1 heading1 heading2
2 row1col1 row1col2
3 row2col1 row2col2

View File

@ -0,0 +1,3 @@
heading1, heading2
row1col1, row1col2
"row2col1",row2col2
1 heading1 heading2
2 row1col1 row1col2
3 row2col1 row2col2