IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add support for finding content in search paths

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-07 17:39:41 -05:00
parent d5f66b326b
commit 59ee5355c4
8 changed files with 220 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -247,6 +247,10 @@ public class NBCLIScenarioParser {
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts() {
List<Content<?>> activities = NBIO.all().prefix("activities").regex().name(".*\\.yaml").list();
broken here
String dir = "activities/";
NBPathsAPI.ForContentSource content = NBIO.all(dir).prefix("activities").exact().(".yaml");

2
nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java
View File

@ -14,8 +14,10 @@ public interface Content<T> {
T getLocation();
URI getURI();
CharBuffer getCharBuffer();
public default String asString() {
return getCharBuffer().toString();
}
Path asPath();
}

82
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java
View File

@ -2,8 +2,12 @@ package io.nosqlbench.nb.api.content;
import io.nosqlbench.nb.api.content.fluent.NBPathsAPI;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Path;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* NBIO is a helper utility packaged as a search builder and fluent API.
@ -167,7 +171,7 @@ public class NBIO implements NBPathsAPI.Facets {
LinkedHashSet<String> specificPathsToSearch = expandSearches();
for (String candidatePath : specificPathsToSearch) {
Content<?> content = resolver.resolve(candidatePath);
if (content!=null) {
if (content != null) {
return Optional.of(content);
}
}
@ -182,7 +186,7 @@ public class NBIO implements NBPathsAPI.Facets {
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
content = resolver.resolve(slotSearchPath);
if (content!=null) {
if (content != null) {
break;
}
}
@ -192,9 +196,10 @@ public class NBIO implements NBPathsAPI.Facets {
return resolved;
}
// for testing
public LinkedHashSet<String> expandSearches() {
LinkedHashSet<String> searchSet = new LinkedHashSet<>(extensions.size()*names.size()*searchPaths.size());
LinkedHashSet<String> searchSet = new LinkedHashSet<>(extensions.size() * names.size() * searchPaths.size());
for (String name : names) {
searchSet.addAll(expandSearches(name));
}
@ -217,13 +222,80 @@ public class NBIO implements NBPathsAPI.Facets {
for (String searchPath : searchPathsToTry) {
for (String extension : extensionsToTry) {
if (!name.endsWith(extension)) {
name = name+extension;
name = name + extension;
}
searchSet.add(Path.of(searchPath,name).toString());
searchSet.add(Path.of(searchPath, name).toString());
}
}
return searchSet;
}
@Override
public List<Content<?>> list() {
List<Content<?>> foundFiles = new ArrayList<>();
for (String searchPath : searchPaths) {
Optional<Path> opath = resolver.resolveDirectory(searchPath);
Path path = opath.orElseThrow();
FileCapture capture = new FileCapture();
for (String name : names) {
for (String extension : extensions) {
if (!extension.startsWith(".")) {
extension = "." + extension;
}
String pattern = name.endsWith(extension) ? name : name + Pattern.quote(extension);
RegexPathFilter filter = new RegexPathFilter(pattern);
NBIOWalker.walk(path, capture, filter);
}
}
for (Path foundPath : capture) {
Path fullPath = path.resolve(foundPath);
foundFiles.add(new PathContent(fullPath));
}
}
return foundFiles;
}
private static class RegexPathFilter implements DirectoryStream.Filter<Path> {
private final Pattern regex;
public RegexPathFilter(String pattern) {
this.regex = Pattern.compile(pattern);
}
@Override
public boolean accept(Path entry) throws IOException {
String input = entry.toString();
Matcher matcher = regex.matcher(input);
boolean matches = matcher.matches();
return matches;
}
public String toString() {
return regex.toString();
}
}
private static class FileCapture implements NBIOWalker.PathVisitor, Iterable<Path> {
List<Path> found = new ArrayList<>();
@Override
public void visit(Path foundPath) {
found.add(foundPath);
}
@Override
public Iterator<Path> iterator() {
return found.iterator();
}
}
}

95
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java Normal file
View File

@ -0,0 +1,95 @@
package io.nosqlbench.nb.api.content;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList;
import java.util.List;
public class NBIOWalker {
private final static Logger logger = LogManager.getLogger(NBIOWalker.class);
public static void walk(Path p, PathVisitor v) {
walk(p,v,NBIOWalker.WALK_ALL);
}
public static List<Path> findAll(Path p) {
Collect fileCollector = new Collect(true, false);
walk(p, fileCollector);
return fileCollector.get();
}
public static void walk(Path p, PathVisitor v, DirectoryStream.Filter<Path> filter) {
try {
FileSystemProvider provider = p.getFileSystem().provider();
DirectoryStream<Path> paths = provider.newDirectoryStream(p, (Path r) -> true);
List<Path> pathlist = new ArrayList<>();
for (Path path : paths) {
pathlist.add(path);
}
for (Path path : pathlist) {
if (path.getFileSystem().provider().readAttributes(path, BasicFileAttributes.class).isDirectory()) {
v.preVisitDir(path);
walk(path, v, filter);
v.postVisitDir(path);
} else if (filter.accept(path)) {
v.preVisitFile(path);
v.visit(path);
v.postVisitFile(path);
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public interface PathVisitor {
void visit(Path p);
default void preVisitFile(Path path) {}
default void postVisitFile(Path path) {}
default void preVisitDir(Path path) {}
default void postVisitDir(Path path) {}
}
public static DirectoryStream.Filter<Path> WALK_ALL = entry -> true;
public static class Collect implements PathVisitor {
private final List<Path> listing = new ArrayList<>();
private final boolean collectFiles;
private final boolean collectDirectories;
public Collect(boolean collectFiles, boolean collectDirectories) {
this.collectFiles = collectFiles;
this.collectDirectories = collectDirectories;
}
public List<Path> get() {
return listing;
}
@Override
public void visit(Path p) {
}
@Override
public void preVisitFile(Path path) {
if (this.collectFiles) {
listing.add(path);
}
}
@Override
public void preVisitDir(Path path) {
if (this.collectDirectories) {
listing.add(path);
}
}
}
}

6
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java
View File

@ -33,8 +33,10 @@ public class ResolverForClasspath implements ContentResolver {
}
URI resolved = URI.create(systemResource.toExternalForm());
if (resolved.getScheme().equals("file")) {
return Path.of(uri.getPath());
// return Path.of(resolved.getPath());
Path current = Paths.get("").toAbsolutePath();
Path logical = Path.of(resolved.getPath());
Path relativePath = current.relativize(logical);
return relativePath;
}
FileSystem fs;

2
nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java
View File

@ -113,6 +113,8 @@ public interface NBPathsAPI {
* @return A list of optional {@code Content<?>} elements.
*/
List<Optional<Content<?>>> resolveEach();
List<Content<?>> list();
}
}

33
nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java
View File

@ -65,4 +65,37 @@ public class NBIOTest {
assertThat(content).isNotNull();
}
@Test
public void testPathSearchForExtension() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.regex()
.name(".*\\.csv")
.extension("csv")
.list();
assertThat(list).hasSize(2);
}
@Test
public void testPathSearchForExtensionMissing() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.regex()
.name(".*")
.extension("csv")
.list();
assertThat(list).hasSize(2);
}
@Test
public void testPathSearchForMultipleExtensions() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.regex()
.name(".*")
.extension("csv","txt")
.list();
assertThat(list).hasSize(3);
}
}

3
nb-api/src/test/resources/nesteddir1/nesteddir2/testdata12.txt Normal file
View File

@ -0,0 +1,3 @@
heading1, heading2
row1col1, row1col2
"row2col1",row2col2