IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

NBIO bug fixes and cleanups

Browse Source
This commit is contained in:
Jonathan Shook 2021-04-06 20:39:38 -05:00
parent abf86ec127
commit 808ba0678b
6 changed files with 103 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
nb-api/src/main/java/io/nosqlbench/nb/api/content/ContentResolver.java
View File

@ -3,7 +3,6 @@ package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;
public interface ContentResolver {
@ -15,12 +14,13 @@ public interface ContentResolver {
*
* <UL>
* <LI>For URL style content, resolution is only successful if a stream to download the content
* * is acquired.</LI>
* is acquired.</LI>
* <LI>For file paths, resolution is only successful if the filesystem does a standard access
* * check for readability of a file that is present.</LI>
* check for readability of a file that is present.</LI>
* </UL>
*
* A content resolver may be given a path which is fundamentally the scheme. It is
* A content resolver may be given a path which is fundamentally incompatible with the
* schemes supported by the resolver implementation. It is
* required that the resolver return null for such URI values.
*
* @param uri The URI of a content location, like a file name or URL.
@ -31,6 +31,14 @@ public interface ContentResolver {
return resolve(URI.create(uri));
}
/**
* Return a list of matching directory Paths for the {@link URI} which are accessible under
* the scheme of the implementing resolver. It is possible that a logical path can return
* more than one physical directory path, such as with jar files.
* @param uri
* @return A list of accessible paths matching the uri, or an empty list
*/
List<Path> resolveDirectory(URI uri);
default List<Path> resolveDirectory(String uri) {
return resolveDirectory(URI.create(uri));

96
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java
View File

@ -8,7 +8,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.nio.CharBuffer;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;
@ -22,7 +21,8 @@ import java.util.stream.Collectors;
* surface area in the JVM (Files, Paths, URIs, accessing data, knowing where it comes
* from, searching for it, etc), more emphasis was put on ease of use and
* clarity than efficiency. This set of classes is not expected to be used
* much in NoSqlBench after initialization.
* much in NoSqlBench after workload initialization, so is not performance oriented
*
*/
public class NBIO implements NBPathsAPI.Facets {
@ -247,7 +247,7 @@ public class NBIO implements NBPathsAPI.Facets {
public List<List<Content<?>>> resolveEach() {
List<List<Content<?>>> resolved = new ArrayList<>();
for (String name : names) {
LinkedHashSet<String> slotSearchPaths = expandSearches(prefixes, List.of(name), extensions, false);
LinkedHashSet<String> slotSearchPaths = expandNamesAndSuffixes(List.of(name), extensions);
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
List<Content<?>> contents = resolver.resolve(slotSearchPath);
@ -259,81 +259,42 @@ public class NBIO implements NBPathsAPI.Facets {
// for testing
public LinkedHashSet<String> expandSearches() {
return expandSearches(prefixes, names, extensions, false);
public LinkedHashSet<String> expandNamesAndSuffixes() {
return expandNamesAndSuffixes(names, extensions);
}
// for testing
public LinkedHashSet<String> expandSearches(List<String> thePrefixes, List<String> names,
List<String> suffixes, boolean eachPrefix) {
List<String> prefixesToSearch = new ArrayList<>(thePrefixes);
List<String> namesToSearch = new ArrayList<>(names);
List<String> suffixesToSearch = new ArrayList<>(suffixes);
if (prefixesToSearch.size() == 0) {
prefixesToSearch.add("");
}
if (namesToSearch.size() == 0) {
namesToSearch.add(".*");
}
if (suffixesToSearch.size() == 0) {
suffixesToSearch.add("");
}
public LinkedHashSet<String> expandNamesAndSuffixes(
List<String> _names,
List<String> _suffixes) {
LinkedHashSet<String> searches = new LinkedHashSet<>();
for (String name : namesToSearch) {
for (String suffix : suffixesToSearch) {
String search = name;
search = (search.endsWith(suffix) ? search : search + suffix);
if (eachPrefix) {
for (String prefix : prefixesToSearch) {
String withPrefix = (prefix.isEmpty() ? prefix :
prefix + FileSystems.getDefault().getSeparator())
+ search;
searches.add(withPrefix);
if (_names.size()==0 && prefixes.size()==0) {
searches.add(".*");
} else if (_names.size()>0 && _suffixes.size()==0) {
searches.addAll(_names);
} else if (_names.size()==0 && _suffixes.size()>0) {
_suffixes.stream().map(s -> ".*"+s).forEach(searches::add);
} else {
for (String name : _names) {
if (!name.equals(".*")) {
searches.add(name);
}
for (String suffix : _suffixes) {
if (!name.endsWith(suffix)) {
searches.add(name+suffix);
}
} else {
searches.add(search);
}
}
}
return searches;
}
// // for testing
// public LinkedHashSet<String> expandSearches(String name) {
//
// LinkedHashSet<String> searchSet = new LinkedHashSet<>();
//
// List<String> searchPathsToTry = new ArrayList<>();
// searchPathsToTry.add("");
// searchPathsToTry.addAll(prefixes);
//
// List<String> extensionsToTry = new ArrayList<>();
//// extensionsToTry.add("");
// extensionsToTry.addAll(extensions);
//
// for (String searchPath : searchPathsToTry) {
// for (String extension : extensionsToTry) {
// if (!name.endsWith(extension)) {
// name = name + extension;
// }
// searchSet.add(Path.of(searchPath, name).toString());
// }
// }
// return searchSet;
// }
@Override
public List<Content<?>> list() {
LinkedHashSet<String> searches = expandSearches();
LinkedHashSet<String> searches = expandNamesAndSuffixes();
LinkedHashSet<Content<?>> foundFiles = new LinkedHashSet<>();
// wrap in local search iterator
@ -342,9 +303,16 @@ public class NBIO implements NBPathsAPI.Facets {
foundFiles.addAll(founds);
}
for (String searchPath : prefixes) {
List<Path> founds = resolver.resolveDirectory(searchPath);
// If this has no names or suffixes included, use a wildcard for all resources found
// under the respective directory roots for the prefixes
if (searches.size()==0) {
searches.add(".*");
}
for (String prefix : prefixes) {
List<Path> founds = resolver.resolveDirectory(prefix);
NBIOWalker.CollectVisitor capture = new NBIOWalker.CollectVisitor(true,false);
for (Path path : founds) {
for (String searchPattern : searches) {
NBIOWalker.RegexFilter filter = new NBIOWalker.RegexFilter(searchPattern,true);

8
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBPathsAPI.java
View File

@ -57,6 +57,8 @@ public interface NBPathsAPI {
interface GetName extends GetExtension {
/**
* Provide the names of the resources to be resolved. More than one resource may be provided.
* If no name is provided, then a wildcard search is assumed.
*
* @param name The name of the resource to load
* @return this builder
*/
@ -66,7 +68,11 @@ public interface NBPathsAPI {
interface GetExtension extends DoSearch {
/**
* provide a list of optional file extensions which should be considered. If the content is
* not found under the provided name, then each of the extensios is tried in order.
* not found under the provided name, then each of the extension is tried in order.
* Any provided names are combined with the extensions to create an expanded list of
* paths to search for. if extensions are provided without a name, then wildcards are created
* with the extensions as suffix patterns.
*
* @param extensions The extension names to try
* @return this builder
*/

62
nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java
View File

@ -16,57 +16,57 @@ public class NBIOTest {
@Test
public void testFullyQualifiedNameSearches() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar");
LinkedHashSet<String> searches = extensions.expandSearches();
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testExpandWildcardAndExtensionsOnly() {
NBIO extensions = (NBIO) NBIO.all().name(".*").extension("foo","bar");
LinkedHashSet<String> searches = extensions.expandSearches();
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly(".*.foo",".*.bar");
}
@Test
public void testExpandNameOnly() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension();
LinkedHashSet<String> searches = extensions.expandSearches();
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testExpandNamesAndExtensions() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("baz","beez");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar.baz","foo.bar.beez");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar","foo.bar.baz","foo.bar.beez");
}
@Test
public void testExpandPrefixesAndFullName() {
NBIO extensions = (NBIO) NBIO.all().prefix("act1","act2").name("foo.bar");
LinkedHashSet<String> searches = extensions.expandSearches();
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testExpandAddExtensionNotNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("bar");
LinkedHashSet<String> searches = extensions.expandSearches();
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo.bar").extension(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandSearches();
LinkedHashSet<String> searchesDot = extensionsDot.expandNamesAndSuffixes();
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testExpandAddExtensionNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo").extension("bar");
LinkedHashSet<String> searches = extensions.expandSearches();
assertThat(searches).containsExactly("foo.bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo","foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo").extension(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandSearches();
assertThat(searchesDot).containsExactly("foo.bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandNamesAndSuffixes();
assertThat(searchesDot).containsExactly("foo","foo.bar");
}
@Test
@ -151,9 +151,43 @@ public class NBIOTest {
// assertThat(list).containsExactly(Paths.get("."));
List<Path> relatives = NBIORelativizer.relativizePaths(Paths.get("target/test-classes/"), list);
assertThat(relatives).hasSize(2);
}
@Test
public void testLoadNamedFileAsYmlExtension() {
List<Content<?>> list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1.yml")
.extension("abc")
.list();
assertThat(list).hasSize(1);
list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1.yml")
.list();
assertThat(list).hasSize(1);
list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1")
.extension("abc","yml")
.list();
assertThat(list).hasSize(1);
}
@Test
public void testLoadAllFilesUnderPath() {
List<Content<?>> list = null;
list = NBIO.classpath().prefix("./").list();
System.out.println("found " + list.size() + " entries for path '.'");
assertThat(list).hasSizeGreaterThan(0);
list = NBIO.fs().prefix("./").list();
System.out.println("found " + list.size() + " entries for path '.'");
assertThat(list).hasSizeGreaterThan(0);
list = NBIO.remote().prefix("./").list();
System.out.println("found " + list.size() + " entries for path '.'");
assertThat(list).hasSize(0);
}
}

2
nb-api/src/test/resources/nesteddir1/nesteddir2/testworkload1.yml Normal file
View File

@ -0,0 +1,2 @@
# This is just a testing file.
description: This is just a testing file

2
nb-api/src/test/resources/nesteddir1/nesteddir2/testworkload2.yaml Normal file
View File

@ -0,0 +1,2 @@
# This is just a testing file.
description: This is just a testing file