IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

NBIO fix for multiple extensions and remote URIs

Browse Source
This commit is contained in:
Jonathan Shook
2021-04-17 19:09:22 -05:00
parent 45e70cd066
commit 619a23c8ad
4 changed files with 71 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java
View File

@@ -53,7 +53,7 @@ public class NBIO implements NBPathsAPI.Facets {
public static List<String> readLines(String filename) {
Content<?> data = NBIO.all().prefix("data").name(filename).first().orElseThrow(
() -> new BasicError("Unable to read lines from " + filename)
() -> new BasicError("Unable to read lines from " + filename)
);
String[] split = data.getCharBuffer().toString().split("\n");
return Arrays.asList(split);
@@ -63,7 +63,7 @@ public class NBIO implements NBPathsAPI.Facets {
return NBIO.readFileDelimCSV(filename, ',', searchPaths);
}
public static CSVParser readFileDelimCSV(String filename,char delim, String... searchPaths) {
public static CSVParser readFileDelimCSV(String filename, char delim, String... searchPaths) {
Reader reader = NBIO.readReader(filename, searchPaths);
CSVFormat format = CSVFormat.DEFAULT.withDelimiter(delim).withFirstRecordAsHeader();
try {
@@ -90,7 +90,7 @@ public class NBIO implements NBPathsAPI.Facets {
public static Path getFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
return first.orElseThrow(
() -> new BasicError("Unable to find loadable content at " + String.join(",",potentials))
() -> new BasicError("Unable to find loadable content at " + String.join(",", potentials))
).asPath();
}
@@ -247,7 +247,7 @@ public class NBIO implements NBPathsAPI.Facets {
public List<List<Content<?>>> resolveEach() {
List<List<Content<?>>> resolved = new ArrayList<>();
for (String name : names) {
LinkedHashSet<String> slotSearchPaths = expandNamesAndSuffixes(List.of(name), extensions);
LinkedHashSet<String> slotSearchPaths = expandNamesAndSuffixes(List.of(name), new HashSet<>(extensions));
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
List<Content<?>> contents = resolver.resolve(slotSearchPath);
@@ -260,31 +260,38 @@ public class NBIO implements NBPathsAPI.Facets {
// for testing
public LinkedHashSet<String> expandNamesAndSuffixes() {
return expandNamesAndSuffixes(names, extensions);
return expandNamesAndSuffixes(names, new HashSet(extensions));
}
// for testing
public LinkedHashSet<String> expandNamesAndSuffixes(
List<String> _names,
List<String> _suffixes) {
Set<String> _suffixes) {
LinkedHashSet<String> searches = new LinkedHashSet<>();
if (_names.size()==0 && prefixes.size()==0) {
if (_names.size() == 0 && prefixes.size() == 0) {
searches.add(".*");
} else if (_names.size()>0 && _suffixes.size()==0) {
} else if (_names.size() > 0 && _suffixes.size() == 0) {
searches.addAll(_names);
} else if (_names.size()==0 && _suffixes.size()>0) {
_suffixes.stream().map(s -> ".*"+s).forEach(searches::add);
} else if (_names.size() == 0 && _suffixes.size() > 0) {
_suffixes.stream().map(s -> ".*" + s).forEach(searches::add);
} else {
for (String name : _names) {
if (!name.equals(".*")) {
searches.add(name);
}
boolean suffixed = false;
for (String suffix : _suffixes) {
if (!name.endsWith(suffix)) {
searches.add(name+suffix);
if (name.endsWith(suffix)) {
suffixed = true;
break;
}
}
if (!suffixed) {
for (String suffix : _suffixes) {
searches.add(name + suffix);
}
}
}
@@ -305,12 +312,12 @@ public class NBIO implements NBPathsAPI.Facets {
// If this has no names or suffixes included, use a wildcard for all resources found
// under the respective directory roots for the prefixes
if (searches.size()==0) {
if (searches.size() == 0) {
searches.add(".*");
}
for (String prefix : prefixes) {
List<Path> directories = resolver.resolveDirectory(prefix);
NBIOWalker.CollectVisitor capture = new NBIOWalker.CollectVisitor(true,false);
NBIOWalker.CollectVisitor capture = new NBIOWalker.CollectVisitor(true, false);
for (Path dirPath : directories) {
@@ -330,8 +337,8 @@ public class NBIO implements NBPathsAPI.Facets {
@Override
public List<Path> relativeTo(String... base) {
String base1 = base[0];
String[] rest = new String[base.length-1];
System.arraycopy(base,1,rest,0,rest.length);
String[] rest = new String[base.length - 1];
System.arraycopy(base, 1, rest, 0, rest.length);
List<Path> paths = new ArrayList<>();
@@ -339,7 +346,7 @@ public class NBIO implements NBPathsAPI.Facets {
for (Content<?> c : list) {
Path path = c.asPath();
Path fsBase = path.getFileSystem().getPath(base1,rest);
Path fsBase = path.getFileSystem().getPath(base1, rest);
Path relative = fsBase.relativize(path);
paths.add(relative);
}

29
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForFilesystem.java
View File

@@ -1,27 +1,18 @@
package io.nosqlbench.nb.api.content;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class ResolverForFilesystem implements ContentResolver {
public static ResolverForFilesystem INSTANCE = new ResolverForFilesystem();
private Path resolvePath(URI uri) {
if (uri.getScheme() != null && !uri.getScheme().isEmpty() && !uri.getScheme().equals("file")) {
return null;
}
Path pathFromUri = Path.of(uri.getPath());
if (Files.isReadable(pathFromUri)) {
return pathFromUri;
}
return null;
}
private final static Logger logger = LogManager.getLogger(ResolverForFilesystem.class);
@Override
public List<Content<?>> resolve(URI uri) {
@@ -45,6 +36,18 @@ public class ResolverForFilesystem implements ContentResolver {
return dirs;
}
private Path resolvePath(URI uri) {
if (uri.getScheme() != null && !uri.getScheme().isEmpty() && !uri.getScheme().equals("file")) {
return null;
}
Path pathFromUri = Path.of(uri.getPath());
if (Files.isReadable(pathFromUri)) {
return pathFromUri;
}
return null;
}
public String toString() {
return this.getClass().getSimpleName();
}

27
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForURL.java
View File

@@ -8,6 +8,7 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -18,21 +19,33 @@ public class ResolverForURL implements ContentResolver {
@Override
public List<Content<?>> resolve(URI uri) {
if (uri.getScheme()==null) {
List<Content<?>> contents = new ArrayList<>();
URLContent urlContent = resolveURI(uri);
if (urlContent!=null) {
return List.of(urlContent);
} else {
return List.of();
}
if (uri.getScheme().equals("http")
|| uri.getScheme().equals("https")) {
}
private URLContent resolveURI(URI uri) {
if (uri.getScheme() == null) {
return null;
}
if (uri.getScheme().equals("http") || uri.getScheme().equals("https")) {
try {
URL url = uri.toURL();
InputStream inputStream = url.openStream();
logger.debug("Found accessible remote file at " + url.toString());
return List.of(new URLContent(url, inputStream));
logger.debug("Found accessible remote file at " + url);
return new URLContent(url, inputStream);
} catch (IOException e) {
throw new RuntimeException(e);
logger.warn("Unable to find content at URI '" + uri + "', this often indicates a configuration error.");
return null;
}
} else {
return null;
}
return List.of(); }
}
@Override
public List<Path> resolveDirectory(URI uri) {

15
nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java
View File

@@ -24,7 +24,7 @@ public class NBIOTest {
public void testExpandWildcardAndExtensionsOnly() {
NBIO extensions = (NBIO) NBIO.all().name(".*").extension("foo","bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly(".*.foo",".*.bar");
assertThat(searches).contains(".*.foo",".*.bar");
}
@Test
@@ -35,11 +35,17 @@ public class NBIOTest {
}
@Test
public void testExpandNamesAndExtensions() {
public void testExpandNamesAndExtensionsIfNotExtended() {
NBIO extensions = (NBIO) NBIO.all().name("foo").extension("baz","beez");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).contains("foo.baz","foo.beez");
}
@Test
public void testExpandNamesAndExtensionsAvoidsExtendedAlreadyExtended() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("baz","beez");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar","foo.bar.baz","foo.bar.beez");
assertThat(searches).contains("foo.bar");
}
@Test
@@ -222,4 +228,5 @@ public class NBIOTest {
List<Content<?>> gammas = gammasSearch.list();
assertThat(gammas).hasSize(1);
}
}