mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2025-02-25 18:55:28 -06:00
Merge branch 'main' of github.com:nosqlbench/nosqlbench
This commit is contained in:
@@ -43,7 +43,7 @@
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-dynamodb</artifactId>
|
||||
<version>1.12.678</version>
|
||||
<version>1.12.681</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
@@ -49,6 +49,7 @@ import io.nosqlbench.engine.core.logging.NBLoggerConfig;
|
||||
import io.nosqlbench.engine.core.metadata.MarkdownFinder;
|
||||
import io.nosqlbench.nb.annotations.Service;
|
||||
import io.nosqlbench.nb.annotations.ServiceSelector;
|
||||
import io.nosqlbench.nb.api.nbio.ResolverForNBIOCache;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.core.config.ConfigurationFactory;
|
||||
@@ -220,6 +221,26 @@ public class NBCLI implements Function<String[], Integer>, NBLabeledElement {
|
||||
NBCLI.logger = LogManager.getLogger("NBCLI");
|
||||
|
||||
NBIO.addGlobalIncludes(options.wantsIncludes());
|
||||
NBIO.setUseNBIOCache(options.wantsToUseNBIOCache());
|
||||
if(options.wantsToUseNBIOCache()) {
|
||||
logger.info(() -> "Configuring options for NBIO Cache");
|
||||
logger.info(() -> "Setting NBIO Cache Force Update to " + options.wantsNbioCacheForceUpdate());
|
||||
ResolverForNBIOCache.setForceUpdate(options.wantsNbioCacheForceUpdate());
|
||||
logger.info(() -> "Setting NBIO Cache Verify Checksum to " + options.wantsNbioCacheVerify());
|
||||
ResolverForNBIOCache.setVerifyChecksum(options.wantsNbioCacheVerify());
|
||||
if (options.getNbioCacheDir() != null) {
|
||||
logger.info(() -> "Setting NBIO Cache directory to " + options.getNbioCacheDir());
|
||||
ResolverForNBIOCache.setCacheDir(options.getNbioCacheDir());
|
||||
}
|
||||
if (options.getNbioCacheMaxRetries() != null) {
|
||||
try {
|
||||
ResolverForNBIOCache.setMaxRetries(Integer.parseInt(options.getNbioCacheMaxRetries()));
|
||||
logger.info(() -> "Setting NBIO Cache max retries to " + options.getNbioCacheMaxRetries());
|
||||
} catch (NumberFormatException e) {
|
||||
logger.error("Invalid value for nbio-cache-max-retries: " + options.getNbioCacheMaxRetries());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (options.wantsBasicHelp()) {
|
||||
System.out.println(this.loadHelpFile("basic.md"));
|
||||
|
||||
@@ -137,6 +137,11 @@ public class NBCLIOptions {
|
||||
private static final String DEFAULT_CONSOLE_PATTERN = "TERSE";
|
||||
private static final String DEFAULT_LOGFILE_PATTERN = "VERBOSE";
|
||||
private final static String ENABLE_DEDICATED_VERIFICATION_LOGGER = "--enable-dedicated-verification-logging";
|
||||
private final static String USE_NBIO_CACHE = "--use-nbio-cache";
|
||||
private final static String NBIO_CACHE_FORCE_UPDATE = "--nbio-cache-force-update";
|
||||
private final static String NBIO_CACHE_NO_VERIFY = "--nbio-cache-no-verify";
|
||||
private final static String NBIO_CACHE_DIR = "--nbio-cache-dir";
|
||||
private final static String NBIO_CACHE_MAX_RETRIES = "--nbio-cache-max-retries";
|
||||
|
||||
// private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n";
|
||||
|
||||
@@ -206,6 +211,11 @@ public class NBCLIOptions {
|
||||
private String metricsLabelSpec = "";
|
||||
private String wantsToCatResource = "";
|
||||
private long heartbeatIntervalMs = 10000;
|
||||
private boolean useNBIOCache = false;
|
||||
private boolean nbioCacheForceUpdate = false;
|
||||
private boolean nbioCacheVerify = true;
|
||||
private String nbioCacheDir;
|
||||
private String nbioCacheMaxRetries;
|
||||
|
||||
public boolean wantsLoggedMetrics() {
|
||||
return this.wantsConsoleMetrics;
|
||||
@@ -651,6 +661,26 @@ public class NBCLIOptions {
|
||||
this.heartbeatIntervalMs =
|
||||
Long.parseLong(this.readWordOrThrow(arglist, "heartbeat interval in ms"));
|
||||
break;
|
||||
case USE_NBIO_CACHE:
|
||||
arglist.removeFirst();
|
||||
this.useNBIOCache = true;
|
||||
break;
|
||||
case NBIO_CACHE_FORCE_UPDATE:
|
||||
arglist.removeFirst();
|
||||
this.nbioCacheForceUpdate = true;
|
||||
break;
|
||||
case NBIO_CACHE_NO_VERIFY:
|
||||
arglist.removeFirst();
|
||||
this.nbioCacheVerify = false;
|
||||
break;
|
||||
case NBCLIOptions.NBIO_CACHE_DIR:
|
||||
arglist.removeFirst();
|
||||
this.nbioCacheDir = this.readWordOrThrow(arglist, "a NBIO cache directory");
|
||||
break;
|
||||
case NBIO_CACHE_MAX_RETRIES:
|
||||
arglist.removeFirst();
|
||||
this.nbioCacheMaxRetries = this.readWordOrThrow(arglist, "the maximum number of attempts to fetch a resource from the cache");
|
||||
break;
|
||||
default:
|
||||
nonincludes.addLast(arglist.removeFirst());
|
||||
}
|
||||
@@ -812,6 +842,21 @@ public class NBCLIOptions {
|
||||
public NBLogLevel getConsoleLogLevel() {
|
||||
return this.consoleLevel;
|
||||
}
|
||||
public boolean wantsToUseNBIOCache() {
|
||||
return this.useNBIOCache;
|
||||
}
|
||||
public boolean wantsNbioCacheForceUpdate() {
|
||||
return nbioCacheForceUpdate;
|
||||
}
|
||||
public boolean wantsNbioCacheVerify() {
|
||||
return nbioCacheVerify;
|
||||
}
|
||||
public String getNbioCacheDir() {
|
||||
return nbioCacheDir;
|
||||
}
|
||||
public String getNbioCacheMaxRetries() {
|
||||
return nbioCacheMaxRetries;
|
||||
}
|
||||
|
||||
private String readWordOrThrow(final LinkedList<String> arglist, final String required) {
|
||||
if (null == arglist.peekFirst())
|
||||
|
||||
@@ -43,6 +43,8 @@ public class NBIO implements NBPathsAPI.Facets {
|
||||
|
||||
private static String[] globalIncludes = new String[0];
|
||||
|
||||
private static boolean useNBIOCache;
|
||||
|
||||
public synchronized static void addGlobalIncludes(String[] globalIncludes) {
|
||||
NBIO.globalIncludes = globalIncludes;
|
||||
}
|
||||
@@ -158,12 +160,25 @@ public class NBIO implements NBPathsAPI.Facets {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public NBPathsAPI.GetPrefixes cachedContent() {
|
||||
this.resolver = URIResolvers.inNBIOCache();
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public NBPathsAPI.GetPrefixes allContent() {
|
||||
if (useNBIOCache) {
|
||||
this.resolver = URIResolvers.inFS().inCP().inNBIOCache();
|
||||
} else {
|
||||
this.resolver = URIResolvers.inFS().inCP().inURLs();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -343,6 +358,14 @@ public class NBIO implements NBPathsAPI.Facets {
|
||||
return new NBIO().remoteContent();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return content from the NBIO cache. If the content is not in the cache look for it in the given
|
||||
* URL and put it in the cache.
|
||||
*
|
||||
* @return this builder
|
||||
*/
|
||||
public static NBPathsAPI.GetPrefixes cached() { return new NBIO().cachedContent(); }
|
||||
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
@@ -628,4 +651,13 @@ public class NBIO implements NBPathsAPI.Facets {
|
||||
", extensionSets=" + extensionSets +
|
||||
'}';
|
||||
}
|
||||
|
||||
public boolean useNBIOCache() {
|
||||
return useNBIOCache;
|
||||
}
|
||||
|
||||
public static void setUseNBIOCache(boolean wantsToUseNBIOCache) {
|
||||
useNBIOCache = wantsToUseNBIOCache;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (c) 2024 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.api.nbio;
|
||||
|
||||
public enum NBIOResolverConditions {
|
||||
UPDATE_AND_VERIFY,
|
||||
UPDATE_NO_VERIFY,
|
||||
LOCAL_VERIFY,
|
||||
LOCAL_NO_VERIFY
|
||||
}
|
||||
@@ -67,6 +67,14 @@ public interface NBPathsAPI {
|
||||
*/
|
||||
GetPrefixes fileContent();
|
||||
|
||||
/**
|
||||
* Return content from the NBIO cache. If the content is not in the cache look for it in the given
|
||||
* URL and put it in the cache.
|
||||
*
|
||||
* @return this builder
|
||||
*/
|
||||
GetPrefixes cachedContent();
|
||||
|
||||
/**
|
||||
* Return content from everywhere, from remote URls, or from the file system and then the internal
|
||||
* bundled content if not found in the file system first.
|
||||
|
||||
@@ -101,11 +101,13 @@ public class ResolverForClasspath implements ContentResolver {
|
||||
public List<Path> resolveDirectory(URI uri) {
|
||||
List<Path> path = resolvePaths(uri);
|
||||
List<Path> dirs = new ArrayList<>();
|
||||
if (path != null) {
|
||||
for (Path dirpath : path) {
|
||||
if (Files.isDirectory(dirpath)) {
|
||||
dirs.add(dirpath);
|
||||
}
|
||||
}
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
/*
|
||||
* Copyright (c) 2024 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package io.nosqlbench.nb.api.nbio;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.ReadableByteChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class ResolverForNBIOCache implements ContentResolver {
|
||||
public static final ResolverForNBIOCache INSTANCE = new ResolverForNBIOCache();
|
||||
private final static Logger logger = LogManager.getLogger(ResolverForNBIOCache.class);
|
||||
private static String cacheDir = System.getProperty("user.home") + "/.nosqlbench/nbio-cache/";
|
||||
private static boolean forceUpdate = false;
|
||||
private static boolean verifyChecksum = true;
|
||||
private static int maxRetries = 3;
|
||||
|
||||
@Override
|
||||
public List<Content<?>> resolve(URI uri) {
|
||||
List<Content<?>> contents = new ArrayList<>();
|
||||
Path path = resolvePath(uri);
|
||||
|
||||
if (path != null) {
|
||||
contents.add(new PathContent(path));
|
||||
}
|
||||
return contents;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is used to resolve the path of a given URI.
|
||||
* It first checks if the URI has a scheme (http or https) and if it does, it tries to resolve the path from the cache.
|
||||
* If the file is not in the cache, it tries to download it from the remote URL.
|
||||
* If the URI does not have a scheme, it returns null.
|
||||
*
|
||||
* @param uri the URI to resolve the path for
|
||||
* @return the resolved Path object, or null if the URI does not have a scheme or the path could not be resolved
|
||||
*/
|
||||
private Path resolvePath(URI uri) {
|
||||
if (uri.getScheme() != null && !uri.getScheme().isEmpty() &&
|
||||
(uri.getScheme().equalsIgnoreCase("http") ||
|
||||
uri.getScheme().equalsIgnoreCase("https"))) {
|
||||
Path cachedFilePath = Path.of(cacheDir + uri.getPath());
|
||||
if (Files.isReadable(cachedFilePath)) {
|
||||
return pathFromLocalCache(cachedFilePath, uri);
|
||||
}
|
||||
else {
|
||||
return pathFromRemoteUrl(uri);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean downloadFile(URI uri, Path cachedFilePath, URLContent checksum) {
|
||||
int retries = 0;
|
||||
boolean success = false;
|
||||
while (retries < maxRetries) {
|
||||
try {
|
||||
if (this.remoteFileExists(uri)) {
|
||||
logger.info(() -> "Downloading remote file " + uri + " to cache at " + cachedFilePath);
|
||||
ReadableByteChannel channel = Channels.newChannel(uri.toURL().openStream());
|
||||
FileOutputStream outputStream = new FileOutputStream(cachedFilePath.toFile());
|
||||
outputStream.getChannel().transferFrom(channel, 0, Long.MAX_VALUE);
|
||||
outputStream.close();
|
||||
channel.close();
|
||||
logger.info(() -> "Downloaded remote file to cache at " + cachedFilePath);
|
||||
if(checksum == null || verifyChecksum(cachedFilePath, checksum)) {
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
logger.error(() -> "Error downloading remote file to cache at " + cachedFilePath + ", retrying...");
|
||||
retries++;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error(() -> "Error downloading remote file to cache at " + cachedFilePath + ", retrying...");
|
||||
retries++;
|
||||
}
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
private boolean verifyChecksum(Path cachedFilePath, URLContent checksum) {
|
||||
try {
|
||||
String localChecksumStr = generateSHA256Checksum(cachedFilePath.toString());
|
||||
Path checksumPath = checksumPath(cachedFilePath);
|
||||
Files.writeString(checksumPath, localChecksumStr);
|
||||
logger.debug(() -> "Generated local checksum and saved to cache at " + checksumPath);
|
||||
String remoteChecksum = stripControlCharacters(new String(checksum.getInputStream().readAllBytes()));
|
||||
if (localChecksumStr.equals(remoteChecksum)) {
|
||||
return true;
|
||||
} else {
|
||||
logger.warn(() -> "checksums do not match for " + checksumPath + " and " + checksum);
|
||||
return false;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static String stripControlCharacters(String input) {
|
||||
return input.replaceAll("[\\p{Cntrl}]+$", "");
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is used to download a file from a remote URL and store it in a local cache.
|
||||
* It first creates the cache directory if it doesn't exist.
|
||||
* Then it tries to download the file and if successful, it generates a SHA256 checksum for the downloaded file.
|
||||
* It then compares the generated checksum with the remote checksum.
|
||||
* If the checksums match, it returns the path to the cached file.
|
||||
* If the checksums don't match or if there was an error during the download, it cleans up the cache and throws a RuntimeException.
|
||||
*
|
||||
* @param uri the URI of the remote file to download
|
||||
* @return the Path to the downloaded file in the local cache
|
||||
* @throws RuntimeException if there was an error during the download or if the checksums don't match
|
||||
*/
|
||||
private Path pathFromRemoteUrl(URI uri) {
|
||||
Path cachedFilePath = Path.of(cacheDir + uri.getPath());
|
||||
createCacheDir(cachedFilePath);
|
||||
if (!verifyChecksum) {
|
||||
return execute(NBIOResolverConditions.UPDATE_NO_VERIFY, cachedFilePath, uri);
|
||||
}
|
||||
else {
|
||||
return execute(NBIOResolverConditions.UPDATE_AND_VERIFY, cachedFilePath, uri);
|
||||
}
|
||||
}
|
||||
|
||||
private void createCacheDir(Path cachedFilePath) {
|
||||
Path dir = cachedFilePath.getParent();
|
||||
if (!Files.exists(dir)) {
|
||||
try {
|
||||
Files.createDirectories(dir);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupCache(Path cachedFilePath) {
|
||||
if (!cachedFilePath.toFile().delete())
|
||||
logger.warn(() -> "Could not delete cached file " + cachedFilePath);
|
||||
Path checksumPath = checksumPath(cachedFilePath);
|
||||
if (!checksumPath.toFile().delete())
|
||||
logger.warn(() -> "Could not delete cached checksum " + checksumPath);
|
||||
}
|
||||
|
||||
private Path execute(NBIOResolverConditions condition, Path cachedFilePath, URI uri) {
|
||||
String remoteChecksumFileStr = uri.getPath() + ".sha256";
|
||||
URLContent checksum = resolveURI(URI.create(uri.toString().replace(uri.getPath(), remoteChecksumFileStr)));
|
||||
switch(condition) {
|
||||
case UPDATE_AND_VERIFY:
|
||||
if (checksum == null) {
|
||||
logger.warn(() -> "Remote checksum file " + remoteChecksumFileStr + " does not exist. Proceeding without verification");
|
||||
}
|
||||
if (downloadFile(uri, cachedFilePath, checksum)) {
|
||||
return cachedFilePath;
|
||||
} else {
|
||||
throw new RuntimeException("Error downloading remote file to cache at " + cachedFilePath);
|
||||
}
|
||||
case UPDATE_NO_VERIFY:
|
||||
logger.warn(() -> "Checksum verification is disabled, downloading remote file to cache at " + cachedFilePath);
|
||||
if (downloadFile(uri, cachedFilePath, null)) {
|
||||
return cachedFilePath;
|
||||
} else {
|
||||
throw new RuntimeException("Error downloading remote file to cache at " + cachedFilePath);
|
||||
}
|
||||
case LOCAL_VERIFY:
|
||||
if (checksum == null) {
|
||||
logger.warn(() -> "Remote checksum file does not exist, returning cached file " + cachedFilePath);
|
||||
return cachedFilePath;
|
||||
}
|
||||
try {
|
||||
String localChecksum = Files.readString(getOrCreateChecksum(cachedFilePath));
|
||||
String remoteChecksum = stripControlCharacters(new String(checksum.getInputStream().readAllBytes()));
|
||||
if (localChecksum.equals(remoteChecksum)) {
|
||||
return cachedFilePath;
|
||||
}
|
||||
else {
|
||||
logger.warn(() -> "Checksums do not match, rehydrating cache " + cachedFilePath);
|
||||
return pathFromRemoteUrl(uri);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
case LOCAL_NO_VERIFY:
|
||||
return cachedFilePath;
|
||||
default:
|
||||
throw new RuntimeException("Invalid NBIO Cache condition");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is used to retrieve a file from the local cache.
|
||||
* It first checks if the file exists in the cache and if a checksum file is present.
|
||||
* If the checksum file is not present, it generates a new one.
|
||||
* If the "force update" option is enabled, it deletes the cached file and downloads it from the remote URL.
|
||||
* If the "checksum verification" option is enabled, it compares the local checksum with the remote checksum.
|
||||
* If the checksums match, it returns the path to the cached file.
|
||||
* If the checksums don't match, it deletes the cached file and downloads it from the remote URL.
|
||||
* If the remote file or checksum does not exist, it returns the cached file.
|
||||
*
|
||||
* @param cachedFilePath the Path to the cached file
|
||||
* @param uri the URI of the remote file
|
||||
* @return the Path to the cached file
|
||||
* @throws RuntimeException if there was an error during the checksum comparison or if the checksums don't match
|
||||
*/
|
||||
private Path pathFromLocalCache(Path cachedFilePath, URI uri) {
|
||||
|
||||
if (forceUpdate) {
|
||||
return pathFromRemoteUrl(uri);
|
||||
}
|
||||
if (!verifyChecksum) {
|
||||
logger.warn(() -> "Checksum verification is disabled, returning cached file " + cachedFilePath);
|
||||
return execute(NBIOResolverConditions.LOCAL_NO_VERIFY, cachedFilePath, uri);
|
||||
} else {
|
||||
return execute(NBIOResolverConditions.LOCAL_VERIFY, cachedFilePath, uri);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private Path getOrCreateChecksum(Path cachedFilePath) {
|
||||
Path checksumPath = checksumPath(cachedFilePath);
|
||||
if (!Files.isReadable(checksumPath)) {
|
||||
try {
|
||||
Files.writeString(checksumPath, generateSHA256Checksum(cachedFilePath.toString()));
|
||||
} catch (IOException | NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
return checksumPath;
|
||||
}
|
||||
|
||||
private Path checksumPath(Path cachedFilePath) {
|
||||
return Path.of(cachedFilePath + ".sha256");
|
||||
}
|
||||
|
||||
private static String generateSHA256Checksum(String filePath) throws IOException, NoSuchAlgorithmException {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-256");
|
||||
try (InputStream is = new FileInputStream(filePath)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int bytesRead;
|
||||
while ((bytesRead = is.read(buffer)) != -1) {
|
||||
md.update(buffer, 0, bytesRead);
|
||||
}
|
||||
}
|
||||
byte[] digest = md.digest();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private URLContent resolveURI(URI uri) {
|
||||
try {
|
||||
URL url = uri.toURL();
|
||||
InputStream inputStream = url.openStream();
|
||||
logger.debug(() -> "Found accessible remote file at " + url);
|
||||
return new URLContent(url, inputStream);
|
||||
} catch (IOException e) {
|
||||
logger.error(() -> "Unable to find content at URI '" + uri + "', this often indicates a configuration error.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean remoteFileExists(URI uri) {
|
||||
try {
|
||||
HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
|
||||
connection.setRequestMethod("HEAD");
|
||||
int responseCode = connection.getResponseCode();
|
||||
return responseCode == HttpURLConnection.HTTP_OK;
|
||||
} catch (Exception e) {
|
||||
return false; // Error occurred or file does not exist
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Path> resolveDirectory(URI uri) {
|
||||
List<Path> dirs = new ArrayList<>();
|
||||
|
||||
Path path = Path.of(cacheDir + uri.getPath());
|
||||
if (Files.isDirectory(path)) {
|
||||
dirs.add(path);
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
public static void setCacheDir(String cacheDir) {
|
||||
ResolverForNBIOCache.cacheDir = cacheDir;
|
||||
}
|
||||
|
||||
public static void setForceUpdate(boolean forceUpdate) {
|
||||
ResolverForNBIOCache.forceUpdate = forceUpdate;
|
||||
}
|
||||
|
||||
public static void setVerifyChecksum(boolean verifyChecksum) {
|
||||
ResolverForNBIOCache.verifyChecksum = verifyChecksum;
|
||||
}
|
||||
|
||||
public static void setMaxRetries(int maxRetries) {
|
||||
ResolverForNBIOCache.maxRetries = maxRetries;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -36,7 +36,8 @@ public class URIResolver implements ContentResolver {
|
||||
private static final List<ContentResolver> EVERYWHERE = List.of(
|
||||
ResolverForURL.INSTANCE,
|
||||
ResolverForFilesystem.INSTANCE,
|
||||
ResolverForClasspath.INSTANCE
|
||||
ResolverForClasspath.INSTANCE,
|
||||
ResolverForNBIOCache.INSTANCE
|
||||
);
|
||||
|
||||
private List<String> extensions;
|
||||
@@ -87,6 +88,16 @@ public class URIResolver implements ContentResolver {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Include resources within the NBIO cache or download them if they are not found.
|
||||
*
|
||||
* @return this URISearch
|
||||
*/
|
||||
public URIResolver inNBIOCache() {
|
||||
loaders.add(ResolverForNBIOCache.INSTANCE);
|
||||
return this;
|
||||
}
|
||||
|
||||
public List<Content<?>> resolve(String uri) {
|
||||
return resolve(URI.create(uri));
|
||||
}
|
||||
|
||||
@@ -52,4 +52,8 @@ public class URIResolvers {
|
||||
public static URIResolver inClasspath() {
|
||||
return new URIResolver().inCP();
|
||||
}
|
||||
|
||||
public static URIResolver inNBIOCache() {
|
||||
return new URIResolver().inNBIOCache();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,12 +47,6 @@
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apfloat</groupId>
|
||||
<artifactId>apfloat</artifactId>
|
||||
<version>1.13.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.matheclipse</groupId>
|
||||
<artifactId>matheclipse-core</artifactId>
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2024 nosqlbench
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.nosqlbench.virtdata.lib.vectors.dnn;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.matheclipse.core.eval.ExprEvaluator;
|
||||
import org.matheclipse.core.expression.F;
|
||||
import org.matheclipse.core.interfaces.IExpr;
|
||||
import org.matheclipse.core.interfaces.ISymbol;
|
||||
import org.matheclipse.core.interfaces.IAST;
|
||||
|
||||
public class DNN_Symbolic_Tests {
|
||||
|
||||
@Test
|
||||
public void testExactRepresentation() {
|
||||
ExprEvaluator util = new ExprEvaluator(false, (short)10);
|
||||
|
||||
// Convert an expression to the internal Java form:
|
||||
// Note: single character identifiers are case sensitive
|
||||
// (the "D()" function identifier must be written as upper case
|
||||
// character)
|
||||
String javaForm = util.toJavaForm("D(sin(x)*cos(x),x)");
|
||||
// prints: D(Times(Sin(x),Cos(x)),x)
|
||||
System.out.println("Out[1]: " + javaForm.toString());
|
||||
|
||||
// Use the Java form to create an expression with F.* static
|
||||
// methods:
|
||||
ISymbol x = F.Dummy("x");
|
||||
IAST function = F.D(F.Times(F.Sin(x), F.Cos(x)), x);
|
||||
IExpr result = util.eval(function);
|
||||
// print: Cos(x)^2-Sin(x)^2
|
||||
System.out.println("Out[2]: " + result.toString());
|
||||
|
||||
// Note "diff" is an alias for the "D" function
|
||||
result = util.eval("diff(sin(x)*cos(x),x)");
|
||||
// print: Cos(x)^2-Sin(x)^2
|
||||
System.out.println("Out[3]: " + result.toString());
|
||||
|
||||
// evaluate the last result (% contains "last answer")
|
||||
result = util.eval("%+cos(x)^2");
|
||||
// print: 2*Cos(x)^2-Sin(x)^2
|
||||
System.out.println("Out[4]: " + result.toString());
|
||||
|
||||
// evaluate an Integrate[] expression
|
||||
result = util.eval("integrate(sin(x)^5,x)");
|
||||
// print: 2/3*Cos(x)^3-1/5*Cos(x)^5-Cos(x)
|
||||
System.out.println("Out[5]: " + result.toString());
|
||||
|
||||
// set the value of a variable "a" to 10
|
||||
result = util.eval("a=10");
|
||||
// print: 10
|
||||
System.out.println("Out[6]: " + result.toString());
|
||||
|
||||
// do a calculation with variable "a"
|
||||
result = util.eval("a*3+b");
|
||||
// print: 30+b
|
||||
System.out.println("Out[7]: " + result.toString());
|
||||
|
||||
// Do a calculation in "numeric mode" with the N() function
|
||||
// Note: single character identifiers are case sensistive
|
||||
// (the "N()" function identifier must be written as upper case
|
||||
// character)
|
||||
result = util.eval("N(sinh(5))");
|
||||
// print: 74.20321057778875
|
||||
System.out.println("Out[8]: " + result.toString());
|
||||
|
||||
// define a function with a recursive factorial function definition.
|
||||
// Note: fac(0) is the stop condition.
|
||||
result = util.eval("fac(x_Integer):=x*fac(x-1);fac(0)=1");
|
||||
// now calculate factorial of 10:
|
||||
result = util.eval("fac(10)");
|
||||
// print: 3628800
|
||||
System.out.println("Out[9]: " + result.toString());
|
||||
|
||||
function = F.Function(F.Divide(F.Gamma(F.Plus(F.C1, F.Slot1)), F.Gamma(F.Plus(F.C1, F.Slot2))));
|
||||
// eval function ( Gamma(1+#1)/Gamma(1+#2) ) & [23,20]
|
||||
result = util.evalFunction(function, "23", "20");
|
||||
// print: 10626
|
||||
System.out.println("Out[10]: " + result.toString());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user