IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Updated code to simply append .sha256 to complete filename or url

Browse Source
This commit is contained in:
Mark Wolters
2024-04-11 14:51:08 -04:00
committed by Jonathan Shook
parent 5021a00fb4
commit aa6326ffd8
2 changed files with 49 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@@ -214,7 +214,7 @@ public class NBCLIOptions {
private boolean useNBIOCache = false; private boolean useNBIOCache = false;
private boolean nbioCacheForceUpdate = false; private boolean nbioCacheForceUpdate = false;
private boolean nbioCacheVerify = true; private boolean nbioCacheVerify = true;
private String nbioCachDir; private String nbioCacheDir;
private String nbioCacheMaxRetries; private String nbioCacheMaxRetries;
public boolean wantsLoggedMetrics() { public boolean wantsLoggedMetrics() {
@@ -675,7 +675,7 @@ public class NBCLIOptions {
break; break;
case NBCLIOptions.NBIO_CACHE_DIR: case NBCLIOptions.NBIO_CACHE_DIR:
arglist.removeFirst(); arglist.removeFirst();
this.nbioCachDir = this.readWordOrThrow(arglist, "a NBIO cache directory"); this.nbioCacheDir = this.readWordOrThrow(arglist, "a NBIO cache directory");
break; break;
case NBIO_CACHE_MAX_RETRIES: case NBIO_CACHE_MAX_RETRIES:
arglist.removeFirst(); arglist.removeFirst();
@@ -852,7 +852,7 @@ public class NBCLIOptions {
return nbioCacheVerify; return nbioCacheVerify;
} }
public String getNbioCacheDir() { public String getNbioCacheDir() {
return nbioCachDir; return nbioCacheDir;
} }
public String getNbioCacheMaxRetries() { public String getNbioCacheMaxRetries() {
return nbioCacheMaxRetries; return nbioCacheMaxRetries;

92
nb-api/src/main/java/io/nosqlbench/nb/api/nbio/ResolverForNBIOCache.java
View File

@@ -65,9 +65,9 @@ public class ResolverForNBIOCache implements ContentResolver {
if (uri.getScheme() != null && !uri.getScheme().isEmpty() && if (uri.getScheme() != null && !uri.getScheme().isEmpty() &&
(uri.getScheme().equalsIgnoreCase("http") || (uri.getScheme().equalsIgnoreCase("http") ||
uri.getScheme().equalsIgnoreCase("https"))) { uri.getScheme().equalsIgnoreCase("https"))) {
Path cachePath = Path.of(cacheDir + uri.getPath()); Path cachedFilePath = Path.of(cacheDir + uri.getPath());
if (Files.isReadable(cachePath)) { if (Files.isReadable(cachedFilePath)) {
return pathFromLocalCache(cachePath, uri); return pathFromLocalCache(cachedFilePath, uri);
} }
else { else {
return pathFromRemoteUrl(uri); return pathFromRemoteUrl(uri);
@@ -76,39 +76,39 @@ public class ResolverForNBIOCache implements ContentResolver {
return null; return null;
} }
private boolean downloadFile(URI uri, Path cachePath, URLContent checksum) { private boolean downloadFile(URI uri, Path cachedFilePath, URLContent checksum) {
int retries = 0; int retries = 0;
boolean success = false; boolean success = false;
while (retries < maxRetries) { while (retries < maxRetries) {
try { try {
if (this.remoteFileExists(uri)) { if (this.remoteFileExists(uri)) {
logger.info(() -> "Downloading remote file " + uri + " to cache at " + cachePath); logger.info(() -> "Downloading remote file " + uri + " to cache at " + cachedFilePath);
ReadableByteChannel channel = Channels.newChannel(uri.toURL().openStream()); ReadableByteChannel channel = Channels.newChannel(uri.toURL().openStream());
FileOutputStream outputStream = new FileOutputStream(cachePath.toFile()); FileOutputStream outputStream = new FileOutputStream(cachedFilePath.toFile());
outputStream.getChannel().transferFrom(channel, 0, Long.MAX_VALUE); outputStream.getChannel().transferFrom(channel, 0, Long.MAX_VALUE);
outputStream.close(); outputStream.close();
channel.close(); channel.close();
logger.info(() -> "Downloaded remote file to cache at " + cachePath); logger.info(() -> "Downloaded remote file to cache at " + cachedFilePath);
if(checksum == null || verifyChecksum(cachePath, checksum)) { if(checksum == null || verifyChecksum(cachedFilePath, checksum)) {
success = true; success = true;
break; break;
} }
} else { } else {
logger.error(() -> "Error downloading remote file to cache at " + cachePath + ", retrying..."); logger.error(() -> "Error downloading remote file to cache at " + cachedFilePath + ", retrying...");
retries++; retries++;
} }
} catch (IOException e) { } catch (IOException e) {
logger.error(() -> "Error downloading remote file to cache at " + cachePath + ", retrying..."); logger.error(() -> "Error downloading remote file to cache at " + cachedFilePath + ", retrying...");
retries++; retries++;
} }
} }
return success; return success;
} }
private boolean verifyChecksum(Path cachePath, URLContent checksum) { private boolean verifyChecksum(Path cachedFilePath, URLContent checksum) {
try { try {
String localChecksumStr = generateSHA256Checksum(cachePath.toString()); String localChecksumStr = generateSHA256Checksum(cachedFilePath.toString());
Path checksumPath = Path.of(cachePath.toString().substring(0, cachePath.toString().lastIndexOf('.')) + ".sha256"); Path checksumPath = Path.of(cachedFilePath + ".sha256");
Files.writeString(checksumPath, localChecksumStr); Files.writeString(checksumPath, localChecksumStr);
logger.debug(() -> "Generated local checksum and saved to cache at " + checksumPath); logger.debug(() -> "Generated local checksum and saved to cache at " + checksumPath);
String remoteChecksum = new String(checksum.getInputStream().readAllBytes()); String remoteChecksum = new String(checksum.getInputStream().readAllBytes());
@@ -136,18 +136,18 @@ public class ResolverForNBIOCache implements ContentResolver {
* @throws RuntimeException if there was an error during the download or if the checksums don't match * @throws RuntimeException if there was an error during the download or if the checksums don't match
*/ */
private Path pathFromRemoteUrl(URI uri) { private Path pathFromRemoteUrl(URI uri) {
Path cachePath = Path.of(cacheDir + uri.getPath()); Path cachedFilePath = Path.of(cacheDir + uri.getPath());
createCacheDir(cachePath); createCacheDir(cachedFilePath);
if (!verifyChecksum) { if (!verifyChecksum) {
return execute(NBIOResolverConditions.UPDATE_NO_VERIFY, cachePath, uri); return execute(NBIOResolverConditions.UPDATE_NO_VERIFY, cachedFilePath, uri);
} }
else { else {
return execute(NBIOResolverConditions.UPDATE_AND_VERIFY, cachePath, uri); return execute(NBIOResolverConditions.UPDATE_AND_VERIFY, cachedFilePath, uri);
} }
} }
private void createCacheDir(Path cachePath) { private void createCacheDir(Path cachedFilePath) {
Path dir = cachePath.getParent(); Path dir = cachedFilePath.getParent();
if (!Files.exists(dir)) { if (!Files.exists(dir)) {
try { try {
Files.createDirectories(dir); Files.createDirectories(dir);
@@ -157,54 +157,54 @@ public class ResolverForNBIOCache implements ContentResolver {
} }
} }
private void cleanupCache(Path cachePath) { private void cleanupCache(Path cachedFilePath) {
if (!cachePath.toFile().delete()) if (!cachedFilePath.toFile().delete())
logger.warn(() -> "Could not delete cached file " + cachePath); logger.warn(() -> "Could not delete cached file " + cachedFilePath);
Path checksumPath = Path.of(cachePath.toString().substring(0, cachePath.toString().lastIndexOf('.')) + ".sha256"); Path checksumPath = Path.of(cachedFilePath + ".sha256");
if (!checksumPath.toFile().delete()) if (!checksumPath.toFile().delete())
logger.warn(() -> "Could not delete cached checksum " + checksumPath); logger.warn(() -> "Could not delete cached checksum " + checksumPath);
} }
private Path execute(NBIOResolverConditions condition, Path cachePath, URI uri) { private Path execute(NBIOResolverConditions condition, Path cachedFilePath, URI uri) {
String remoteChecksumFileStr = uri.getPath().substring(0, uri.getPath().indexOf('.')) + ".sha256"; String remoteChecksumFileStr = uri.getPath() + ".sha256";
URLContent checksum = resolveURI(URI.create(uri.toString().replace(uri.getPath(), remoteChecksumFileStr))); URLContent checksum = resolveURI(URI.create(uri.toString().replace(uri.getPath(), remoteChecksumFileStr)));
switch(condition) { switch(condition) {
case UPDATE_AND_VERIFY: case UPDATE_AND_VERIFY:
if (checksum == null) { if (checksum == null) {
logger.warn(() -> "Remote checksum file " + remoteChecksumFileStr + " does not exist. Proceeding without verification"); logger.warn(() -> "Remote checksum file " + remoteChecksumFileStr + " does not exist. Proceeding without verification");
} }
if (downloadFile(uri, cachePath, checksum)) { if (downloadFile(uri, cachedFilePath, checksum)) {
return cachePath; return cachedFilePath;
} else { } else {
throw new RuntimeException("Error downloading remote file to cache at " + cachePath); throw new RuntimeException("Error downloading remote file to cache at " + cachedFilePath);
} }
case UPDATE_NO_VERIFY: case UPDATE_NO_VERIFY:
logger.warn(() -> "Checksum verification is disabled, downloading remote file to cache at " + cachePath); logger.warn(() -> "Checksum verification is disabled, downloading remote file to cache at " + cachedFilePath);
if (downloadFile(uri, cachePath, null)) { if (downloadFile(uri, cachedFilePath, null)) {
return cachePath; return cachedFilePath;
} else { } else {
throw new RuntimeException("Error downloading remote file to cache at " + cachePath); throw new RuntimeException("Error downloading remote file to cache at " + cachedFilePath);
} }
case LOCAL_VERIFY: case LOCAL_VERIFY:
if (checksum == null) { if (checksum == null) {
logger.warn(() -> "Remote checksum file does not exist, returning cached file " + cachePath); logger.warn(() -> "Remote checksum file does not exist, returning cached file " + cachedFilePath);
return cachePath; return cachedFilePath;
} }
try { try {
String localChecksum = Files.readString(getOrCreateChecksum(cachePath)); String localChecksum = Files.readString(getOrCreateChecksum(cachedFilePath));
String remoteChecksum = new String(checksum.getInputStream().readAllBytes()); String remoteChecksum = new String(checksum.getInputStream().readAllBytes());
if (localChecksum.equals(remoteChecksum)) { if (localChecksum.equals(remoteChecksum)) {
return cachePath; return cachedFilePath;
} }
else { else {
logger.warn(() -> "Checksums do not match, rehydrating cache " + cachePath); logger.warn(() -> "Checksums do not match, rehydrating cache " + cachedFilePath);
return pathFromRemoteUrl(uri); return pathFromRemoteUrl(uri);
} }
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
case LOCAL_NO_VERIFY: case LOCAL_NO_VERIFY:
return cachePath; return cachedFilePath;
default: default:
throw new RuntimeException("Invalid NBIO Cache condition"); throw new RuntimeException("Invalid NBIO Cache condition");
} }
@@ -220,30 +220,30 @@ public class ResolverForNBIOCache implements ContentResolver {
* If the checksums don't match, it deletes the cached file and downloads it from the remote URL. * If the checksums don't match, it deletes the cached file and downloads it from the remote URL.
* If the remote file or checksum does not exist, it returns the cached file. * If the remote file or checksum does not exist, it returns the cached file.
* *
* @param cachePath the Path to the cached file * @param cachedFilePath the Path to the cached file
* @param uri the URI of the remote file * @param uri the URI of the remote file
* @return the Path to the cached file * @return the Path to the cached file
* @throws RuntimeException if there was an error during the checksum comparison or if the checksums don't match * @throws RuntimeException if there was an error during the checksum comparison or if the checksums don't match
*/ */
private Path pathFromLocalCache(Path cachePath, URI uri) { private Path pathFromLocalCache(Path cachedFilePath, URI uri) {
if (forceUpdate) { if (forceUpdate) {
return pathFromRemoteUrl(uri); return pathFromRemoteUrl(uri);
} }
if (!verifyChecksum) { if (!verifyChecksum) {
logger.warn(() -> "Checksum verification is disabled, returning cached file " + cachePath); logger.warn(() -> "Checksum verification is disabled, returning cached file " + cachedFilePath);
return execute(NBIOResolverConditions.LOCAL_NO_VERIFY, cachePath, uri); return execute(NBIOResolverConditions.LOCAL_NO_VERIFY, cachedFilePath, uri);
} else { } else {
return execute(NBIOResolverConditions.LOCAL_VERIFY, cachePath, uri); return execute(NBIOResolverConditions.LOCAL_VERIFY, cachedFilePath, uri);
} }
} }
private Path getOrCreateChecksum(Path cachePath) { private Path getOrCreateChecksum(Path cachedFilePath) {
Path checksumPath = Path.of(cachePath.toString().substring(0, cachePath.toString().lastIndexOf('.')) + ".sha256"); Path checksumPath = Path.of(cachedFilePath + ".sha256");
if (!Files.isReadable(checksumPath)) { if (!Files.isReadable(checksumPath)) {
try { try {
Files.writeString(checksumPath, generateSHA256Checksum(cachePath.toString())); Files.writeString(checksumPath, generateSHA256Checksum(cachedFilePath.toString()));
} catch (IOException | NoSuchAlgorithmException e) { } catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }