IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

add check for download too large for filesystem

Browse Source
This commit is contained in:
Mark Wolters 2024-06-26 17:16:48 -04:00
parent 8ce9cba559
commit e29f3d8719
2 changed files with 43 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/nbio/NBIOCacheException.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package io.nosqlbench.nb.api.nbio;
public class NBIOCacheException extends Throwable {
public NBIOCacheException(String s) {
super(s);
}
}

19
nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/nbio/ResolverForNBIOCache.java
View File

@ -110,6 +110,7 @@ public class ResolverForNBIOCache implements ContentResolver {
while (retries < maxRetries) {
try {
if (this.remoteFileExists(uri)) {
checkLocalDiskSpace(uri);
Timer timer = new Timer();
try {
logger.info(() -> "Downloading remote file " + uri + " to cache at " + cachedFilePath);
@ -146,11 +147,29 @@ public class ResolverForNBIOCache implements ContentResolver {
logger.error(() -> "Error downloading remote file to cache at " + cachedFilePath + ":" + e + ", " +
"retrying...");
retries++;
} catch (NBIOCacheException e) {
logger.error(() -> "Error downloading remote file to cache at " + cachedFilePath + ":" + e.getMessage());
throw new RuntimeException(e);
}
}
return success;
}
private void checkLocalDiskSpace(URI uri) throws NBIOCacheException {
try {
HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
connection.setRequestMethod("HEAD");
int length = connection.getContentLength();
long freeSpace = Files.getFileStore(Path.of(cacheDir)).getUsableSpace();
if (length > (freeSpace * 0.9)) {
throw new NBIOCacheException("Not enough space to download file " + uri + " of size " + length +
" to cache at " + cacheDir + " with only " + freeSpace + " bytes free");
}
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
}
private boolean verifyChecksum(Path cachedFilePath, URLContent checksum) {
try {
String localChecksumStr = generateSHA256Checksum(cachedFilePath.toString());