From ff769c7bed66070d93d2cafa0fe50748c6107e97 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Mon, 11 Oct 2021 17:22:59 -0500 Subject: [PATCH] Add S3 URL Handler --- .../s3urlhandler}/S3ClientCache.java | 4 +- .../s3urlhandler}/S3UrlConnection.java | 2 +- .../s3urlhandler}/S3UrlFields.java | 16 ++++++- .../s3urlhandler}/S3UrlStreamHandler.java | 2 +- .../S3UrlStreamHandlerProvider.java | 2 +- .../s3urlhandler}/S3UrlStreamHandlerTest.java | 9 ++-- .../nb/addins/s3/s3utils/S3UploaderDemo.java | 43 +++++++++++++++++++ .../nb/addins/s3/s3utils/S3UploaderTest.java | 21 +++++++++ 8 files changed, 89 insertions(+), 10 deletions(-) rename nb-api/src/main/java/io/nosqlbench/nb/addins/{s3urls => s3/s3urlhandler}/S3ClientCache.java (91%) rename nb-api/src/main/java/io/nosqlbench/nb/addins/{s3urls => s3/s3urlhandler}/S3UrlConnection.java (94%) rename nb-api/src/main/java/io/nosqlbench/nb/addins/{s3urls => s3/s3urlhandler}/S3UrlFields.java (86%) rename nb-api/src/main/java/io/nosqlbench/nb/addins/{s3urls => s3/s3urlhandler}/S3UrlStreamHandler.java (91%) rename nb-api/src/main/java/io/nosqlbench/nb/addins/{s3urls => s3/s3urlhandler}/S3UrlStreamHandlerProvider.java (92%) rename nb-api/src/test/java/io/nosqlbench/nb/addins/{s3urls => s3/s3urlhandler}/S3UrlStreamHandlerTest.java (88%) create mode 100644 nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderDemo.java create mode 100644 nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderTest.java diff --git a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3ClientCache.java b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3ClientCache.java similarity index 91% rename from nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3ClientCache.java rename to nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3ClientCache.java index 8908c76c1..a49d49665 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3ClientCache.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3ClientCache.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.addins.s3urls; +package io.nosqlbench.nb.addins.s3.s3urlhandler; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -22,7 +22,7 @@ public class S3ClientCache { } public AmazonS3 get(S3UrlFields fields) { - AmazonS3 s3 = cache.computeIfAbsent(fields.credentialsFingerprint(), + AmazonS3 s3 = cache.computeIfAbsent(fields.getCredentialsFingerprint(), cfp -> createAuthorizedClient(fields)); return s3; } diff --git a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlConnection.java b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlConnection.java similarity index 94% rename from nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlConnection.java rename to nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlConnection.java index 093661326..e9175e973 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlConnection.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlConnection.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.addins.s3urls; +package io.nosqlbench.nb.addins.s3.s3urlhandler; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.S3Object; diff --git a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlFields.java b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlFields.java similarity index 86% rename from nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlFields.java rename to nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlFields.java index e77331f4f..1b56922ad 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlFields.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlFields.java @@ -1,5 +1,6 @@ -package io.nosqlbench.nb.addins.s3urls; +package io.nosqlbench.nb.addins.s3.s3urlhandler; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -13,6 +14,15 @@ public class S3UrlFields { public final String accessKey; private final String endpoint; + public static S3UrlFields fromURLString(String urlString) { + URL url = null; + try { + url = new URL(urlString); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + return new S3UrlFields(url); + } public S3UrlFields(URL url) { String accessKey = null; @@ -52,6 +62,10 @@ public class S3UrlFields { return new CredentialsFingerprint(this); } + public CredentialsFingerprint getCredentialsFingerprint() { + return null; + } + public static class CredentialsFingerprint { private final S3UrlFields fields; diff --git a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandler.java b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandler.java similarity index 91% rename from nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandler.java rename to nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandler.java index cc0c65d67..f77ff0aaa 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandler.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandler.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.addins.s3urls; +package io.nosqlbench.nb.addins.s3.s3urlhandler; import java.io.IOException; import java.net.URL; diff --git a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandlerProvider.java b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerProvider.java similarity index 92% rename from nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandlerProvider.java rename to nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerProvider.java index 99f7399a9..389903c40 100644 --- a/nb-api/src/main/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandlerProvider.java +++ b/nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerProvider.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.addins.s3urls; +package io.nosqlbench.nb.addins.s3.s3urlhandler; import io.nosqlbench.nb.annotations.Service; diff --git a/nb-api/src/test/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandlerTest.java b/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerTest.java similarity index 88% rename from nb-api/src/test/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandlerTest.java rename to nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerTest.java index be9f5fcd8..0139431ab 100644 --- a/nb-api/src/test/java/io/nosqlbench/nb/addins/s3urls/S3UrlStreamHandlerTest.java +++ b/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerTest.java @@ -1,4 +1,4 @@ -package io.nosqlbench.nb.addins.s3urls; +package io.nosqlbench.nb.addins.s3.s3urlhandler; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -16,6 +16,10 @@ import static org.assertj.core.api.Assertions.assertThat; public class S3UrlStreamHandlerTest { + public static String bucketName = "nb-extension-test"; + public static String keyName = "key-name"; + public static String testValue = "test-value"; + /** * This test requires that you have credentials already configured on your local system * for S3. It creates an object using the s3 client directly, then uses a generic @@ -26,9 +30,6 @@ public class S3UrlStreamHandlerTest { public void sanityCheckS3UrlHandler() { AmazonS3 client = AmazonS3ClientBuilder.defaultClient(); - String bucketName = "nb-extension-test"; - String keyName = "key-name"; - String testValue = "test-value"; Bucket bucket = null; diff --git a/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderDemo.java b/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderDemo.java new file mode 100644 index 000000000..040002af7 --- /dev/null +++ b/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderDemo.java @@ -0,0 +1,43 @@ +package io.nosqlbench.nb.addins.s3.s3utils; + +import com.amazonaws.services.s3.transfer.MultipleFileUpload; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import io.nosqlbench.nb.addins.s3.s3urlhandler.S3ClientCache; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; + +/** + * This is a generic s3 directory uploader which is neither a scripting plugin nor a standard URL handler. + */ +public class S3UploaderDemo { + + private final S3ClientCache clientCache = new S3ClientCache(); + + private static final Logger logger = LogManager.getLogger(S3UploaderDemo.class); + + public MultipleFileUpload syncup(Path sourcePath, String bucket, String prefix) { + + if (!FileSystems.getDefault().equals(sourcePath.getFileSystem())) { + throw new RuntimeException("The file must reside on the default filesystem to be uploaded by S3."); + } + if (!Files.isDirectory(sourcePath, LinkOption.NOFOLLOW_LINKS)) { + throw new RuntimeException("path '" + sourcePath + "' is not a directory."); + } + TransferManager tm = TransferManagerBuilder.defaultTransferManager(); + MultipleFileUpload mfu = tm.uploadDirectory(bucket, prefix, sourcePath.toFile(), true); + try { + mfu.waitForCompletion(); + } catch (InterruptedException e) { + throw new RuntimeException("Multi-file upload was interrupted!"); + } + tm.shutdownNow(); + return mfu; + } + +} diff --git a/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderTest.java b/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderTest.java new file mode 100644 index 000000000..218587dcb --- /dev/null +++ b/nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderTest.java @@ -0,0 +1,21 @@ +package io.nosqlbench.nb.addins.s3.s3utils; + +import com.amazonaws.services.s3.transfer.MultipleFileUpload; +import io.nosqlbench.nb.addins.s3.s3urlhandler.S3UrlStreamHandlerTest; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; + +public class S3UploaderTest { + + @Disabled + @Test + public void testDirUpload() { + Path path = Path.of("src/test/resources/nesteddir1"); + S3UploaderDemo ul = new S3UploaderDemo(); + MultipleFileUpload mfu = ul.syncup(path, S3UrlStreamHandlerTest.bucketName, "test-prefix"); + System.out.println(mfu); + } + +}