Add S3 URL Handler

nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3ClientCache.java

@@ -1,4 +1,4 @@
-package io.nosqlbench.nb.addins.s3urls;
+package io.nosqlbench.nb.addins.s3.s3urlhandler;
 
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
@@ -22,7 +22,7 @@ public class S3ClientCache {
     }
 
     public AmazonS3 get(S3UrlFields fields) {
-        AmazonS3 s3 = cache.computeIfAbsent(fields.credentialsFingerprint(),
+        AmazonS3 s3 = cache.computeIfAbsent(fields.getCredentialsFingerprint(),
             cfp -> createAuthorizedClient(fields));
         return s3;
     }

nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlConnection.java

@@ -1,4 +1,4 @@
-package io.nosqlbench.nb.addins.s3urls;
+package io.nosqlbench.nb.addins.s3.s3urlhandler;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.S3Object;

nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlFields.java

@@ -1,5 +1,6 @@
-package io.nosqlbench.nb.addins.s3urls;
+package io.nosqlbench.nb.addins.s3.s3urlhandler;
 
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
@@ -13,6 +14,15 @@ public class S3UrlFields {
     public final String accessKey;
     private final String endpoint;
 
+    public static S3UrlFields fromURLString(String urlString) {
+        URL url = null;
+        try {
+            url = new URL(urlString);
+        } catch (MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
+        return new S3UrlFields(url);
+    }
     public S3UrlFields(URL url) {
 
         String accessKey = null;
@@ -52,6 +62,10 @@ public class S3UrlFields {
         return new CredentialsFingerprint(this);
     }
 
+    public CredentialsFingerprint getCredentialsFingerprint() {
+        return null;
+    }
+
     public static class CredentialsFingerprint {
         private final S3UrlFields fields;
 

nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandler.java

@@ -1,4 +1,4 @@
-package io.nosqlbench.nb.addins.s3urls;
+package io.nosqlbench.nb.addins.s3.s3urlhandler;
 
 import java.io.IOException;
 import java.net.URL;

nb-api/src/main/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerProvider.java

@@ -1,4 +1,4 @@
-package io.nosqlbench.nb.addins.s3urls;
+package io.nosqlbench.nb.addins.s3.s3urlhandler;
 
 import io.nosqlbench.nb.annotations.Service;
 

nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3urlhandler/S3UrlStreamHandlerTest.java

@@ -1,4 +1,4 @@
-package io.nosqlbench.nb.addins.s3urls;
+package io.nosqlbench.nb.addins.s3.s3urlhandler;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
@@ -16,6 +16,10 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 public class S3UrlStreamHandlerTest {
 
+    public static String bucketName = "nb-extension-test";
+    public static String keyName = "key-name";
+    public static String testValue = "test-value";
+
     /**
      * This test requires that you have credentials already configured on your local system
      * for S3. It creates an object using the s3 client directly, then uses a generic
@@ -26,9 +30,6 @@ public class S3UrlStreamHandlerTest {
     public void sanityCheckS3UrlHandler() {
         AmazonS3 client = AmazonS3ClientBuilder.defaultClient();
 
-        String bucketName = "nb-extension-test";
-        String keyName = "key-name";
-        String testValue = "test-value";
 
         Bucket bucket = null;
 

nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderDemo.java

@@ -0,0 +1,43 @@
+package io.nosqlbench.nb.addins.s3.s3utils;
+
+import com.amazonaws.services.s3.transfer.MultipleFileUpload;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import io.nosqlbench.nb.addins.s3.s3urlhandler.S3ClientCache;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+
+/**
+ * This is a generic s3 directory uploader which is neither a scripting plugin nor a standard URL handler.
+ */
+public class S3UploaderDemo {
+
+    private final S3ClientCache clientCache = new S3ClientCache();
+
+    private static final Logger logger = LogManager.getLogger(S3UploaderDemo.class);
+
+    public MultipleFileUpload syncup(Path sourcePath, String bucket, String prefix) {
+
+        if (!FileSystems.getDefault().equals(sourcePath.getFileSystem())) {
+            throw new RuntimeException("The file must reside on the default filesystem to be uploaded by S3.");
+        }
+        if (!Files.isDirectory(sourcePath, LinkOption.NOFOLLOW_LINKS)) {
+            throw new RuntimeException("path '" + sourcePath + "' is not a directory.");
+        }
+        TransferManager tm = TransferManagerBuilder.defaultTransferManager();
+        MultipleFileUpload mfu = tm.uploadDirectory(bucket, prefix, sourcePath.toFile(), true);
+        try {
+            mfu.waitForCompletion();
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Multi-file upload was interrupted!");
+        }
+        tm.shutdownNow();
+        return mfu;
+    }
+
+}

nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderTest.java

@@ -0,0 +1,21 @@
+package io.nosqlbench.nb.addins.s3.s3utils;
+
+import com.amazonaws.services.s3.transfer.MultipleFileUpload;
+import io.nosqlbench.nb.addins.s3.s3urlhandler.S3UrlStreamHandlerTest;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Path;
+
+public class S3UploaderTest {
+
+    @Disabled
+    @Test
+    public void testDirUpload() {
+        Path path = Path.of("src/test/resources/nesteddir1");
+        S3UploaderDemo ul = new S3UploaderDemo();
+        MultipleFileUpload mfu = ul.syncup(path, S3UrlStreamHandlerTest.bucketName, "test-prefix");
+        System.out.println(mfu);
+    }
+
+}