improve NBEnvironment and S3 interaction

engine-extensions/src/main/java/io/nosqlbench/engine/extensions/s3uploader/S3Uploader.java

@@ -18,6 +18,7 @@ import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 public class S3Uploader implements ScenarioMetadataAware {
@@ -32,19 +33,25 @@ public class S3Uploader implements ScenarioMetadataAware {
         this.scriptContext = scriptContext;
     }
 
+    /**
+     * Upload the local file path to the specified S3 URL, then return the URL of the bucket
+     * in its fully expanded form. See the details on token expansions in the s3.md help docs.
+     * @param localFilePath The path to the local directory
+     * @param urlTemplate A template that is expanded to a valid S3 URL
+     * @return The fully expanded name of the URL used for upload
+     */
     public String uploadDirToUrl(String localFilePath, String urlTemplate) {
         return uploadDirToUrlTokenized(localFilePath, urlTemplate, Map.of());
     }
 
     /**
-     * Upload the local file path to the specified URL, and then return the URL of the
-     * bucket in its fully expanded form.
-     * This requires you to specify a valid S3 url with place-holder tokens, such as
-     * <pre>{@code s3://bucketname/prefix1/prefix2/DATA}</pre>
-     * Before processing, some tokens will be automatically expanded based on local node
-     * info. These tokens include:
-     *
-     * @return The bucket URL of the expaneded name.
+     * Upload the local file path to the specified S3 URL, then return the URL of the bucket
+     * in its fully expanded form. See the details on token expansions in the s3.md help docs.
+     * Any params which are provided supersede the normally provided values from the system.
+     * @param localFilePath The path to the local directory
+     * @param urlTemplate A template that is expanded to a valid S3 URL
+     * @param params Additional token expansions which will take precedence over other available values.
+     * @return The fully expanded name of the URL used for upload
      */
     public String uploadDirToUrlTokenized(String localFilePath, String urlTemplate, Map<String,String> params) {
 
@@ -58,9 +65,16 @@ public class S3Uploader implements ScenarioMetadataAware {
         }
         File sourceDir = sourcePath.toFile();
 
-        String url = NBEnvironment.INSTANCE
-            .interpolateWithTimestamp(urlTemplate,scenarioMetadata.getStartedAt(),params)
+        Map<String,String> combined = new LinkedHashMap<>(params);
+        combined.putAll(scenarioMetadata.asMap());
+        String url = NBEnvironment.INSTANCE.interpolateWithTimestamp(
+            urlTemplate,
+                scenarioMetadata.getStartedAt(),
+                combined
+            )
             .orElseThrow();
+        logger.debug("S3 composite URL is '" + url + "'");
+
         S3UrlFields fields = S3UrlFields.fromURLString(url);
         S3ClientCache s3ClientCache = new S3ClientCache();
         AmazonS3 s3 = s3ClientCache.get(fields);

engine-extensions/src/main/java/io/nosqlbench/engine/extensions/s3uploader/s3.md

@@ -0,0 +1,68 @@
+S3 extension
+==============
+
+Allow uploading of a local directory on the default filesystem
+to an S3 bucket, using an S3 URI to specify the bucket, location, and so on.
+
+The URL is specified in the standard S3 format, such as:
+
+1. `s3://mybucket/mypath-as-a-key/with-any-level-of-depth`
+2. `s3://myuser:mypass@mybucket/mypath-as-a-key/with-any-level-of-depth`
+
+In addition, any tokens which are supported by the standard NoSQLBench
+token substitution mechanism will be used to construct a URL at the time
+of usage. These forms include the following:
+
+- Scenario Metadata - There are several key fields initialized for a scenario which can be used as common
+  reference points. These occlude the environment variables of the same name. These are:
+  - SESSION_NAME - The name auto-generated for a session, used in the logfile names, and so on.
+  - SYSTEM_ID - The string form of the most canonically identifying IP address, excluding
+    known symbolic interface names (docker*, for example) and all localhost addresses.
+  - SYSTEM_FINGERPRINT - a stable and anonymized identifier for a given system. This will be
+    stable as long as the networking configuration does not change.
+- System Properties
+  - Any parameter in `$word1.word2...` form -- any multi-part variable name with separating dots
+    is taken as a system property to the JVM. These are expanded in place. Both `$word1.word2`
+    and `${word1.word2}` patterns are supported, whereas the latter is more strict and thus safer.
+- Environment Variables
+  - As with System Properties, environment variable form the shell are also supported, as long
+    as they do not include a dot.
+- Temporal Fields from the Scenario start time
+  - Any field specifier that you can use with the temporal types in Java's standard String.
+    format can be used. The reference time for these is always the scenario start time.
+  - Example: The default session name template looks like `scenario_%tY%tm%td_%tH%tM%tS_%tL`
+
+## Examples
+```
+// If you have local logical identifiers in your scenario script which you want
+// to templatize into your upload paths, you can provide your own js object
+// as the third parameter
+s3.uploadDirToUrlTokenized(
+ 'metrics',
+ 's3://test-results/${HOSTNAME}/${testid}-${testversion}/metrics',
+ {
+  'testid':'20210343',
+  'testversion':'v2'
+ }
+);
+
+// Otherwise, use the two-parameter version:
+s3.uploadDirToUrl('metrics','s3://test-results/${HOSTNAME}/metrics');
+```
+
+## Post-Hoc upload
+
+Scripting extensions only run if the scenario is not halted before they are invoked
+in the main scenario script. If you want to ensure that this one runs after a test,
+regardless of when or why the test stopped, it is possible to wrap it within
+a shutdown hook which will run after scenario completion.
+
+This is an example of how to do so:
+
+```
+shutdown.addShutdownHook('upload_metrics', function f() {
+ s3.uploadDirToUrl('metrics','s3://test-results/${HOSTNAME}/metrics');
+});
+
+```
+

nb-api/src/main/java/io/nosqlbench/nb/api/NBEnvironment.java

@@ -92,16 +92,26 @@ public class NBEnvironment {
      * @param defaultValue The value to return if the name is not found
      * @return the system property or environment variable's value, or the default value
      */
-    public String getOr(String name, String defaultValue) {
-        String value = peek(name);
+    public String getOr(String name, String defaultValue, Map<String,String> supplemental) {
+        String value = peek(name, supplemental);
         if (value == null) {
             value = defaultValue;
         }
         return reference(name, value);
     }
 
-    private String peek(String name) {
+    public String getOr(String name, String defaultValue) {
+        return getOr(name, defaultValue, Map.of());
+    }
+
+    private String peek(String name, Map<String,String> supplemental) {
         String value = null;
+        if (supplemental.containsKey(name)) {
+            value = supplemental.get(name);
+            if (value!=null) {
+                return value;
+            }
+        }
         if (name.contains(".")) {
             value = System.getProperty(name.toLowerCase());
             if (value != null) {
@@ -145,7 +155,11 @@ public class NBEnvironment {
     }
 
     public boolean containsKey(String name) {
-        String value = peek(name);
+        return containsKey(name, Map.of());
+    }
+
+    public boolean containsKey(String name, Map<String,String> supplemental) {
+        String value = peek(name, supplemental);
         return (value != null);
     }
 
@@ -162,7 +176,7 @@ public class NBEnvironment {
      * @param word The word to interpolate the environment values into
      * @return The interpolated value, after substitutions, or null if any lookup failed
      */
-    public Optional<String> interpolate(String word) {
+    public Optional<String> interpolate(String word, Map<String,String> supplemental) {
         Pattern envpattern = Pattern.compile("(\\$(?<env1>[a-zA-Z_][A-Za-z0-9_.]+)|\\$\\{(?<env2>[^}]+)\\})");
         Matcher matcher = envpattern.matcher(word);
         StringBuilder sb = new StringBuilder();
@@ -171,7 +185,7 @@ public class NBEnvironment {
             if (envvar == null) {
                 envvar = matcher.group("env2");
             }
-            String value = peek(envvar);
+            String value = peek(envvar,supplemental);
             if (value == null) {
                 if (logger != null) {
                     logger.debug("no value found for '" + envvar + "', returning Optional.empty() for '" + word + "'");
@@ -186,6 +200,10 @@ public class NBEnvironment {
         return Optional.of(sb.toString());
     }
 
+    public Optional<String> interpolate(String word) {
+        return interpolate(word,Map.of());
+    }
+
     public List<String> interpolateEach(CharSequence delim, String toBeRecombined) {
         String[] split = toBeRecombined.split(delim.toString());
         List<String> mapped = new ArrayList<>();
@@ -224,12 +242,8 @@ public class NBEnvironment {
      */
     public final Optional<String> interpolateWithTimestamp(String rawtext, long millis, Map<String, String> map) {
         String result = rawtext;
-        for (String key : map.keySet()) {
-            String value = map.get(key);
-            result = result.replaceAll(Pattern.quote(key), value);
-        }
         result = SessionNamer.format(result, millis);
-        return interpolate(result);
+        return interpolate(result,map);
     }
 
     public final Optional<String> interpolateWithTimestamp(String rawText, long millis) {

nb-api/src/test/java/io/nosqlbench/nb/addins/s3/s3utils/S3UploaderDemo.java

@@ -26,9 +26,11 @@ public class S3UploaderDemo {
         if (!FileSystems.getDefault().equals(sourcePath.getFileSystem())) {
             throw new RuntimeException("The file must reside on the default filesystem to be uploaded by S3.");
         }
+
         if (!Files.isDirectory(sourcePath, LinkOption.NOFOLLOW_LINKS)) {
             throw new RuntimeException("path '" + sourcePath + "' is not a directory.");
         }
+
         TransferManager tm = TransferManagerBuilder.defaultTransferManager();
         MultipleFileUpload mfu = tm.uploadDirectory(bucket, prefix, sourcePath.toFile(), true);
         try {

nb-api/src/test/java/io/nosqlbench/nb/api/NBEnvironmentTest.java

@@ -3,6 +3,7 @@ package io.nosqlbench.nb.api;
 import org.junit.jupiter.api.Test;
 
 import java.util.Map;
+import java.util.Optional;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -23,7 +24,16 @@ public class NBEnvironmentTest {
         long millis = 1633964892320L;
         String time1 = env.interpolateWithTimestamp("word WOO %td %% end", millis, Map.of("WOO","WOW")).orElse(null);
         assertThat(time1).isEqualTo("word WOW 11 % end");
+    }
 
+    @Test
+    public void testInterpolationPrecedence() {
+        NBEnvironment env = new NBEnvironment();
+        Optional<String> superseded = env.interpolate("$TEST_KEY, $USER", Map.of("TEST_KEY", "supersedes1", "USER", "supersedes2"));
+        assertThat(superseded).contains("supersedes1, supersedes2");
+        superseded = env.interpolate("$USER", Map.of("TEST_KEY", "supersedes1"));
+        assertThat(superseded).isPresent();
+        assertThat(superseded.get()).isNotEqualTo("supersedes2");
     }
 
 }