http: explain http status codes in diagnostic output

driver-http/src/main/java/io/nosqlbench/activitytype/http/HttpConsoleFormats.java

@@ -51,8 +51,9 @@ public class HttpConsoleFormats {
         redirects(_REDIRECTS),
         requests(_REQUESTS),
         responses(_RESPONSES),
+        codes(_CODES),
         brief(_HEADERS | _STATS | _REQUESTS | _RESPONSES | _DATA10),
-        all(_HEADERS | _STATS | _REDIRECTS | _REQUESTS | _RESPONSES | _DATA);
+        all(_HEADERS | _STATS | _REDIRECTS | _REQUESTS | _RESPONSES | _DATA | _CODES);
 
         private final long mask;
 
@@ -191,6 +192,10 @@ public class HttpConsoleFormats {
         out.println(RESPONSE_CUE + (caption != null ? caption : " RESPONSE") +
             " status=" + response.statusCode() + " took=" + (nanos / 1_000_000) + "ms");
 
+        if (Diag.codes.includedIn(mask)) {
+            out.println(DETAIL_CUE + "STATUS: " + HttpStatusCodes.lookup(response.statusCode()));
+        }
+
         if (e != null) {
             out.println(MESSAGE_CUE + " EXCEPTION: " + e.getMessage());
         }

driver-http/src/main/java/io/nosqlbench/activitytype/http/statuscodes/HttpStatusCodes.java

@@ -0,0 +1,64 @@
+package io.nosqlbench.activitytype.http.statuscodes;
+
+import io.nosqlbench.nb.api.content.Content;
+import io.nosqlbench.nb.api.content.NBIO;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class HttpStatusCodes {
+
+    private static final IetfStatusCode[] codes = loadMap();
+
+    private static IetfStatusCode[] loadMap() {
+        Content<?> csv = NBIO.local().name("ietf-http-status-codes").extension("csv").one();
+        InputStreamReader isr = new InputStreamReader(csv.getInputStream());
+        IetfStatusCode[] codes = new IetfStatusCode[600];
+
+        try {
+            CSVParser parser = new CSVParser(isr,CSVFormat.DEFAULT.withFirstRecordAsHeader());
+            for (CSVRecord record : parser) {
+                String values = record.get("Value");
+                String description = record.get("Description");
+                String reference = record.get("Reference");
+
+                int min, max=0;
+                if (values.contains("-")) {
+                    min=Integer.parseInt(values.substring(0,values.indexOf('-')));
+                    max=Integer.parseInt(values.substring(values.indexOf('-')));
+                } else {
+                    min = max = Integer.parseInt(values);
+                }
+                HttpStatusRanges category = HttpStatusRanges.valueOfCode(min);
+                IetfStatusCode code = new IetfStatusCode(values,description,reference,category);
+                for (int value = min; value <=max ; value++) {
+                    codes[value]=code;
+                }
+            }
+
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        return codes;
+    }
+
+    public static IetfStatusCode lookup(int code) {
+        if (code<1||code>codes.length-1) {
+            return UNKNOWN(code);
+        }
+        IetfStatusCode found = codes[code];
+        if (found!=null) {
+            return found;
+        } else {
+            return UNKNOWN(code);
+        }
+    }
+
+    private static IetfStatusCode UNKNOWN(int code) {
+        return new IetfStatusCode(String.valueOf(code),null, "[check https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml]", HttpStatusRanges.valueOfCode(code));
+    }
+}

driver-http/src/main/java/io/nosqlbench/activitytype/http/statuscodes/HttpStatusRanges.java

@@ -0,0 +1,35 @@
+package io.nosqlbench.activitytype.http.statuscodes;
+
+enum HttpStatusRanges {
+    Informational("INFORMATIONAL", 100, 199, "Request received, continuing process"),
+    Success("SUCCESS",200, 299, "Request successfully received, understood, and accepted"),
+    Redirection("REDIRECTION", 300, 399, "Further action must be taken in order to complete the request."),
+    Client_Error("CLIENT_ERROR",400, 499, "The request contains bad syntax or cannot be fulfilled."),
+    Server_Error("SERVER_ERROR",500, 599, "The server failed to fulfill an apparently valid request."),
+    Unknown("UNKNOWN_ERROR",0,0,"This error type is not known based on IANA registered HTTP status codes.");
+
+    private final String name;
+    private final String description;
+    private final int min;
+    private final int max;
+
+    HttpStatusRanges(String name, int min, int max, String description) {
+        this.name = name;
+        this.min = min;
+        this.max = max;
+        this.description = description;
+    }
+
+    public static HttpStatusRanges valueOfCode(int code) {
+        for (HttpStatusRanges value : HttpStatusRanges.values()) {
+            if (code >= value.min && code <= value.max) {
+                return value;
+            }
+        }
+        return HttpStatusRanges.Unknown;
+    }
+
+    public String toString() {
+        return this.name + " (" + this.description + ")";
+    }
+}

driver-http/src/main/java/io/nosqlbench/activitytype/http/statuscodes/IetfStatusCode.java

@@ -0,0 +1,47 @@
+package io.nosqlbench.activitytype.http.statuscodes;
+
+public class IetfStatusCode {
+    private final String values;
+    private final String description;
+    private final String reference;
+    private final HttpStatusRanges category;
+
+    public IetfStatusCode(String values, String description, String reference, HttpStatusRanges category) {
+        this.values = values;
+        this.description = description;
+        this.reference = reference;
+        this.category = category;
+    }
+
+    public String getValues() {
+        return values;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public String getReference() {
+        return reference;
+    }
+
+    public HttpStatusRanges getCategory() {
+        return category;
+    }
+
+    public String toString(int code) {
+        if (values.equals(String.valueOf(code))) {
+            return toString();
+        } else {
+            return code + ": " + this;
+        }
+    }
+
+    public String toString() {
+        String ref = reference
+            .replaceFirst("\\[RFC(\\d+), Section (.+?)]","[https://www.iana.org/go/rfc$1#section-$2]") // https://www.rfc-editor.org/rfc/rfc7231.html#section-6.3.1
+            .replaceFirst("\\[RFC(\\d+)(.*)]","[https://www.iana.org/go/rfc$1$2]");  // https://www.iana.org/go/rfc7231
+
+        return (values!=null ? values : "") + (description!=null ? ", "+description :"") + ", " + ref + ", " + category.toString();
+    }
+}

driver-http/src/main/resources/http.md

@@ -106,7 +106,7 @@ statements:
 
 The above two examples are semantically identical, only the format is
 different. Notice that the expansion of the URI is still captured in a
-field called uri, with all of the dynamic pieces stitched together in the
+field called uri, with all the dynamic pieces stitched together in the
 value. You can't use arbitrary request fields. Every request field must
 from (method, uri, version, body, ok-status, ok-body) or otherwise be
 capitalized to signify an HTTP header.
@@ -122,7 +122,7 @@ cached at startup.
 
 ## Request Fields
 
-At a minimum, a **URI** must be provided. These are enough to build a
+At a minimum, a **URI** must be provided. This is enough to build a
 request with. All other request fields are optional and have reasonable
 defaults:
 
@@ -172,7 +172,7 @@ By default, a request which encounters an exception is retried up to 10
 times. If you want to change this, set another value to the
 `retries=` activity parameters.
 
-Presently, no determination is made about whether or not an errored
+Presently, no determination is made about whether an errored
 response *should* be retryable, but it is possible to configure this if
 you have a specific exception type that indicates a retryable operation.
 
@@ -200,7 +200,10 @@ Presently, this driver only does basic request-response style requests.
 Thus, adding headers which take TCP socket control away from the
 HttpClient will likely yield inconsistent (or undefined)
 results. Support may be added for long-lived connections in a future
-release.
+release. However, chunked encoding responses are supported, although they
+will be received fully before being processed further. Connecting to a long-lived
+connection that streams chunked encoding responses indefinitely will have
+undefined results.
 
 ## HTTP Activity Parameters
 
@@ -217,11 +220,11 @@ release.
   including only brief details as explained below.
 
   This setting is a selector for what level of verbosity you will get on
-  the console. If you set this to true, you'll get every request and
+  the console. If you set this to `diag=all`, you'll get every request and
   response logged to console. This is only for verifying that a test is
   configured and to spot check services before running higher scale tests.
 
-  All of the data shown in diagnostics is post-hoc, directly from the
+  All the data shown in diagnostics is post-hoc, directly from the
   response provided by the internal HTTP client in the Java runtime.
 
   If you want finer control over how much information diagnostics
@@ -229,18 +232,19 @@ release.
 
     - headers - show headers
     - stats - show basic stats of each request
+    - data - show all of each response body this setting
     - data10 - show only the first 10 characters of each response body
+      this setting supersedes `data`
     - data100 - show only the first 100 characters of each response body
       this setting supersedes `data10`
     - data1000 - show only the first 1000 characters of each response body
       this setting supersedes `data100`
-    - data - show all of each response body this setting
-      supersedes `data1000`
     - redirects - show details for interstitial request which are made
       when the client follows a redirect directive like a `location`
-      header.
+      header
     - requests - show details for requests
     - responses - show details for responses
+    - codes - shows explanatory details (high-level) of http response status codes
     - brief - Show headers, stats, requests, responses, and 10 characters
     - all - Show everything, including full payloads and redirects
     - a modulo - any number, like 3000 - causes the diagnostics to be
@@ -248,12 +252,10 @@ release.
       then you will get the brief diagnostic output for every 300th
       response.
 
-  The requests, responses, and redirects setting work intersectionally.
+  The requests, responses, and redirects settings work in combination.
   For example, if you specify responses, and redirect, but not requests,
   then you will only see the response portion of all calls made by the
-  client.
-
-  All of the diagnostic filters are incrementally added.
+  client. All available filters layer together in this way.
 
 - **timeout** - default: forever - Sets the timeout of each request in
   milliseconds.

driver-http/src/test/java/io/nosqlbench/activitytype/http/statuscodes/HttpStatusCodesTest.java

@@ -0,0 +1,43 @@
+package io.nosqlbench.activitytype.http.statuscodes;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class HttpStatusCodesTest {
+
+    @Test
+    public void testLookup() {
+        IetfStatusCode result = HttpStatusCodes.lookup(404);
+        assertThat(result.getCategory()).isSameAs(HttpStatusRanges.Client_Error);
+        assertThat(result.getReference()).isEqualTo("[RFC7231, Section 6.5.4]");
+        assertThat(result.getValues()).isEqualTo("404");
+        assertThat(result.getDescription()).isEqualTo("Not Found");
+        System.out.println(result.toString(404));
+        assertThat(result.toString(404)).isEqualTo("404, Not Found, [https://www.iana.org/go/rfc7231#section-6.5.4], CLIENT_ERROR (The request contains bad syntax or cannot be fulfilled.)");
+    }
+
+    @Test
+    public void testUnknownCodeLookupGap() {
+        IetfStatusCode result = HttpStatusCodes.lookup(496);
+        assertThat(result.getCategory()).isSameAs(HttpStatusRanges.Client_Error);
+        assertThat(result.getReference()).isEqualTo("[check https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml]");
+        assertThat(result.getValues()).isEqualTo("496");
+        assertThat(result.getDescription()).isNullOrEmpty();
+        System.out.println(result.toString(496));
+        assertThat(result.toString(496)).isEqualTo("496, [check https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml], CLIENT_ERROR (The request contains bad syntax or cannot be fulfilled.)");
+    }
+
+    @Test
+    public void testUnknownCodeLookupRange() {
+        IetfStatusCode result = HttpStatusCodes.lookup(747);
+        assertThat(result.getCategory()).isSameAs(HttpStatusRanges.Unknown);
+        assertThat(result.getReference()).isEqualTo("[check https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml]");
+        assertThat(result.getValues()).isEqualTo("747");
+        assertThat(result.getDescription()).isNullOrEmpty();
+        System.out.println(result.toString(747));
+        assertThat(result.toString(747)).isEqualTo("747, [check https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml], UNKNOWN_ERROR (This error type is not known based on IANA registered HTTP status codes.)");
+
+    }
+
+}