md5 deprecation (#909)

md5 deprecation per code scan recommendations.

nbr/src/main/resources/examples/bindings-bytebuffers.yaml

@@ -33,19 +33,17 @@ bindings:
   # convert a ByteBuffer to a hex-encoded string with lower case
   bb_to_hex_lc: ByteBufferSizedHashed(20); ToHexString(false);
 
-  # generate a byte buffer of 1000 bytes, and then compute a MD5
+  # generate a byte buffer of 1000 bytes, and then compute a SHA-256
   # digest into another byte buffer
-  digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('MD5'); ToHexString();
+  digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('SHA-256'); ToHexString();
 
-  # Md5 digest as above, but using a long as input, short-circuiting
+  # SHA-256 digest as above, but using a long as input, short-circuiting
   # the byte buffer construction spelled out above. This is easier
   # to use and faster to generate, although any digest will be
   # more intensive to calculate as test data, so only use digests
   # where you have specific testing requirements for them.
-  digest_bb_direct: DigestToByteBuffer('MD5');
+  digest_bb_direct: DigestToByteBuffer('SHA-256');
 
-  # A canned version of the above
-  long_md5_bb: ToMD5ByteBuffer(); ToHexString();
 
   # The example below show various type-specialized ByteBuffer
   # functions which are automatically selected depending on the

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBuffer.java

@@ -20,9 +20,12 @@ import io.nosqlbench.virtdata.api.annotations.Categories;
 import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
 import org.apache.commons.codec.digest.MessageDigestAlgorithms;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.nio.ByteBuffer;
 import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
 import java.util.function.LongFunction;
 import java.util.function.Supplier;
@@ -33,44 +36,58 @@ import java.util.stream.Collectors;
 @ThreadSafeMapper
 public class DigestToByteBuffer implements LongFunction<ByteBuffer> {
 
-    private transient static ThreadLocal<TL_State> tl_state;
+    private static final Logger logger = LogManager.getLogger(DigestToByteBuffer.class);
+    private static ThreadLocal<ThreadLocalState> state = null;
 
     public DigestToByteBuffer(String digestType) {
 
-        for (String digestName : MessageDigestAlgorithms.values()) {
+        if (!Arrays.asList(MessageDigestAlgorithms.values()).contains(digestType)) {
-            if (digestName.equals(digestType)) {
+            throw new RuntimeException("A digest of type " + digestType +
-                Supplier<MessageDigest> mds = () -> getDigest(digestName);
+                    " was not found. Select a digest type from: "
-                tl_state = ThreadLocal.withInitial(() -> new TL_State(mds));
+                    + Arrays.stream(MessageDigestAlgorithms.values()).
-                break;
+                    collect(Collectors.joining(",", "[", "]")));
-            }
         }
-        if (tl_state==null) {
-            tl_state = ThreadLocal.withInitial(() -> new TL_State(() -> getDigest(digestType)));
-        }
-    }
 
-    private static MessageDigest getDigest(String type) {
         try {
-            return MessageDigest.getInstance(type);
+            if (digestType.equalsIgnoreCase("md5") ||digestType.equalsIgnoreCase("md2") ) {
+                logger.warn("Not recommended to use 'MD5 or MD2'. A stronger message digest algorithm is recommended.");
+            }
+
+            if (state != null) {
+                state.remove();
+            }
+
+            final Supplier<MessageDigest> mds = () -> {
+                try {
+                    return MessageDigest.getInstance(digestType);
+                } catch (NoSuchAlgorithmException e) {
+                    throw new RuntimeException(e);
+                }
+            };
+            state = ThreadLocal.withInitial(() -> new ThreadLocalState(mds));
+
         } catch (Exception e) {
-            throw new RuntimeException("A digest of type " + type + " was not found. Select a digest type from: " +
+            throw new RuntimeException("Unexpected error: ", e);
-                    Arrays.stream(MessageDigestAlgorithms.values()).collect(Collectors.joining(",", "[", "]")));
         }
+
     }
 
     @Override
     public ByteBuffer apply(long value) {
-        TL_State state = tl_state.get();
+        if (DigestToByteBuffer.state != null) {
-        state.buf.putLong(0,value);
+            final ThreadLocalState tlState = DigestToByteBuffer.state.get();
-        byte[] digest = state.digest.digest(state.buf.array());
+            tlState.buf.putLong(0, value);
-        return ByteBuffer.wrap(digest);
+            byte[] digest = tlState.digest.digest(tlState.buf.array());
+            return ByteBuffer.wrap(digest);
+        }
+        throw new RuntimeException("Unable to apply long value as state is not initialized.");
     }
 
-    private final static class TL_State {
+    private static final class ThreadLocalState {
         private final MessageDigest digest;
         private final ByteBuffer buf = ByteBuffer.allocate(Long.BYTES);
 
-        public TL_State(Supplier<MessageDigest> mds) {
+        public ThreadLocalState(Supplier<MessageDigest> mds) {
             digest = mds.get();
         }
     }

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ToMD5ByteBuffer.java

@@ -22,48 +22,41 @@ import io.nosqlbench.virtdata.api.annotations.Example;
 import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
 
 import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
 import java.util.function.LongFunction;
 
 /**
  * Converts the byte image of the input long to a MD5 digest in ByteBuffer form.
+ * Deprecated usage due to unsafe MD5 digest.
+ * Replaced with DigestToByteBuffer with MD5 when absolutely needed for existing NB tests.
+ * However, stronger encryption algorithms (e.g. SHA-256) are recommended due to MD5's limitations.
  */
-@Categories({Category.conversion,Category.premade})
+@Categories({Category.conversion, Category.premade})
 @ThreadSafeMapper
+@Deprecated(since = "NB5", forRemoval = true)
 public class ToMD5ByteBuffer implements LongFunction<ByteBuffer> {
 
-    private final MessageDigest md5;
-    private transient static final ThreadLocal<TLState> tl_state = ThreadLocal.withInitial(TLState::new);
 
-    @Example({"MD5ByteBuffer()","convert the a input to an md5 digest of its bytes"})
+    /**
+     * Deprecated usage due to unsafe MD5 digest.
+     * Use the DigestToByteBuffer with alternatives other than MD5.
+     */
+    @Example({"MD5ByteBuffer()", "convert the a input to an md5 digest of its bytes"})
+    @Deprecated
     public ToMD5ByteBuffer() {
-        try {
+        throw new RuntimeException("No longer available.  Please use the DigestToByteBuffer with " +
-            md5 = MessageDigest.getInstance("MD5");
+                "alternatives other than MD5");
-        } catch (NoSuchAlgorithmException e) {
-            throw new RuntimeException(e);
-        }
     }
 
+    /**
+     * Deprecated usage due to unsafe MD5 digest used in this class.
+     * Use the DigestToByteBuffer with alternatives other than MD5.
+     */
     @Override
+    @Deprecated
     public ByteBuffer apply(long value) {
-        TLState state = tl_state.get();
+        throw new RuntimeException("No longer available.  Please use the DigestToByteBuffer with " +
-        state.md5.reset();
+                "alternatives other than MD5");
-        state.bytes.putLong(0,value);
-        byte[] digest = md5.digest(state.bytes.array());
-        return ByteBuffer.wrap(digest);
     }
 
-    private final static class TLState {
-        public final ByteBuffer bytes = ByteBuffer.allocate(160);
-        public final MessageDigest md5;
-        public TLState() {
-            try {
-                md5 = MessageDigest.getInstance("MD5");
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }
 
-    }
 }

virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/MD5HexString.java

@@ -1,71 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
-
-import io.nosqlbench.virtdata.api.annotations.Categories;
-import io.nosqlbench.virtdata.api.annotations.Category;
-import io.nosqlbench.virtdata.api.annotations.Example;
-import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
-import org.apache.commons.codec.binary.Hex;
-
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.function.LongFunction;
-
-/**
- * Computes the MD5 digest of the byte image of the input long, and
- * returns it in hexadecimal String form.
- */
-@Categories(Category.conversion)
-@ThreadSafeMapper
-public class MD5HexString implements LongFunction<String> {
-
-    private final MessageDigest md5;
-    private static final transient ThreadLocal<TLState> tl_state = ThreadLocal.withInitial(TLState::new);
-
-    @Example({"MD5String()","Convert a long input to an md5 digest over its bytes, and then to a hexadecimal string."})
-    public MD5HexString() {
-        try {
-            md5 = MessageDigest.getInstance("MD5");
-        } catch (NoSuchAlgorithmException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public String apply(long value) {
-        TLState state = tl_state.get();
-        state.bytes.putLong(0,value);
-        byte[] digest = md5.digest(state.bytes.array());
-        String hexDigest = Hex.encodeHexString(digest);
-        return hexDigest;
-    }
-
-    private final static class TLState {
-        public final ByteBuffer bytes = ByteBuffer.allocate(16);
-        public final MessageDigest md5;
-        public TLState() {
-            try {
-                md5 = MessageDigest.getInstance("MD5");
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-    }
-}

virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBufferTest.java

@@ -23,13 +23,12 @@ import org.junit.jupiter.api.Test;
 
 import java.nio.ByteBuffer;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.*;
-import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 
-public class DigestToByteBufferTest {
+class DigestToByteBufferTest {
 
     @Test
-    public void testWithMD5() {
+    void testWithMD5() {
         DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.MD5);
         ByteBuffer digest = d1.apply(233423L);
         byte[] bytes;
@@ -43,7 +42,7 @@ public class DigestToByteBufferTest {
     }
 
     @Test
-    public void testWithSHA1() {
+    void testWithSHA1() {
         DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.SHA_1);
         ByteBuffer digest = d1.apply(233423L);
         byte[] bytes;
@@ -57,10 +56,45 @@ public class DigestToByteBufferTest {
     }
 
     @Test
-    public void testInvalidName() {
+    void testInvalidNames() {
-        DigestToByteBuffer d1 = new DigestToByteBuffer("Whoops");
+
         assertThatExceptionOfType(RuntimeException.class)
-                .isThrownBy(() -> d1.apply(233423L));
+                .isThrownBy(() -> new DigestToByteBuffer("Whoops"));
+
+        assertThatExceptionOfType(RuntimeException.class)
+                .isThrownBy(() -> new DigestToByteBuffer(""));
+
+    }
+
+    @Test
+    void testInstances() {
+
+        DigestToByteBuffer sha256 = new DigestToByteBuffer("SHA-256");
+        DigestToByteBuffer sha512 = new DigestToByteBuffer("SHA-512");
+
+        try {
+            ByteBuffer sha256Digest = sha256.apply(8675309L);
+            ByteBuffer sha512Digest = sha512.apply(8675309L);
+
+            byte[] bytesFromSha256;
+            byte[] bytesFromSha512;
+            try {
+                bytesFromSha256 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b");
+                bytesFromSha512 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b");
+
+            } catch (DecoderException e) {
+                throw new RuntimeException(e);
+            }
+
+            // System.out.println(Hex.encodeHexString(sha256Digest));
+            // System.out.println(Hex.encodeHexString(sha512Digest));
+
+            assertThat(sha256Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha256));
+            assertThat(sha512Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha512));
+        } catch(Exception e) {
+            fail("unexpected exception found.");
+        }
+
     }
 
 }

virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ToMD5ByteBufferTest.java

@@ -1,32 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
-
-import org.junit.jupiter.api.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class ToMD5ByteBufferTest {
-
-    @Test
-    public void testMD5String() {
-        MD5HexString ms = new MD5HexString();
-        String apply = ms.apply(3L);
-        assertThat(apply).isEqualTo("1fb332efe1406a104b11ffa1fa04fa7a");
-    }
-
-}

virtdata-userlibs/src/main/resources/docs-for-virtdata/bindings/funcref_conversion.md

@@ -228,6 +228,7 @@ Convert the input value to a long.
 
 
 ## ToMD5ByteBuffer
+**[ DEPRECATED ]** no longer supported use DigestToByteBuffer with stronger digest algorithm.
 
 Converts the byte image of the input long to a MD5 digest in ByteBuffer form.