From 4484292cb60c74f7651cfb6e0b237e0fb551792b Mon Sep 17 00:00:00 2001 From: Jeff Banks Date: Thu, 12 Jan 2023 10:04:19 -0600 Subject: [PATCH] md5 deprecation (#909) md5 deprecation per code scan recommendations. --- .../examples/bindings-bytebuffers.yaml | 10 ++- .../to_bytebuffer/DigestToByteBuffer.java | 59 +++++++++------ .../to_bytebuffer/ToMD5ByteBuffer.java | 47 ++++++------ .../from_long/to_string/MD5HexString.java | 71 ------------------- .../to_bytebuffer/DigestToByteBufferTest.java | 50 ++++++++++--- .../to_string/ToMD5ByteBufferTest.java | 32 --------- .../bindings/funcref_conversion.md | 1 + 7 files changed, 105 insertions(+), 165 deletions(-) delete mode 100644 virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/MD5HexString.java delete mode 100644 virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ToMD5ByteBufferTest.java diff --git a/nbr/src/main/resources/examples/bindings-bytebuffers.yaml b/nbr/src/main/resources/examples/bindings-bytebuffers.yaml index 5966c87ae..19e3f51a0 100644 --- a/nbr/src/main/resources/examples/bindings-bytebuffers.yaml +++ b/nbr/src/main/resources/examples/bindings-bytebuffers.yaml @@ -33,19 +33,17 @@ bindings: # convert a ByteBuffer to a hex-encoded string with lower case bb_to_hex_lc: ByteBufferSizedHashed(20); ToHexString(false); - # generate a byte buffer of 1000 bytes, and then compute a MD5 + # generate a byte buffer of 1000 bytes, and then compute a SHA-256 # digest into another byte buffer - digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('MD5'); ToHexString(); + digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('SHA-256'); ToHexString(); - # Md5 digest as above, but using a long as input, short-circuiting + # SHA-256 digest as above, but using a long as input, short-circuiting # the byte buffer construction spelled out above. This is easier # to use and faster to generate, although any digest will be # more intensive to calculate as test data, so only use digests # where you have specific testing requirements for them. - digest_bb_direct: DigestToByteBuffer('MD5'); + digest_bb_direct: DigestToByteBuffer('SHA-256'); - # A canned version of the above - long_md5_bb: ToMD5ByteBuffer(); ToHexString(); # The example below show various type-specialized ByteBuffer # functions which are automatically selected depending on the diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBuffer.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBuffer.java index 7e0fd785a..6ea0d4636 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBuffer.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBuffer.java @@ -20,9 +20,12 @@ import io.nosqlbench.virtdata.api.annotations.Categories; import io.nosqlbench.virtdata.api.annotations.Category; import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; import org.apache.commons.codec.digest.MessageDigestAlgorithms; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.nio.ByteBuffer; import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.function.LongFunction; import java.util.function.Supplier; @@ -33,44 +36,58 @@ import java.util.stream.Collectors; @ThreadSafeMapper public class DigestToByteBuffer implements LongFunction { - private transient static ThreadLocal tl_state; + private static final Logger logger = LogManager.getLogger(DigestToByteBuffer.class); + private static ThreadLocal state = null; public DigestToByteBuffer(String digestType) { - for (String digestName : MessageDigestAlgorithms.values()) { - if (digestName.equals(digestType)) { - Supplier mds = () -> getDigest(digestName); - tl_state = ThreadLocal.withInitial(() -> new TL_State(mds)); - break; - } + if (!Arrays.asList(MessageDigestAlgorithms.values()).contains(digestType)) { + throw new RuntimeException("A digest of type " + digestType + + " was not found. Select a digest type from: " + + Arrays.stream(MessageDigestAlgorithms.values()). + collect(Collectors.joining(",", "[", "]"))); } - if (tl_state==null) { - tl_state = ThreadLocal.withInitial(() -> new TL_State(() -> getDigest(digestType))); - } - } - private static MessageDigest getDigest(String type) { try { - return MessageDigest.getInstance(type); + if (digestType.equalsIgnoreCase("md5") ||digestType.equalsIgnoreCase("md2") ) { + logger.warn("Not recommended to use 'MD5 or MD2'. A stronger message digest algorithm is recommended."); + } + + if (state != null) { + state.remove(); + } + + final Supplier mds = () -> { + try { + return MessageDigest.getInstance(digestType); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + }; + state = ThreadLocal.withInitial(() -> new ThreadLocalState(mds)); + } catch (Exception e) { - throw new RuntimeException("A digest of type " + type + " was not found. Select a digest type from: " + - Arrays.stream(MessageDigestAlgorithms.values()).collect(Collectors.joining(",", "[", "]"))); + throw new RuntimeException("Unexpected error: ", e); } + } @Override public ByteBuffer apply(long value) { - TL_State state = tl_state.get(); - state.buf.putLong(0,value); - byte[] digest = state.digest.digest(state.buf.array()); - return ByteBuffer.wrap(digest); + if (DigestToByteBuffer.state != null) { + final ThreadLocalState tlState = DigestToByteBuffer.state.get(); + tlState.buf.putLong(0, value); + byte[] digest = tlState.digest.digest(tlState.buf.array()); + return ByteBuffer.wrap(digest); + } + throw new RuntimeException("Unable to apply long value as state is not initialized."); } - private final static class TL_State { + private static final class ThreadLocalState { private final MessageDigest digest; private final ByteBuffer buf = ByteBuffer.allocate(Long.BYTES); - public TL_State(Supplier mds) { + public ThreadLocalState(Supplier mds) { digest = mds.get(); } } diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ToMD5ByteBuffer.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ToMD5ByteBuffer.java index 25c3ac8b5..010430751 100644 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ToMD5ByteBuffer.java +++ b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ToMD5ByteBuffer.java @@ -22,48 +22,41 @@ import io.nosqlbench.virtdata.api.annotations.Example; import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.function.LongFunction; /** * Converts the byte image of the input long to a MD5 digest in ByteBuffer form. + * Deprecated usage due to unsafe MD5 digest. + * Replaced with DigestToByteBuffer with MD5 when absolutely needed for existing NB tests. + * However, stronger encryption algorithms (e.g. SHA-256) are recommended due to MD5's limitations. */ -@Categories({Category.conversion,Category.premade}) +@Categories({Category.conversion, Category.premade}) @ThreadSafeMapper +@Deprecated(since = "NB5", forRemoval = true) public class ToMD5ByteBuffer implements LongFunction { - private final MessageDigest md5; - private transient static final ThreadLocal tl_state = ThreadLocal.withInitial(TLState::new); - @Example({"MD5ByteBuffer()","convert the a input to an md5 digest of its bytes"}) + /** + * Deprecated usage due to unsafe MD5 digest. + * Use the DigestToByteBuffer with alternatives other than MD5. + */ + @Example({"MD5ByteBuffer()", "convert the a input to an md5 digest of its bytes"}) + @Deprecated public ToMD5ByteBuffer() { - try { - md5 = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + throw new RuntimeException("No longer available. Please use the DigestToByteBuffer with " + + "alternatives other than MD5"); } + /** + * Deprecated usage due to unsafe MD5 digest used in this class. + * Use the DigestToByteBuffer with alternatives other than MD5. + */ @Override + @Deprecated public ByteBuffer apply(long value) { - TLState state = tl_state.get(); - state.md5.reset(); - state.bytes.putLong(0,value); - byte[] digest = md5.digest(state.bytes.array()); - return ByteBuffer.wrap(digest); + throw new RuntimeException("No longer available. Please use the DigestToByteBuffer with " + + "alternatives other than MD5"); } - private final static class TLState { - public final ByteBuffer bytes = ByteBuffer.allocate(160); - public final MessageDigest md5; - public TLState() { - try { - md5 = MessageDigest.getInstance("MD5"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } } diff --git a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/MD5HexString.java b/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/MD5HexString.java deleted file mode 100644 index 520ddad75..000000000 --- a/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/MD5HexString.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string; - -import io.nosqlbench.virtdata.api.annotations.Categories; -import io.nosqlbench.virtdata.api.annotations.Category; -import io.nosqlbench.virtdata.api.annotations.Example; -import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; -import org.apache.commons.codec.binary.Hex; - -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.function.LongFunction; - -/** - * Computes the MD5 digest of the byte image of the input long, and - * returns it in hexadecimal String form. - */ -@Categories(Category.conversion) -@ThreadSafeMapper -public class MD5HexString implements LongFunction { - - private final MessageDigest md5; - private static final transient ThreadLocal tl_state = ThreadLocal.withInitial(TLState::new); - - @Example({"MD5String()","Convert a long input to an md5 digest over its bytes, and then to a hexadecimal string."}) - public MD5HexString() { - try { - md5 = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - - @Override - public String apply(long value) { - TLState state = tl_state.get(); - state.bytes.putLong(0,value); - byte[] digest = md5.digest(state.bytes.array()); - String hexDigest = Hex.encodeHexString(digest); - return hexDigest; - } - - private final static class TLState { - public final ByteBuffer bytes = ByteBuffer.allocate(16); - public final MessageDigest md5; - public TLState() { - try { - md5 = MessageDigest.getInstance("MD5"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - } -} diff --git a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBufferTest.java b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBufferTest.java index 2304bab14..e4b818ff8 100644 --- a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBufferTest.java +++ b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBufferTest.java @@ -23,13 +23,12 @@ import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.*; -public class DigestToByteBufferTest { +class DigestToByteBufferTest { @Test - public void testWithMD5() { + void testWithMD5() { DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.MD5); ByteBuffer digest = d1.apply(233423L); byte[] bytes; @@ -43,7 +42,7 @@ public class DigestToByteBufferTest { } @Test - public void testWithSHA1() { + void testWithSHA1() { DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.SHA_1); ByteBuffer digest = d1.apply(233423L); byte[] bytes; @@ -57,10 +56,45 @@ public class DigestToByteBufferTest { } @Test - public void testInvalidName() { - DigestToByteBuffer d1 = new DigestToByteBuffer("Whoops"); + void testInvalidNames() { + assertThatExceptionOfType(RuntimeException.class) - .isThrownBy(() -> d1.apply(233423L)); + .isThrownBy(() -> new DigestToByteBuffer("Whoops")); + + assertThatExceptionOfType(RuntimeException.class) + .isThrownBy(() -> new DigestToByteBuffer("")); + + } + + @Test + void testInstances() { + + DigestToByteBuffer sha256 = new DigestToByteBuffer("SHA-256"); + DigestToByteBuffer sha512 = new DigestToByteBuffer("SHA-512"); + + try { + ByteBuffer sha256Digest = sha256.apply(8675309L); + ByteBuffer sha512Digest = sha512.apply(8675309L); + + byte[] bytesFromSha256; + byte[] bytesFromSha512; + try { + bytesFromSha256 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b"); + bytesFromSha512 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b"); + + } catch (DecoderException e) { + throw new RuntimeException(e); + } + + // System.out.println(Hex.encodeHexString(sha256Digest)); + // System.out.println(Hex.encodeHexString(sha512Digest)); + + assertThat(sha256Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha256)); + assertThat(sha512Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha512)); + } catch(Exception e) { + fail("unexpected exception found."); + } + } } diff --git a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ToMD5ByteBufferTest.java b/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ToMD5ByteBufferTest.java deleted file mode 100644 index a42e1b2fb..000000000 --- a/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ToMD5ByteBufferTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string; - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ToMD5ByteBufferTest { - - @Test - public void testMD5String() { - MD5HexString ms = new MD5HexString(); - String apply = ms.apply(3L); - assertThat(apply).isEqualTo("1fb332efe1406a104b11ffa1fa04fa7a"); - } - -} diff --git a/virtdata-userlibs/src/main/resources/docs-for-virtdata/bindings/funcref_conversion.md b/virtdata-userlibs/src/main/resources/docs-for-virtdata/bindings/funcref_conversion.md index 687db1d7f..9cffdab22 100644 --- a/virtdata-userlibs/src/main/resources/docs-for-virtdata/bindings/funcref_conversion.md +++ b/virtdata-userlibs/src/main/resources/docs-for-virtdata/bindings/funcref_conversion.md @@ -228,6 +228,7 @@ Convert the input value to a long. ## ToMD5ByteBuffer +**[ DEPRECATED ]** no longer supported use DigestToByteBuffer with stronger digest algorithm. Converts the byte image of the input long to a MD5 digest in ByteBuffer form.