IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

md5 deprecation (#909)

Browse Source 
md5 deprecation per code scan recommendations.
This commit is contained in:
Jeff Banks
2023-01-12 10:04:19 -06:00
committed by GitHub
parent a95acae200
commit 4484292cb6
7 changed files with 105 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
nbr/src/main/resources/examples/bindings-bytebuffers.yaml
View File

@@ -33,19 +33,17 @@ bindings:
# convert a ByteBuffer to a hex-encoded string with lower case
bb_to_hex_lc: ByteBufferSizedHashed(20); ToHexString(false);
# generate a byte buffer of 1000 bytes, and then compute a MD5
# generate a byte buffer of 1000 bytes, and then compute a SHA-256
# digest into another byte buffer
digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('MD5'); ToHexString();
digest_bb: ByteBufferSizedHashed(1000); DigestToByteBuffer('SHA-256'); ToHexString();
# Md5 digest as above, but using a long as input, short-circuiting
# SHA-256 digest as above, but using a long as input, short-circuiting
# the byte buffer construction spelled out above. This is easier
# to use and faster to generate, although any digest will be
# more intensive to calculate as test data, so only use digests
# where you have specific testing requirements for them.
digest_bb_direct: DigestToByteBuffer('MD5');
digest_bb_direct: DigestToByteBuffer('SHA-256');
# A canned version of the above
long_md5_bb: ToMD5ByteBuffer(); ToHexString();
# The example below show various type-specialized ByteBuffer
# functions which are automatically selected depending on the

59
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBuffer.java
View File

@@ -20,9 +20,12 @@ import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.commons.codec.digest.MessageDigestAlgorithms;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.function.LongFunction;
import java.util.function.Supplier;
@@ -33,44 +36,58 @@ import java.util.stream.Collectors;
@ThreadSafeMapper
public class DigestToByteBuffer implements LongFunction<ByteBuffer> {
private transient static ThreadLocal<TL_State> tl_state;
private static final Logger logger = LogManager.getLogger(DigestToByteBuffer.class);
private static ThreadLocal<ThreadLocalState> state = null;
public DigestToByteBuffer(String digestType) {
for (String digestName : MessageDigestAlgorithms.values()) {
if (digestName.equals(digestType)) {
Supplier<MessageDigest> mds = () -> getDigest(digestName);
tl_state = ThreadLocal.withInitial(() -> new TL_State(mds));
break;
}
if (!Arrays.asList(MessageDigestAlgorithms.values()).contains(digestType)) {
throw new RuntimeException("A digest of type " + digestType +
" was not found. Select a digest type from: "
+ Arrays.stream(MessageDigestAlgorithms.values()).
collect(Collectors.joining(",", "[", "]")));
}
if (tl_state==null) {
tl_state = ThreadLocal.withInitial(() -> new TL_State(() -> getDigest(digestType)));
}
}
private static MessageDigest getDigest(String type) {
try {
return MessageDigest.getInstance(type);
if (digestType.equalsIgnoreCase("md5") ||digestType.equalsIgnoreCase("md2") ) {
logger.warn("Not recommended to use 'MD5 or MD2'. A stronger message digest algorithm is recommended.");
}
if (state != null) {
state.remove();
}
final Supplier<MessageDigest> mds = () -> {
try {
return MessageDigest.getInstance(digestType);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
};
state = ThreadLocal.withInitial(() -> new ThreadLocalState(mds));
} catch (Exception e) {
throw new RuntimeException("A digest of type " + type + " was not found. Select a digest type from: " +
Arrays.stream(MessageDigestAlgorithms.values()).collect(Collectors.joining(",", "[", "]")));
throw new RuntimeException("Unexpected error: ", e);
}
}
@Override
public ByteBuffer apply(long value) {
TL_State state = tl_state.get();
state.buf.putLong(0,value);
byte[] digest = state.digest.digest(state.buf.array());
return ByteBuffer.wrap(digest);
if (DigestToByteBuffer.state != null) {
final ThreadLocalState tlState = DigestToByteBuffer.state.get();
tlState.buf.putLong(0, value);
byte[] digest = tlState.digest.digest(tlState.buf.array());
return ByteBuffer.wrap(digest);
}
throw new RuntimeException("Unable to apply long value as state is not initialized.");
}
private final static class TL_State {
private static final class ThreadLocalState {
private final MessageDigest digest;
private final ByteBuffer buf = ByteBuffer.allocate(Long.BYTES);
public TL_State(Supplier<MessageDigest> mds) {
public ThreadLocalState(Supplier<MessageDigest> mds) {
digest = mds.get();
}
}

47
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ToMD5ByteBuffer.java
View File

@@ -22,48 +22,41 @@ import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.function.LongFunction;
/**
* Converts the byte image of the input long to a MD5 digest in ByteBuffer form.
* Deprecated usage due to unsafe MD5 digest.
* Replaced with DigestToByteBuffer with MD5 when absolutely needed for existing NB tests.
* However, stronger encryption algorithms (e.g. SHA-256) are recommended due to MD5's limitations.
*/
@Categories({Category.conversion,Category.premade})
@Categories({Category.conversion, Category.premade})
@ThreadSafeMapper
@Deprecated(since = "NB5", forRemoval = true)
public class ToMD5ByteBuffer implements LongFunction<ByteBuffer> {
private final MessageDigest md5;
private transient static final ThreadLocal<TLState> tl_state = ThreadLocal.withInitial(TLState::new);
@Example({"MD5ByteBuffer()","convert the a input to an md5 digest of its bytes"})
/**
* Deprecated usage due to unsafe MD5 digest.
* Use the DigestToByteBuffer with alternatives other than MD5.
*/
@Example({"MD5ByteBuffer()", "convert the a input to an md5 digest of its bytes"})
@Deprecated
public ToMD5ByteBuffer() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
throw new RuntimeException("No longer available. Please use the DigestToByteBuffer with " +
"alternatives other than MD5");
}
/**
* Deprecated usage due to unsafe MD5 digest used in this class.
* Use the DigestToByteBuffer with alternatives other than MD5.
*/
@Override
@Deprecated
public ByteBuffer apply(long value) {
TLState state = tl_state.get();
state.md5.reset();
state.bytes.putLong(0,value);
byte[] digest = md5.digest(state.bytes.array());
return ByteBuffer.wrap(digest);
throw new RuntimeException("No longer available. Please use the DigestToByteBuffer with " +
"alternatives other than MD5");
}
private final static class TLState {
public final ByteBuffer bytes = ByteBuffer.allocate(160);
public final MessageDigest md5;
public TLState() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
}

71
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/MD5HexString.java
View File

@@ -1,71 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.commons.codec.binary.Hex;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.function.LongFunction;
/**
* Computes the MD5 digest of the byte image of the input long, and
* returns it in hexadecimal String form.
*/
@Categories(Category.conversion)
@ThreadSafeMapper
public class MD5HexString implements LongFunction<String> {
private final MessageDigest md5;
private static final transient ThreadLocal<TLState> tl_state = ThreadLocal.withInitial(TLState::new);
@Example({"MD5String()","Convert a long input to an md5 digest over its bytes, and then to a hexadecimal string."})
public MD5HexString() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
@Override
public String apply(long value) {
TLState state = tl_state.get();
state.bytes.putLong(0,value);
byte[] digest = md5.digest(state.bytes.array());
String hexDigest = Hex.encodeHexString(digest);
return hexDigest;
}
private final static class TLState {
public final ByteBuffer bytes = ByteBuffer.allocate(16);
public final MessageDigest md5;
public TLState() {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
}

50
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/DigestToByteBufferTest.java
View File

@@ -23,13 +23,12 @@ import org.junit.jupiter.api.Test;
import java.nio.ByteBuffer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.assertj.core.api.Assertions.*;
public class DigestToByteBufferTest {
class DigestToByteBufferTest {
@Test
public void testWithMD5() {
void testWithMD5() {
DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.MD5);
ByteBuffer digest = d1.apply(233423L);
byte[] bytes;
@@ -43,7 +42,7 @@ public class DigestToByteBufferTest {
}
@Test
public void testWithSHA1() {
void testWithSHA1() {
DigestToByteBuffer d1 = new DigestToByteBuffer(MessageDigestAlgorithms.SHA_1);
ByteBuffer digest = d1.apply(233423L);
byte[] bytes;
@@ -57,10 +56,45 @@ public class DigestToByteBufferTest {
}
@Test
public void testInvalidName() {
DigestToByteBuffer d1 = new DigestToByteBuffer("Whoops");
void testInvalidNames() {
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> d1.apply(233423L));
.isThrownBy(() -> new DigestToByteBuffer("Whoops"));
assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> new DigestToByteBuffer(""));
}
@Test
void testInstances() {
DigestToByteBuffer sha256 = new DigestToByteBuffer("SHA-256");
DigestToByteBuffer sha512 = new DigestToByteBuffer("SHA-512");
try {
ByteBuffer sha256Digest = sha256.apply(8675309L);
ByteBuffer sha512Digest = sha512.apply(8675309L);
byte[] bytesFromSha256;
byte[] bytesFromSha512;
try {
bytesFromSha256 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b");
bytesFromSha512 = Hex.decodeHex("4b74fe6b7d11205bf8714425d30e8d89f994d7b9e381622a8f419619c156bea990708b7e8e7eea47854a81e5aa00c2a16dfa7d75e0f57961be51215a2b9f255b");
} catch (DecoderException e) {
throw new RuntimeException(e);
}
// System.out.println(Hex.encodeHexString(sha256Digest));
// System.out.println(Hex.encodeHexString(sha512Digest));
assertThat(sha256Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha256));
assertThat(sha512Digest).isEqualTo(ByteBuffer.wrap(bytesFromSha512));
} catch(Exception e) {
fail("unexpected exception found.");
}
}
}

32
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ToMD5ByteBufferTest.java
View File

@@ -1,32 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class ToMD5ByteBufferTest {
@Test
public void testMD5String() {
MD5HexString ms = new MD5HexString();
String apply = ms.apply(3L);
assertThat(apply).isEqualTo("1fb332efe1406a104b11ffa1fa04fa7a");
}
}

1
virtdata-userlibs/src/main/resources/docs-for-virtdata/bindings/funcref_conversion.md
View File

@@ -228,6 +228,7 @@ Convert the input value to a long.
## ToMD5ByteBuffer
**[ DEPRECATED ]** no longer supported use DigestToByteBuffer with stronger digest algorithm.
Converts the byte image of the input long to a MD5 digest in ByteBuffer form.