Skip to content

Commit

Permalink
Use BLAKE3 instead of SHA256 for cryptographic hash computation
Browse files Browse the repository at this point in the history
Closes #1088
  • Loading branch information
rui314 committed Aug 16, 2023
1 parent 3833a4a commit 7f7a744
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 115 deletions.
20 changes: 13 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,19 @@ else()
target_link_libraries(mold PRIVATE zlibstatic)
endif()

# Find zstd compression library. Just like zlib, if libzstd.so is not
# found, we compile a bundled one and statically-link it to mold.
# Find BLAKE3 cryptographic hash library. Just like zlib, if libblkae3.so
# is not found, we compile a bundled one and statically-link it to mold.
find_package(BLAKE3 QUIET)
if(BLAKE3_FOUND)
target_link_libraries(mold PRIVATE BLAKE3::blake3)
else()
add_subdirectory(third-party/blake3/c EXCLUDE_FROM_ALL)
target_link_libraries(mold PRIVATE blake3)
target_include_directories(mold PUBLIC third-party/blake3/c)
endif()

# Find zstd compression library. If libzstd.so is not found, we compile a
# bundled one and statically-link it to mold.
include(CheckIncludeFile)
check_include_file(zstd.h HAVE_ZSTD_H)

Expand Down Expand Up @@ -237,11 +248,6 @@ if(NOT APPLE AND NOT WIN32)
target_sources(mold-wrapper PRIVATE elf/mold-wrapper.c)
endif()

if(NOT APPLE AND NOT WIN32 AND NOT MOLD_MOSTLY_STATIC)
find_package(OpenSSL REQUIRED COMPONENTS Crypto)
target_link_libraries(mold PRIVATE OpenSSL::Crypto)
endif()

# If atomics doesn't work by default, add -latomic.
# We need the flag on riscv, armv6 and m68k.
include(CheckCXXSourceCompiles)
Expand Down
81 changes: 0 additions & 81 deletions common/sha.h

This file was deleted.

30 changes: 17 additions & 13 deletions elf/icf.cc
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
// conditions.

#include "mold.h"
#include "../common/sha.h"
#include "blake3.h"

#include <array>
#include <cstdio>
Expand Down Expand Up @@ -132,9 +132,11 @@ static bool is_eligible(Context<E> &ctx, InputSection<E> &isec) {
!is_init && !is_fini && !is_enumerable && !is_addr_taken;
}

static Digest digest_final(SHA256Hash &sha) {
u8 buf[SHA256_SIZE];
sha.finish(buf);
static Digest digest_final(blake3_hasher *hasher) {
assert(HASH_SIZE <= BLAKE3_OUT_LEN);

u8 buf[BLAKE3_OUT_LEN];
blake3_hasher_finalize(hasher, buf, BLAKE3_OUT_LEN);

Digest digest;
memcpy(digest.data(), buf, HASH_SIZE);
Expand Down Expand Up @@ -234,15 +236,16 @@ static void merge_leaf_nodes(Context<E> &ctx) {

template <typename E>
static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
SHA256Hash sha;
blake3_hasher hasher;
blake3_hasher_init(&hasher);

auto hash = [&](auto val) {
sha.update((u8 *)&val, sizeof(val));
blake3_hasher_update(&hasher, (u8 *)&val, sizeof(val));
};

auto hash_string = [&](std::string_view str) {
hash(str.size());
sha.update((u8 *)str.data(), str.size());
blake3_hasher_update(&hasher, (u8 *)str.data(), str.size());
};

auto hash_symbol = [&](Symbol<E> &sym) {
Expand Down Expand Up @@ -298,7 +301,7 @@ static Digest compute_digest(Context<E> &ctx, InputSection<E> &isec) {
hash_symbol(*isec.file.symbols[rel.r_sym]);
}

return digest_final(sha);
return digest_final(&hasher);
}

template <typename E>
Expand Down Expand Up @@ -411,16 +414,17 @@ static i64 propagate(std::span<std::vector<Digest>> digests,
if (converged.get(i))
return;

SHA256Hash sha;
sha.update(digests[2][i].data(), HASH_SIZE);
blake3_hasher hasher;
blake3_hasher_init(&hasher);
blake3_hasher_update(&hasher, digests[2][i].data(), HASH_SIZE);

i64 begin = edge_indices[i];
i64 end = (i + 1 == num_digests) ? edges.size() : edge_indices[i + 1];

for (i64 j : edges.subspan(begin, end - begin))
sha.update(digests[slot][j].data(), HASH_SIZE);
blake3_hasher_update(&hasher, digests[slot][j].data(), HASH_SIZE);

digests[!slot][i] = digest_final(sha);
digests[!slot][i] = digest_final(&hasher);

if (digests[slot][i] == digests[!slot][i]) {
// This node has converged. Skip further iterations as it will
Expand Down Expand Up @@ -563,7 +567,7 @@ void icf_sections(Context<E> &ctx) {
}
}

// Group sections by SHA digest.
// Group sections by BLAKE3 digest.
{
Timer t(ctx, "group");

Expand Down
2 changes: 0 additions & 2 deletions elf/mold.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@

namespace mold::elf {

static constexpr i32 SHA256_SIZE = 32;

template <typename E> class InputFile;
template <typename E> class InputSection;
template <typename E> class MergedSection;
Expand Down
29 changes: 17 additions & 12 deletions elf/output-chunks.cc
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#include "mold.h"
#include "../common/sha.h"
#include "blake3.h"

#include <cctype>
#include <set>
Expand Down Expand Up @@ -2519,19 +2519,28 @@ void BuildIdSection<E>::copy_buf(Context<E> &ctx) {
memcpy(base + 3, "GNU", 4); // Name string
}

// BLAKE3 is a cryptographic hash function just like SHA256.
// We use it instead of SHA256 because it's faster.
static void blake3_hash(u8 *buf, i64 size, u8 *out) {
blake3_hasher hasher;
blake3_hasher_init(&hasher);
blake3_hasher_update(&hasher, buf, size);
blake3_hasher_finalize(&hasher, out, BLAKE3_OUT_LEN);
}

template <typename E>
static void compute_sha256(Context<E> &ctx, i64 offset) {
static void compute_blake3(Context<E> &ctx, i64 offset) {
u8 *buf = ctx.buf;
i64 filesize = ctx.output_file->filesize;

i64 shard_size = 4096 * 1024;
i64 num_shards = align_to(filesize, shard_size) / shard_size;
std::vector<u8> shards(num_shards * SHA256_SIZE);
std::vector<u8> shards(num_shards * BLAKE3_OUT_LEN);

tbb::parallel_for((i64)0, num_shards, [&](i64 i) {
u8 *begin = buf + shard_size * i;
u8 *end = (i == num_shards - 1) ? buf + filesize : begin + shard_size;
sha256_hash(begin, end - begin, shards.data() + i * SHA256_SIZE);
blake3_hash(begin, end - begin, shards.data() + i * BLAKE3_OUT_LEN);

#ifndef _WIN32
// We call munmap early for each chunk so that the last munmap
Expand All @@ -2543,10 +2552,10 @@ static void compute_sha256(Context<E> &ctx, i64 offset) {
#endif
});

assert(ctx.arg.build_id.size() <= SHA256_SIZE);
assert(ctx.arg.build_id.size() <= BLAKE3_OUT_LEN);

u8 digest[SHA256_SIZE];
sha256_hash(shards.data(), shards.size(), digest);
u8 digest[BLAKE3_OUT_LEN];
blake3_hash(shards.data(), shards.size(), digest);
memcpy(buf + offset, digest, ctx.arg.build_id.size());

#ifndef _WIN32
Expand All @@ -2567,11 +2576,7 @@ void BuildIdSection<E>::write_buildid(Context<E> &ctx) {
ctx.arg.build_id.value);
return;
case BuildId::HASH:
// Modern x86 processors have purpose-built instructions to accelerate
// SHA256 computation, and SHA256 outperforms MD5 on such computers.
// So, we always compute SHA256 and truncate it if smaller digest was
// requested.
compute_sha256(ctx, this->shdr.sh_offset + HEADER_SIZE);
compute_blake3(ctx, this->shdr.sh_offset + HEADER_SIZE);
return;
case BuildId::UUID: {
std::array<u8, 16> uuid = get_uuid_v4();
Expand Down

0 comments on commit 7f7a744

Please sign in to comment.