From a0da889a02b6bc715ca57537c3f5d261c69de185 Mon Sep 17 00:00:00 2001 From: comp500 Date: Mon, 23 Dec 2019 16:31:36 +0000 Subject: [PATCH] Optimise memory usage while computing Murmur2 --- .../installer/metadata/hash/Murmur2Hasher.kt | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/link/infra/packwiz/installer/metadata/hash/Murmur2Hasher.kt b/src/main/kotlin/link/infra/packwiz/installer/metadata/hash/Murmur2Hasher.kt index 1beffb2..d061acf 100644 --- a/src/main/kotlin/link/infra/packwiz/installer/metadata/hash/Murmur2Hasher.kt +++ b/src/main/kotlin/link/infra/packwiz/installer/metadata/hash/Murmur2Hasher.kt @@ -10,7 +10,7 @@ class Murmur2Hasher : IHasher { val tempBuffer = Buffer() override val hash: Hash by lazy(LazyThreadSafetyMode.NONE) { - val data = computeNormalizedArray(internalBuffer.readByteArray()) + val data = internalBuffer.readByteArray() Murmur2Hash(Murmur2Lib.hash32(data, data.size, 1)) } @@ -19,27 +19,42 @@ class Murmur2Hasher : IHasher { val out = delegate.read(tempBuffer, byteCount) if (out > -1) { sink.write(tempBuffer.clone(), out) - internalBuffer.write(tempBuffer, out) + computeNormalizedBufferFaster(tempBuffer, internalBuffer) } return out } // Credit to https://github.com/modmuss50/CAV2/blob/master/murmur.go - private fun computeNormalizedArray(input: ByteArray): ByteArray { - val output = ByteArray(input.size) +// private fun computeNormalizedArray(input: ByteArray): ByteArray { +// val output = ByteArray(input.size) +// var index = 0 +// for (b in input) { +// when (b) { +// 9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {} +// else -> { +// output[index] = b +// index++ +// } +// } +// } +// val outputTrimmed = ByteArray(index) +// System.arraycopy(output, 0, outputTrimmed, 0, index) +// return outputTrimmed +// } + + private fun computeNormalizedBufferFaster(input: Buffer, output: Buffer) { var index = 0 - for (b in input) { + val arr = input.readByteArray() + for (b in arr) { when (b) { 9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {} else -> { - output[index] = b + arr[index] = b index++ } } } - val outputTrimmed = ByteArray(index) - System.arraycopy(output, 0, outputTrimmed, 0, index) - return outputTrimmed + output.write(arr, 0, index) } }