Skip to content

Commit

Permalink
Optimise memory usage while computing Murmur2
Browse files Browse the repository at this point in the history
  • Loading branch information
comp500 committed Dec 23, 2019
1 parent 432bb4e commit a0da889
Showing 1 changed file with 24 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ class Murmur2Hasher : IHasher {
val tempBuffer = Buffer()

override val hash: Hash by lazy(LazyThreadSafetyMode.NONE) {
val data = computeNormalizedArray(internalBuffer.readByteArray())
val data = internalBuffer.readByteArray()
Murmur2Hash(Murmur2Lib.hash32(data, data.size, 1))
}

Expand All @@ -19,27 +19,42 @@ class Murmur2Hasher : IHasher {
val out = delegate.read(tempBuffer, byteCount)
if (out > -1) {
sink.write(tempBuffer.clone(), out)
internalBuffer.write(tempBuffer, out)
computeNormalizedBufferFaster(tempBuffer, internalBuffer)
}
return out
}

// Credit to https://github.com/modmuss50/CAV2/blob/master/murmur.go
private fun computeNormalizedArray(input: ByteArray): ByteArray {
val output = ByteArray(input.size)
// private fun computeNormalizedArray(input: ByteArray): ByteArray {
// val output = ByteArray(input.size)
// var index = 0
// for (b in input) {
// when (b) {
// 9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {}
// else -> {
// output[index] = b
// index++
// }
// }
// }
// val outputTrimmed = ByteArray(index)
// System.arraycopy(output, 0, outputTrimmed, 0, index)
// return outputTrimmed
// }

private fun computeNormalizedBufferFaster(input: Buffer, output: Buffer) {
var index = 0
for (b in input) {
val arr = input.readByteArray()
for (b in arr) {
when (b) {
9.toByte(), 10.toByte(), 13.toByte(), 32.toByte() -> {}
else -> {
output[index] = b
arr[index] = b
index++
}
}
}
val outputTrimmed = ByteArray(index)
System.arraycopy(output, 0, outputTrimmed, 0, index)
return outputTrimmed
output.write(arr, 0, index)
}

}
Expand Down

0 comments on commit a0da889

Please sign in to comment.