address review feedback

configs/blabber.toml

@@ -54,8 +54,9 @@ start = 1
 topics = 1
 topic_len = 1
 message_len = 64
-# specify an approximate compression ratio for the message payload
-compression_ratio = 1.0
+# optionally, specify an approximate compression ratio for the message payload.
+# Defaults to 1.0 meaning the message is high-entropy and not compressible.
+# compression_ratio = 1.0
 weight = 1
 # the total number of clients that will subscribe to this set of topics
 subscriber_poolsize = 100

configs/kafka.toml

@@ -84,7 +84,8 @@ topic_names = ["hello", "world"]
 partitions = 10
 # the value length, in bytes
 message_len = 512
-# specify an approximate compression ratio for the message payload
-compression_ration = 1.0
+# optionally, specify an approximate compression ratio for the message payload.
+# Defaults to 1.0 meaning the message is high-entropy and not compressible.
+compression_ratio = 1.0
 # the key length, in bytes
 key_len = 8

configs/memcached.toml

@@ -63,7 +63,8 @@ nkeys = 1_000_000
 vlen = 128
 # use random bytes for the values
 vkind = "bytes"
-# specify an approximate compression ratio for the value
+# optionally, specify an approximate compression ratio for the value payload.
+# Defaults to 1.0 meaning the message is high-entropy and not compressible.
 compression_ratio = 1.0
 # optionally: specify a TTL for the keys, by default there is no expiration
 # ttl = "15m"

configs/momento.toml

@@ -73,7 +73,8 @@ klen = 32
 nkeys = 1_000_000
 # sets the value length, in bytes
 vlen = 128
-# specify an approximate compression ratio for the value payload
+# optionally, specify an approximate compression ratio for the value payload.
+# Defaults to 1.0 meaning the message is high-entropy and not compressible.
 compression_ratio = 1.0
 # use random bytes for the values
 vkind = "bytes"

configs/momento_pubsub.toml

@@ -86,5 +86,7 @@ topics = 1
 topic_len = 32
 # sets the value length, in bytes
 message_len = 128
-# specify an approximate compression ratio for the message payload
+# optionally, specify an approximate compression ratio for the message payload.
+# Defaults to 1.0 meaning the message is high-entropy and not compressible.
 compression_ratio = 1.0
+

configs/redis.toml

@@ -69,6 +69,9 @@ nkeys = 1_000_000
 vlen = 128
 # use random bytes for the values
 vkind = "bytes"
+# optionally, specify an approximate compression ratio for the value payload.
+# Defaults to 1.0 meaning the message is high-entropy and not compressible.
+compression_ratio = 1.0
 # optionally: specify a TTL for the keys, by default there is no expiration
 # ttl = "15m"
 # controls what commands will be used in this keyspace

configs/segcache.toml

@@ -86,7 +86,8 @@ nkeys = 1_000_000
 vlen = 128
 # use random bytes for the values
 vkind = "bytes"
-# specify an approximate compression ratio for the value payload
+# optionally, specify an approximate compression ratio for the value payload.
+# Defaults to 1.0 meaning the message is high-entropy and not compressible.
 compression_ratio = 1.0
 # controls what commands will be used in this keyspace
 commands = [

configs/smoketest.toml

@@ -2,8 +2,8 @@
 
 [general]
 protocol = "memcache"
-interval = 60
-duration = 300
+interval = 1
+duration = 60
 metrics_output = "rpcperf.parquet"
 metrics_format = "parquet"
 admin = "127.0.0.1:9090"

src/workload/mod.rs

@@ -404,40 +404,7 @@ impl Topics {
             // this all bytes will be random
             topics.message_len()
         } else {
-            // we need to approximate the number of random bytes to send, we do
-            // this iteratively assuming gzip compression.
-
-            // doesn't matter what seed we use here
-            let mut rng = Xoshiro512PlusPlus::from_seed(config.general().initial_seed());
-
-            // message buffer
-            let mut m = vec![0; topics.message_len()];
-
-            let mut best = 0;
-
-            for idx in 0..m.len() {
-                // zero all bytes
-                for b in &mut m {
-                    *b = 0
-                }
-
-                // fill first N bytes with pseudorandom data
-                rng.fill_bytes(&mut m[0..idx]);
-
-                let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
-                let _ = encoder.write_all(&m);
-                let compressed = encoder.finish().unwrap();
-
-                let ratio = m.len() as f64 / compressed.len() as f64;
-
-                if ratio < topics.compression_ratio() {
-                    break;
-                }
-
-                best = idx;
-            }
-
-            best
+            estimate_random_bytes_needed(topics.message_len(), topics.compression_ratio())
         };
 
         // ntopics must be >= 1
@@ -555,46 +522,15 @@ impl Distribution {
 
 impl Keyspace {
     pub fn new(config: &Config, keyspace: &config::Keyspace) -> Self {
+        let vlen = keyspace.vlen().unwrap_or(0);
+
         let value_random_bytes = if keyspace.compression_ratio() <= 1.0 || keyspace.vlen().is_none()
         {
             // this indicates the message should not be compressible, to achieve
             // this all bytes will be random
-            keyspace.vlen().unwrap_or(0)
+            vlen
         } else {
-            // we need to approximate the number of random bytes to send, we do
-            // this iteratively assuming gzip compression.
-
-            // doesn't matter what seed we use here
-            let mut rng = Xoshiro512PlusPlus::from_seed(config.general().initial_seed());
-
-            // message buffer
-            let mut m = vec![0; keyspace.vlen().unwrap_or(0)];
-
-            let mut best = 0;
-
-            for idx in 0..m.len() {
-                // zero all bytes
-                for b in &mut m {
-                    *b = 0
-                }
-
-                // fill first N bytes with pseudorandom data
-                rng.fill_bytes(&mut m[0..idx]);
-
-                let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
-                let _ = encoder.write_all(&m);
-                let compressed = encoder.finish().unwrap();
-
-                let ratio = m.len() as f64 / compressed.len() as f64;
-
-                if ratio < keyspace.compression_ratio() {
-                    break;
-                }
-
-                best = idx;
-            }
-
-            best
+            estimate_random_bytes_needed(vlen, keyspace.compression_ratio())
         };
 
         // nkeys must be >= 1
@@ -895,3 +831,40 @@ impl Ratelimit {
         limit
     }
 }
+
+fn estimate_random_bytes_needed(length: usize, compression_ratio: f64) -> usize {
+    // we need to approximate the number of random bytes to send, we do
+    // this iteratively assuming gzip compression.
+
+    // doesn't matter what seed we use here
+    let mut rng = Xoshiro512PlusPlus::seed_from_u64(0);
+
+    // message buffer
+    let mut m = vec![0; length];
+
+    let mut best = 0;
+
+    for idx in 0..m.len() {
+        // zero all bytes
+        for b in &mut m {
+            *b = 0
+        }
+
+        // fill first N bytes with pseudorandom data
+        rng.fill_bytes(&mut m[0..idx]);
+
+        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+        let _ = encoder.write_all(&m);
+        let compressed = encoder.finish().unwrap();
+
+        let ratio = m.len() as f64 / compressed.len() as f64;
+
+        if ratio < compression_ratio {
+            break;
+        }
+
+        best = idx;
+    }
+
+    best
+}