Speeding up Array Consistency Check by using sorted int-int AVL Tree

README.md

@@ -692,7 +692,7 @@ Events are processed at over 220K/second except for:
  - wildcard matches, which are processed at over 170K/second.
  - anything-but matches, which are processed at over 150K/second.
  - numeric matches, which are processed at over 120K/second.
- - complex array matches, which are processed at over 2.5K/second.
+ - complex array matches, which are processed at over 5K/second.
 
 ### Suggestions for better performance
 

pom.xml

@@ -32,6 +32,7 @@
   <properties>
     <jdk.version>1.8</jdk.version>
     <jackson.version>2.14.0</jackson.version>
+    <fastutil.version>8.5.9</fastutil.version>
     <jsr.version>3.0.2</jsr.version>
     <junit.version>4.13.2</junit.version>
     <compiler.plugin.version>3.10.1</compiler.plugin.version>
@@ -98,6 +99,11 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>it.unimi.dsi</groupId>
+      <artifactId>fastutil-core</artifactId>
+      <version>${fastutil.version}</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -147,7 +153,6 @@
         <version>${checkstyle.plugin.version}</version>
         <configuration>
           <configLocation>${basedir}/config/checkstyle/checkstyle.xml</configLocation>
-          <encoding>${project.build.sourceEncoding}</encoding>
           <consoleOutput>true</consoleOutput>
           <failsOnError>false</failsOnError>
           <linkXRef>false</linkXRef>

src/main/software/amazon/event/ruler/ArrayMembership.java

@@ -1,28 +1,39 @@
 package software.amazon.event.ruler;
 
+import it.unimi.dsi.fastutil.ints.Int2IntAVLTreeMap;
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+
 /**
  * Represents which JSON arrays within an Event structure a particular field appears within, and at which position.
  *  The arrays are identified using integers.
  */
 class ArrayMembership {
-    private static final IntIntMap EMPTY = new IntIntMap();
+    private static final Int2IntAVLTreeMap EMPTY = createNewIntIntMap();
+    public static final int NO_VALUE = -1; // Keys and values may only be positive.
 
-    private IntIntMap membership;
+    private final Int2IntAVLTreeMap membership;
 
     ArrayMembership() {
-        membership = new IntIntMap();
+        this.membership = createNewIntIntMap();
     }
 
     ArrayMembership(final ArrayMembership membership) {
         if (membership.size() == 0) {
             this.membership = EMPTY;
         } else {
-            this.membership = (IntIntMap) membership.membership.clone();
+            this.membership = createNewIntIntMap();
+            this.membership.putAll(membership.membership);
         }
     }
 
+    private static Int2IntAVLTreeMap createNewIntIntMap() {
+        final Int2IntAVLTreeMap membership = new Int2IntAVLTreeMap();
+        membership.defaultReturnValue(NO_VALUE);
+        return membership;
+    }
+
     void putMembership(int array, int index) {
-        if (index == IntIntMap.NO_VALUE) {
+        if (index == NO_VALUE) {
             membership.remove(array);
         } else {
             membership.put(array, index);
@@ -44,8 +55,8 @@ private int size() {
     // for debugging
     public String toString() {
         StringBuilder sb = new StringBuilder();
-        for (IntIntMap.Entry entry : membership.entries()) {
-            sb.append(entry.getKey()).append('[').append(entry.getValue()).append("] ");
+        for (Int2IntMap.Entry entry : membership.int2IntEntrySet()) {
+            sb.append(entry.getIntKey()).append('[').append(entry.getIntValue()).append("] ");
         }
         return sb.toString();
     }
@@ -70,12 +81,12 @@ static ArrayMembership checkArrayConsistency(final ArrayMembership membershipSoF
 
         // any change will come from memberships in the new field we're investigating. For each of its memberships
         ArrayMembership newMembership = null;
-        for (IntIntMap.Entry arrayEntry : fieldMembership.membership.entries()) {
-            final int array = arrayEntry.getKey();
-            final int indexInThisArrayOfThisField = arrayEntry.getValue();
+        for (Int2IntMap.Entry arrayEntry : fieldMembership.membership.int2IntEntrySet()) {
+            final int array = arrayEntry.getIntKey();
+            final int indexInThisArrayOfThisField = arrayEntry.getIntValue();
             final int indexInThisArrayPreviouslyAppearingInMatch = membershipSoFar.getMembership(array);
 
-            if (indexInThisArrayPreviouslyAppearingInMatch == IntIntMap.NO_VALUE) {
+            if (indexInThisArrayPreviouslyAppearingInMatch == NO_VALUE) {
 
                 // if there's no membership so far, this is an acceptable delta. Update the new memberships, first
                 //  creating it if necessary