Refactor query and search for better performance (#674)

* Refactor query and search for better performance

* FIXUP

* FIXUP format

* FIXUP format

* FIXUP format

---------

Co-authored-by: Jonas Östlund <ostlund.jobtech@gmail.com>

src/datahike/array.cljc

@@ -48,6 +48,25 @@
                    (n/-order-on-edn-types b))))
             (raw-array-compare a b))))
 
+#?(:clj (defn string-from-bytes
+          "Represents a byte array as a string. Two byte arrays are said to be equal iff their corresponding values after applying this function are equal. That way, we rely on the equality and hash code implementations of the String class to compare byte arrays."
+          [x]
+          (let [n (alength x)
+                dst (char-array n)]
+            (dotimes [i n]
+              (aset dst i (char (aget x i))))
+            (String. dst))))
+
+(defrecord WrappedBytes [string-repr])
+
+(defn wrap-comparable
+  "This functions is such that `(a= x y)` is equivalent to `(= (wrap-comparable x) (wrap-comparable y))`. This lets us also use these semantics in hash-sets or as keys in maps."
+  [x]
+  (if (bytes? x)
+    (WrappedBytes. #?(:clj (string-from-bytes x)
+                      :cljs x))
+    x))
+
 (defn a=
   "Extension of Clojure's equality to things we also want to treat like values,
   e.g. certain array types."

src/datahike/db.cljc

@@ -204,6 +204,10 @@
   (-> ((contextual-search-fn context) db pattern)
       (post-process-datoms db context)))
 
+(defn contextual-batch-search [db pattern-mask batch-fn context]
+  (-> ((contextual-search-fn context) db pattern-mask batch-fn)
+      (post-process-datoms db context)))
+
 (defn contextual-datoms [db index-type cs context]
   (-> (case (dbi/context-temporal? context)
         true (dbu/temporal-datoms db index-type cs)
@@ -316,6 +320,8 @@
   (-search-context [db] dbi/base-context)
   (-search [db pattern context]
            (contextual-search db pattern context))
+  (-batch-search [db pattern-mask batch-fn context]
+                 (contextual-batch-search db pattern-mask batch-fn context))
 
   dbi/IIndexAccess
   (-datoms [db index-type cs context]
@@ -395,6 +401,8 @@
                          (filter (.-pred db))))
   (-search [db pattern context]
            (dbi/-search unfiltered-db pattern context))
+  (-batch-search [db pattern-mask batch-fn context]
+                 (dbi/-batch-search unfiltered-db pattern-mask batch-fn context))
 
   dbi/IIndexAccess
   (-datoms [db index cs context]
@@ -467,6 +475,8 @@
                        dbi/context-with-history))
   (-search [db pattern context]
            (dbi/-search origin-db pattern context))
+  (-batch-search [db pattern-mask batch-fn context]
+                 (dbi/-batch-search origin-db pattern-mask batch-fn context))
 
   dbi/IIndexAccess
   (-datoms [db index-type cs context] (dbi/-datoms origin-db index-type cs context))
@@ -530,6 +540,8 @@
                          (as-of-pred time-point)))
   (-search [db pattern context]
            (dbi/-search origin-db pattern context))
+  (-batch-search [db pattern batch-fn context]
+                 (dbi/-batch-search origin-db pattern batch-fn context))
 
   dbi/IIndexAccess
   (-datoms [db index-type cs context]
@@ -595,6 +607,8 @@
                          (since-pred time-point)))
   (-search [db pattern context]
            (dbi/-search origin-db pattern context))
+  (-batch-search [db pattern batch-fn context]
+                 (dbi/-batch-search origin-db pattern batch-fn context))
 
   dbi/IIndexAccess
   (dbi/-datoms [db index-type cs context]

src/datahike/db/interface.cljc

@@ -70,11 +70,19 @@
 
 (defprotocol ISearch
   (-search-context [data])
-  (-search [data pattern context]))
+  (-search [data pattern context])
+  (-batch-search [data pattern-mask batch-fn context]))
 
 (defn search [data pattern]
   (-search data pattern (-search-context data)))
 
+(defn batch-search [data pattern-mask batch-fn final-xform]
+  (-batch-search data
+                 pattern-mask
+                 batch-fn
+                 (context-with-xform-after (-search-context data)
+                                           final-xform)))
+
 (defprotocol IIndexAccess
   (-datoms [db index components context])
   (-seek-datoms [db index components context])