Release 10.008

CHANGELOG.md

@@ -1,11 +1,14 @@
+## 10.008
+ * various optimizations running duckdb and reading very large datasets.
+
 ## 10.006
  * fix to quartile-X, median.
  * added local-time->microseconds for duckdb integration.
- 
+
 ## 10.005
  * fix to round
  * small perf enhancement to native-buffer/reduce
- 
+
 ## 10.004
  * Optimizations around a performance case using duckdb.
  * tech.resource now uses arraylists for stack tracking and reverses them just before release.

deps.edn

@@ -18,7 +18,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "cnuernber"
                :artifact-id "dtype-next"
-               :version "10.007"
+               :version "10.008"
                :name "DType Next"
                :description "High Performance Clojure Primitives"
                :metadata {:doc/format :markdown}

src/tech/v3/datatype/datetime/operations.clj

@@ -11,14 +11,15 @@
             [tech.v3.datatype.binary-op :as bin-op]
             [tech.v3.datatype.argops :as argops]
             [tech.v3.datatype.errors :as errors]
+            [ham-fisted.api :as hamf]
             [clojure.set :as set])
   (:import [tech.v3.datatype DoubleReader LongReader ObjectReader Buffer
             BinaryOperator
             BinaryOperators$LongBinaryOperator]
            [java.time.temporal ChronoUnit Temporal ChronoField
             WeekFields TemporalAmount TemporalField
             TemporalAccessor]
-           [java.util Locale Iterator]))
+           [java.util Locale Iterator Set]))
 
 
 (set! *warn-on-reflection* true)
@@ -28,10 +29,12 @@
 (def datetime-datatypes (set/union dt-packing/datatypes
                                    dt-base/datatypes))
 
+(def ^Set datetime-datatypes-java (hamf/java-hashset datetime-datatypes))
+
 
 (defn datetime-datatype?
   [datatype]
-  (boolean (datetime-datatypes datatype)))
+  (.contains datetime-datatypes-java datatype))
 
 
 (def millisecond-datatypes #{:duration :packed-duration})

src/tech/v3/datatype/native_buffer.clj

@@ -22,7 +22,8 @@
            [clojure.lang RT IObj Counted Indexed IFn
             IFn$LL IFn$LD IFn$LO IFn$LLO IFn$LDO IFn$LOO IFn$OLO
             IFn$LLL IFn$LLD IFn$LLO IFn$LLLO IFn$LLDO IFn$LLOO]
-           [ham_fisted Casts Transformables IMutList ChunkedList Reductions]))
+           [ham_fisted Casts Transformables IMutList ChunkedList Reductions
+            ArrayHelpers]))
 
 (set! *warn-on-reflection* true)
 (set! *unchecked-math* :warn-on-boxed)
@@ -691,15 +692,49 @@
     (dtype-proto/->native-buffer item)))
 
 
+(defn native-buffer->string
+  "Convert a :int8 native buffer to a java string."
+  ([^NativeBuffer nbuf ^long off ^long len]
+   (when-not (identical? (.elemwise-datatype nbuf) :int8)
+     (throw (RuntimeException. (str "Native buffer is not convertible to string - " (.datatype nbuf)))))
+   (if (== 0 len)
+     ""
+     (let [us (unsafe)
+           final-len (+ off len)
+           _ (when-not (<= final-len (.n-elems nbuf))
+               (throw (RuntimeException. (str "Requested end point: " final-len
+                                              " is past native buffer len" (.n-elems nbuf)))))
+           bdata (byte-array len)
+           addr (+ (.address nbuf) off)]
+       (loop [idx 0]
+         (when (< idx len)
+           (ArrayHelpers/aset bdata idx (.getByte us (+ addr idx)))
+           (recur (unchecked-inc idx))))
+       (String. bdata))))
+  ([^NativeBuffer nbuf]
+   (when-not (identical? (.elemwise-datatype nbuf) :int8)
+     (throw (RuntimeException. "Native buffer is not convertible to string.")))
+   (let [len (.n-elems nbuf)]
+     (if (== 0 len)
+       ""
+       (let [bdata (byte-array len)
+             addr (.address nbuf)
+             us (unsafe)]
+         (loop [idx 0]
+           (when (< idx len)
+             (ArrayHelpers/aset bdata idx (.getByte us (+ addr idx)))))
+         (String. bdata))))))
+
+
 (defn native-buffer-byte-len
   "Get the length, in bytes, of a native buffer."
   ^long [^NativeBuffer nb]
-  (let [original-size (.n-elems nb)]
-    (if (or (identical? (.datatype nb) :int8)
-            (identical? (.datatype nb) :uint8))
+  (let [original-size (.n-elems nb)
+        nbdt (.elemwise-datatype nb)]
+    (if (or (identical? nbdt :int8)
+            (identical? nbdt :uint8))
       original-size
-      (* original-size (casting/numeric-byte-width
-                        (dtype-proto/elemwise-datatype nb))))))
+      (* original-size (casting/numeric-byte-width nbdt)))))
 
 
 (defn set-native-datatype
@@ -885,13 +920,15 @@
   will be linked to the native buffer such that gc-obj will not be garbage
   collected before native buffer is garbage collected."
   (^NativeBuffer [address n-bytes datatype endianness gc-obj]
-   (errors/when-not-error
-    (or (not= 0 (long address))
-        (== 0 (long n-bytes)))
-    "Attempt to wrap 0 as an address for a native buffer")
-   (let [byte-width (casting/numeric-byte-width datatype)]
-     (NativeBuffer. address (quot (long n-bytes) byte-width)
-                    datatype endianness #{:gc} nil nil gc-obj)))
+   (let [address (long address)
+         n-bytes (long n-bytes)]
+     (errors/when-not-error
+      (or (not (== 0 address))
+          (== 0 n-bytes))
+      "Attempt to wrap 0 as an address for a native buffer")
+     (let [byte-width (casting/numeric-byte-width datatype)]
+       (NativeBuffer. address (quot n-bytes byte-width)
+                      datatype endianness #{:gc} nil nil gc-obj))))
   (^NativeBuffer [address n-bytes gc-obj]
    (wrap-address address n-bytes :int8 (dtype-proto/platform-endianness)
                  gc-obj)))

src/tech/v3/datatype/protocols.clj

@@ -239,12 +239,11 @@ Only arraybuffers and native buffers need implement this pathway."))
   [item]
   (or item :little-endian))
 
+(defonce platform-endianness* (delay (if (= (ByteOrder/nativeOrder) ByteOrder/LITTLE_ENDIAN)
+                                       :little-endian
+                                       :big-endian)))
 
-(defn platform-endianness
-  []
-  (if (= (ByteOrder/nativeOrder) ByteOrder/LITTLE_ENDIAN)
-    :little-endian
-    :big-endian))
+(defn platform-endianness [] @platform-endianness*)
 
 
 (defprotocol PToBitmap