fixed feature descriptors for float/other types

.vscode/settings.json

@@ -1,4 +1,8 @@
 {
+    "cmake.configureSettings": {
+        "CMAKE_TOOLCHAIN_FILE": "${workspaceRoot}/.vcpkg/vcpkg/scripts/buildsystems/vcpkg.cmake",
+        "VCPKG_BUILD": "ON",
+    },
     "files.exclude": {
         "**/.git": true,
         "**/.svn": true,
@@ -10,7 +14,6 @@
         "packages/": true,
         "paket-files/": true,
         ".vscode/*.log": true,
-        ".vcpkg": true,
         ".paket": true,
         ".ionide": true,
     }

RELEASE_NOTES.md

@@ -1,3 +1,6 @@
+### 1.1.11
+* fixed feature descriptors for float/other types
+
 ### 1.1.10
 * added SQPNP solver kind 
 

src/MiniCV/OpenCV.fs

@@ -36,7 +36,7 @@ type RecoverPoseConfig  =
     end
 
 [<Struct>]
-type KeyPoint(index : int, pos : V2d, size : float, angle : float, response : float, octave : int, descriptorDimension : int, descriptors : uint8[]) =
+type KeyPoint(index : int, pos : V2d, size : float, angle : float, response : float, octave : int, descriptorDimension : int, descriptorType : Type, descriptors : System.Array) =
     member x.Index = index
     member x.Position = pos
     member x.Size = size
@@ -45,16 +45,17 @@ type KeyPoint(index : int, pos : V2d, size : float, angle : float, response : fl
     member x.Octave = int
 
     member x.Descriptor =
-        let fst = int64 descriptorDimension * int64 index
-        Vector<uint8>(descriptors, fst, int64 descriptorDimension, 1L)
+        let res = System.Array.CreateInstance (descriptorType, descriptorDimension)
+        System.Array.Copy(descriptors, descriptorDimension * index, res, 0, descriptorDimension)
+        res
 
     override x.ToString() =
         sprintf "{ pos = %A; size = %A; angle = %A }" pos size angle
 
 type ImageFeatures =
     {
         points                  : KeyPoint[]
-        descriptors             : uint8[]
+        descriptors             : System.Array
         descriptorDimension     : int
     }
 
@@ -169,26 +170,6 @@ module ImageFeatures =
         else
             current
 
-    let matches (l : ImageFeatures) (r : ImageFeatures) =
-        let rf = r.points |> Array.map (fun pt -> pt.Descriptor)
-        let perm = Array.init rf.Length id
-        pointKdTree perm rf 0 rf.Length 0
-
-        let sub (l : byte) (r : byte) =
-            (float l - float r) / 255.0
-
-        l.points |> Array.choosei (fun li lf ->
-            let closest = nearest ClosestPoints.empty 2 Double.PositiveInfinity sub perm rf 0 rf.Length 0 lf.Descriptor
-            let closest = 
-                closest.matches 
-                    |> MapExt.toSeq 
-                    |> Seq.collect (fun (d,s) -> s |> Seq.map (fun ri -> li, ri, d)) 
-                    |> Seq.atMost 2
-                    |> Seq.toList
-            match closest with
-                | [] -> None
-                | c -> Some c
-        )
 
 
 
@@ -208,11 +189,30 @@ module OpenCV =
 
         end
 
+// #define CV_8U   0
+// #define CV_8S   1
+// #define CV_16U  2
+// #define CV_16S  3
+// #define CV_32S  4
+// #define CV_32F  5
+// #define CV_64F  6
+// #define CV_16F  7
+    type ElementType =
+        | UInt8 = 0
+        | Int8 = 1
+        | UInt16 = 2
+        | Int16 = 3
+        | Int32 = 4
+        | Float = 5
+        | Double = 6
+        | Half = 7
+
     [<StructLayout(LayoutKind.Sequential)>]
     type DetectorResult =
         struct
             val mutable public PointCount : int
             val mutable public DescriptorEntries : int
+            val mutable public DescriptorElementType : ElementType
             val mutable public Points : nativeptr<KeyPoint2d>
             val mutable public Descriptors : nativeptr<uint8>
         end
@@ -339,24 +339,61 @@ module OpenCV =
             if v.PointCount = 0 then
                 {
                     points                  = [||]
-                    descriptors             = [||]
+                    descriptors             = Array.empty<byte>
                     descriptorDimension     = 61
                 }
             else
                 let pts : KeyPoint2d[] = Array.zeroCreate v.PointCount
-                let descriptors : uint8[] = Array.zeroCreate v.DescriptorEntries
-
                 copy v.Points pts pts.Length
-                copy v.Descriptors descriptors descriptors.Length
+                let mutable typ = null
+                let descriptors =
+                    match v.DescriptorElementType with
+                    | ElementType.Float ->
+                        typ <- typeof<float32>
+                        let descriptors : float32[] = Array.zeroCreate v.DescriptorEntries
+                        copy (NativePtr.cast v.Descriptors) descriptors descriptors.Length
+                        descriptors :> System.Array
+                    | ElementType.Double ->
+                        typ <- typeof<float>
+                        let descriptors : float[] = Array.zeroCreate v.DescriptorEntries
+                        copy (NativePtr.cast v.Descriptors) descriptors descriptors.Length
+                        descriptors :> System.Array
+                    | ElementType.Int8 ->
+                        typ <- typeof<int8>
+                        let descriptors : int8[] = Array.zeroCreate v.DescriptorEntries
+                        copy (NativePtr.cast v.Descriptors) descriptors descriptors.Length
+                        descriptors :> System.Array
+                    | ElementType.UInt8 ->
+                        typ <- typeof<uint8>
+                        let descriptors : uint8[] = Array.zeroCreate v.DescriptorEntries
+                        copy v.Descriptors descriptors descriptors.Length
+                        descriptors :> System.Array
+                    | ElementType.UInt16 ->
+                        typ <- typeof<uint16>
+                        let descriptors : uint16[] = Array.zeroCreate v.DescriptorEntries
+                        copy (NativePtr.cast v.Descriptors) descriptors descriptors.Length
+                        descriptors :> System.Array
+                    | ElementType.Int16 ->
+                        typ <- typeof<int16>
+                        let descriptors : int16[] = Array.zeroCreate v.DescriptorEntries
+                        copy (NativePtr.cast v.Descriptors) descriptors descriptors.Length
+                        descriptors :> System.Array
+                    | ElementType.Int32 ->
+                        typ <- typeof<int>
+                        let descriptors : int[] = Array.zeroCreate v.DescriptorEntries
+                        copy (NativePtr.cast v.Descriptors) descriptors descriptors.Length
+                        descriptors :> System.Array
+                    | t ->
+                        failwithf "bad descriptor type: %A" t
 
                 let dim = descriptors.Length / pts.Length
-
-                let points = pts |> Array.mapi (fun i pt -> KeyPoint(i, V2d pt.pt, float pt.size, float pt.angle, float pt.response, pt.octave, dim, descriptors))
+                let points = pts |> Array.mapi (fun i pt -> KeyPoint(i, V2d pt.pt, float pt.size, float pt.angle, float pt.response, pt.octave, dim, typ, descriptors))
                 {
                     points                  = points
                     descriptors             = descriptors
                     descriptorDimension     = dim
                 }
+
         finally
             Native.cvFreeFeatures_ ptr
             gc.Free()

src/MiniCVNative/MiniCVNative.cpp

@@ -197,7 +197,7 @@ DllExport(bool) cvRecoverPoses(const RecoverPoseConfig* config, const int N, con
 DllExport(int) cvRecoverPose(const RecoverPoseConfig* config, const int N, const Point2d* pa, const Point2d* pb, Matx33d& rMat, Vec3d& tVec, uint8_t* ms) {
 	vector<Point2d> a(pa, pa + N);
 	vector<Point2d> b(pb, pb + N);
-
+	
 	Mat E;
 	Mat mask;
 
@@ -219,7 +219,7 @@ void cvCornerSubPix(const cv::Mat img, const vector<Vec2d> corners) {
 }
 
 DllExport(DetectorResult*) cvDetectFeatures(char* data, int width, int height, int channels, int mode = FEATURE_MODE_AKAZE) {
-
+	
 	int fmt;
 	int convert;
 	switch (channels)
@@ -247,6 +247,7 @@ DllExport(DetectorResult*) cvDetectFeatures(char* data, int width, int height, i
 
 	Mat input(height, width, fmt, (void*)data);
 	cv::Ptr<cv::FeatureDetector> detector;
+
 	switch (mode)
 	{
 	case FEATURE_MODE_AKAZE:
@@ -288,9 +289,9 @@ DllExport(DetectorResult*) cvDetectFeatures(char* data, int width, int height, i
 	}
 	else
 	{
-
+		auto elemSize = descriptorsM.elemSize();
 		auto points1 = new KeyPoint2d[points.size()];
-		auto descriptors1 = new uchar[descriptorCount];
+		auto descriptors1 = new uchar[descriptorCount * elemSize];
 
 		for (int i = 0; i < points.size(); i++)
 		{
@@ -301,16 +302,15 @@ DllExport(DetectorResult*) cvDetectFeatures(char* data, int width, int height, i
 			points1[i].response = points[i].response;
 			points1[i].size = points[i].size;
 		}
-		if (descriptorsM.elemSize() != 1)
-		{
-			printf("BAD DESCRIPTORS\n");
-		}
-		memcpy(descriptors1, descriptorsM.data, descriptorCount);
+
+
+		memcpy(descriptors1, descriptorsM.data, descriptorCount * elemSize);
 
 		detector->clear();
 		img.release();
 
 		auto res = new DetectorResult();
+		res->DescriptorElementType = descriptorsM.type();
 		res->PointCount = (int)points.size();
 		res->DescriptorEntries = (int)descriptorCount;
 		res->Points = points1;

src/MiniCVNative/MiniCVNative.h

@@ -23,6 +23,7 @@ typedef struct KeyPoint2d_ {
 typedef struct DetectorResult_ {
 	int PointCount;
 	int DescriptorEntries;
+	int DescriptorElementType;
 	KeyPoint2d* Points;
 	uchar* Descriptors;
 } DetectorResult;
@@ -44,4 +45,4 @@ typedef struct ArucoMarkerInfo_ {
 #define FEATURE_MODE_AKAZE 1
 #define FEATURE_MODE_ORB 2
 #define FEATURE_MODE_BRISK 3
-#define FEATURE_MODE_SIFT 4
+#define FEATURE_MODE_SIFT 4

src/Test/Program.fs

@@ -63,12 +63,21 @@ let main argv =
         use ms = new MemoryStream(data)
         PixImageSharp.Create(ms).ToPixImage<byte>(Col.Format.RGBA)
 
+    printfn "SIFT"
+    let ftrs = MiniCV.OpenCV.detectFeatures MiniCV.OpenCV.DetectorMode.Sift img
+    printfn "  count: %A" ftrs.points.Length
+    if ftrs.points.Length > 0 then 
+        let p = ftrs.points.[0]
+        printfn "  dim:           %A" ftrs.descriptorDimension
+        printfn "  descriptor[0]: %0A" p.Descriptor
+        printfn "  point:         %A" p
+
+    printfn "AKAZE"
     let ftrs = MiniCV.OpenCV.detectFeatures MiniCV.OpenCV.DetectorMode.Akaze img
-    printfn "%A" ftrs.points.Length
+    printfn "  count: %A" ftrs.points.Length
     if ftrs.points.Length > 0 then 
-        let d = ftrs.descriptors.[0]
         let p = ftrs.points.[0]
-        printfn "%A" ftrs.descriptorDimension
-        printfn "%A" d
-        printfn "%A" p
+        printfn "  dim:           %A" ftrs.descriptorDimension
+        printfn "  descriptor[0]: %0A" p.Descriptor
+        printfn "  point:         %A" p
     0 // return an integer exit code