structured point clouds: implementation (wip)

aardvark-platform · Sep 21, 2023 · e1d6c92 · e1d6c92
1 parent fe6e332
commit e1d6c92
Show file tree

Hide file tree

Showing 25 changed files with 328 additions and 183 deletions.
diff --git a/src/Aardvark.Algodat.Tests/DeleteTests.cs b/src/Aardvark.Algodat.Tests/DeleteTests.cs
@@ -130,12 +130,16 @@ public void DeleteDelete()
             {
                 var q1 = new Box3d(new V3d(0.0), new V3d(0.1));
                 var a = CreateRandomPointsInUnitCube(50000, 1024);
+
+                // 1. delete a subset of points
                 var b = a.Delete(n => q1.Contains(n.BoundingBoxExactGlobal), n => !(q1.Contains(n.BoundingBoxExactGlobal) || q1.Intersects(n.BoundingBoxExactGlobal)), p => q1.Contains(p), a.Storage, CancellationToken.None);
                 b.ValidateTree();
                 Assert.IsTrue(b.Root?.Value.NoPointIn(p => q1.Contains(p)));
+
+                // 2. delete ALL remaining points
                 var c = b.Delete(n => true, n => false, p => true, a.Storage, CancellationToken.None);
-                c.ValidateTree();
-                Assert.IsTrue(c.PointCount == 0L);
+                // if all points are deleted, then 'Delete' returns null
+                Assert.Null(c);
             }
         }
 
@@ -166,10 +170,9 @@ public void DeleteAll()
         {
             var a = CreateRegularPointsInUnitCube(10, 1);
             var b = a.Delete(n => true, n => false, p => true, a.Storage, CancellationToken.None);
-            b.ValidateTree();
-            Assert.IsTrue(a.PointCount != b.PointCount);
-            Assert.IsTrue(b.PointCount == 0);
-            Assert.IsTrue(a.Id != b.Id);
+
+            // if all points are deleted, then 'Delete' returns null
+            Assert.Null(b);
         }
 
         [Test]

diff --git a/src/Aardvark.Algodat.Tests/InlinedNodeTests.cs b/src/Aardvark.Algodat.Tests/InlinedNodeTests.cs
@@ -54,7 +54,7 @@ 1 1 0 255   0
                 ;
 
             var pointset = PointCloud.Import(Data.Points.Import.Ply.Chunks(ms, 0, config.ParseConfig), config);
-            var pcl = pointset.Root.Value;
+            var _ /*pcl*/ = pointset.Root.Value;
 
             var inlineConfig = new InlineConfig(collapse: true, gzipped: true);
             var inlinedNodes = store.EnumerateOctreeInlined(key, inlineConfig);

diff --git a/src/Aardvark.Algodat.Tests/ParsingTests.cs b/src/Aardvark.Algodat.Tests/ParsingTests.cs
@@ -59,11 +59,11 @@ public void ChunkStreamAtNewlines_ThrowsIfMaxChunkSizeIsZeroOrNegative()
         [Test]
         public void ChunkStreamAtNewlines_EmptyStreamGivesEmptySequence()
         {
-            var buffer = new byte[0];
+            var buffer = Array.Empty<byte>();
             var ms = new MemoryStream(buffer);
 
             var xs = ms.ChunkStreamAtNewlines(10, 10, CancellationToken.None);
-            Assert.IsTrue(xs.Count() == 0);
+            Assert.IsTrue(!xs.Any());
         }
 
         [Test]
@@ -102,7 +102,7 @@ public void ParseBuffers_Works()
             var buffer = new byte[10] { 1, 1, 10, 2, 2, 10, 3, 3, 10, 4 };
             var ms = new MemoryStream(buffer);
 
-            static Chunk parse(byte[] _, int __, double ___) => new Chunk(new[] { V3d.Zero });
+            static Chunk parse(byte[] _, int __, double ___) => new(new[] { V3d.Zero });
 
             var xs = ms
                 .ChunkStreamAtNewlines(10, 5, CancellationToken.None)

diff --git a/src/Aardvark.Algodat.Tests/PointSetNodeTests.cs b/src/Aardvark.Algodat.Tests/PointSetNodeTests.cs
@@ -54,6 +54,10 @@ internal static PointSetNode CreateNode(Storage storage)
             var kd = ps.BuildKdTree();
             storage.Add(kdId, kd.Data);
 
+            var csId = Guid.NewGuid();
+            var cs = new[] { C4b.Yellow };
+            storage.Add(csId, cs);
+
             var data = EmptyData
                 .Add(Durable.Octree.NodeId, id)
                 .Add(Durable.Octree.Cell, cell)
@@ -62,6 +66,7 @@ internal static PointSetNode CreateNode(Storage storage)
                 .Add(Durable.Octree.PointRkdTreeFDataReference, kdId)
                 .Add(Durable.Octree.BoundingBoxExactLocal, new Box3f(ps))
                 .Add(Durable.Octree.BoundingBoxExactGlobal, ((Box3d)new Box3f(ps)) + cell.GetCenter())
+                .Add(Durable.Octree.Colors4bReference, csId)
                 ;
 
             return new PointSetNode(data, storage, writeToStore: true);

diff --git a/src/Aardvark.Algodat.Tests/PointSetTests.cs b/src/Aardvark.Algodat.Tests/PointSetTests.cs
@@ -47,7 +47,6 @@ public void CanCreateEmptyPointSet()
             Assert.IsTrue(pointset.Id == "PointSet.Empty");
             Assert.IsTrue(pointset.IsEmpty == true);
             Assert.IsTrue(pointset.PointCount == 0);
-            Assert.IsTrue(pointset.Root == null);
             Assert.IsTrue(pointset.SplitLimit == 0);
             pointset.ValidateTree();
         }
@@ -268,9 +267,9 @@ public void PointSet_PartIndexRange_Serialization()
             var json = pointset.ToJson();
             var reloaded = PointSet.Parse(json, storage);
 
-            Assert.IsTrue(pointset.Id == reloaded.Id);
-            Assert.IsTrue(pointset.SplitLimit == reloaded.SplitLimit);
-            Assert.IsTrue(pointset.Root.Value.Id == reloaded.Root.Value.Id);
+            Assert.IsTrue(pointset.Id             == reloaded.Id            );
+            Assert.IsTrue(pointset.SplitLimit     == reloaded.SplitLimit    );
+            Assert.IsTrue(pointset.Root.Value.Id  == reloaded.Root.Value.Id );
             Assert.IsTrue(pointset.PartIndexRange == reloaded.PartIndexRange);
         }
 
@@ -287,9 +286,9 @@ public void PointSet_PartIndexRange_Serialization_NoRange()
             var json = pointset.ToJson();
             var reloaded = PointSet.Parse(json, storage);
 
-            Assert.IsTrue(pointset.Id == reloaded.Id);
-            Assert.IsTrue(pointset.SplitLimit == reloaded.SplitLimit);
-            Assert.IsTrue(pointset.Root.Value.Id == reloaded.Root.Value.Id);
+            Assert.IsTrue(pointset.Id             == reloaded.Id            );
+            Assert.IsTrue(pointset.SplitLimit     == reloaded.SplitLimit    );
+            Assert.IsTrue(pointset.Root.Value.Id  == reloaded.Root.Value.Id );
             Assert.IsTrue(pointset.PartIndexRange == reloaded.PartIndexRange);
             Assert.IsTrue(reloaded.PartIndexRange.IsInvalid);
         }

diff --git a/src/Aardvark.Algodat.Tests/TestUtils.cs b/src/Aardvark.Algodat.Tests/TestUtils.cs
@@ -18,42 +18,52 @@ public static void ValidateTree(this IPointCloudNode node, int splitLimit, bool
         {
             if (node != null)
             {
-                Assert.IsTrue(node.PointCountTree > 0);
+                //Assert.IsTrue(node.PointCountTree > 0);
                 Assert.IsTrue(node.HasBoundingBoxExactGlobal || node.HasBoundingBoxExactLocal);
 
                 if (node.IsLeaf || hasLod)
                 {
-                    Assert.IsTrue(node.PointCountCell > 0);
-                    Assert.IsTrue(node.HasPositions);
-                    var ps = node.PositionsAbsolute;
-                    var realBb = node.BoundingBoxExactGlobal;
-                    var bb = realBb.EnlargedByRelativeEps(0.01);
-                    Assert.IsTrue(ps.All(p => bb.Contains(p)));
-                    Assert.IsTrue(ps.Length <= 1 || node.HasKdTree);
+                    if (node.Id != Guid.Empty)
+                    {
+                        Assert.IsTrue(node.PointCountCell > 0);
+                        Assert.IsTrue(node.HasPositions);
+                        var ps = node.PositionsAbsolute;
+                        var realBb = node.BoundingBoxExactGlobal;
+                        var bb = realBb.EnlargedByRelativeEps(0.01);
+                        Assert.IsTrue(ps.Length == 0 || ps.All(p => bb.Contains(p)));
+                        Assert.IsTrue(ps.Length <= 1 || node.HasKdTree);
+                    }
 
-                    if (node.HasNormals) Assert.IsTrue(node.Normals.Value.Length == ps.Length);
-                    if (node.HasColors) Assert.IsTrue(node.Colors.Value.Length == ps.Length);
-                    if (node.HasIntensities) Assert.IsTrue(node.Intensities.Value.Length == ps.Length);
-                    if (node.HasClassifications) Assert.IsTrue(node.Classifications.Value.Length == ps.Length);
+                    if (node.HasNormals        ) Assert.IsTrue(node.Normals        .Value.Length == node.PointCountCell);
+                    if (node.HasColors         ) Assert.IsTrue(node.Colors         .Value.Length == node.PointCountCell);
+                    if (node.HasIntensities    ) Assert.IsTrue(node.Intensities    .Value.Length == node.PointCountCell);
+                    if (node.HasClassifications) Assert.IsTrue(node.Classifications.Value.Length == node.PointCountCell);
                 }
 
                 if (node.IsLeaf)
                 {
-                    var ps = node.PositionsAbsolute;
                     Assert.IsTrue(node.PointCountCell == node.PointCountTree);
-                    Assert.IsTrue(ps.Length == node.PointCountTree);
-                    Assert.IsTrue(ps.Length <= splitLimit);
+                    if (node.Id != Guid.Empty)
+                    {
+                        var ps = node.PositionsAbsolute;
+                        Assert.IsTrue(ps.Length == node.PointCountTree);
+                        Assert.IsTrue(ps.Length <= splitLimit);
+                    }
                 }
                 else
                 {
-                    var realBb = node.BoundingBoxExactGlobal;
-                    var bb = realBb.EnlargedByRelativeEps(0.01);
-                    Assert.IsTrue(node.PointCountTree > splitLimit);
                     var nodes = node.Subnodes.Map(n => n?.Value);
-                    var nodeBB = new Box3d(nodes.Select(n => n != null ? n.BoundingBoxExactGlobal : Box3d.Invalid));
-                    Assert.IsTrue(realBb == nodeBB);
+                    Assert.IsTrue(node.PointCountTree > splitLimit);
                     Assert.IsTrue(node.PointCountTree == nodes.Sum(n => n != null ? n.PointCountTree : 0));
 
+                    if (node.Id != Guid.Empty)
+                    {
+                        var realBb = node.BoundingBoxExactGlobal;
+                        var bb = realBb.EnlargedByRelativeEps(0.01);
+                        var nodeBB = new Box3d(nodes.Select(n => n != null ? n.BoundingBoxExactGlobal : Box3d.Invalid));
+                        Assert.IsTrue(realBb == nodeBB);
+                    }
+
                     foreach (var n in nodes)
                     {
                         ValidateTree(n, splitLimit, hasLod);

diff --git a/src/Aardvark.Data.Points.Ascii/Parsing.cs b/src/Aardvark.Data.Points.Ascii/Parsing.cs
@@ -152,7 +152,7 @@ CancellationToken ct
                         var samples = optionalSamples;
                         bounds.ExtendBy(new Box3d(samples.Positions));
                         Interlocked.Add(ref sampleCount, samples.Count);
-                        var r = new Chunk(samples.Positions, samples.Colors, samples.Normals, samples.Intensities, samples.Classifications, samples.BoundingBox);
+                        var r = new Chunk(samples.Positions, samples.Colors, samples.Normals, samples.Intensities, samples.Classifications, samples.PartIndices, samples.BoundingBox);
 
                         Interlocked.Add(ref sampleCountYielded, r.Count);
                         Interlocked.Add(ref totalBytesRead, buffer.Count);

diff --git a/src/Aardvark.Data.Points.Base/Chunk.cs b/src/Aardvark.Data.Points.Base/Chunk.cs
@@ -256,6 +256,17 @@ public Chunk(
                     );
             }
 
+            #region part indices
+
+            switch (partIndices)
+            {
+                case null: break;
+                case uint: break;
+                case IList<byte>: break;
+                case IList<short>: break;
+                case IList<int>: break;
+                default: throw new Exception($"Unexpected part indices type {partIndices.GetType().FullName}. Error fc9d196d-508e-4977-8f04-2167c71e38b0.");
+            }
             IList<byte>? qs1b = null;
             if (partIndices is IList<byte> _qs1b && _qs1b.Count != positions.Count)
             {
@@ -287,6 +298,8 @@ public Chunk(
                     );
             }
 
+            #endregion
+
             if (countMismatch)
             {
                 var minCount = positions.Count;
@@ -358,6 +371,7 @@ public Chunk Union(Chunk other)
                     Append(Normals, other.Normals),
                     Append(Intensities, other.Intensities),
                     Append(Classifications, other.Classifications),
+                    partIndices: PartIndexUtils.Union(PartIndices, other.PartIndices),
                     Box.Union(BoundingBox, other.BoundingBox)
                 );
         }

diff --git a/src/Aardvark.Data.Points.Base/PartIndexUtils.cs b/src/Aardvark.Data.Points.Base/PartIndexUtils.cs
@@ -0,0 +1,74 @@
+/*
+   Aardvark Platform
+   Copyright (C) 2006-2023  Aardvark Platform Team
+   https://aardvark.graphics
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+using Aardvark.Base;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+#pragma warning disable CS1591
+
+namespace Aardvark.Data.Points;
+
+/// <summary>
+/// Utils for part indices.
+/// </summary>
+public static class PartIndexUtils
+{
+    /// <summary>
+    /// Returns union of two part indices (uint, IList of [byte|short|int]).
+    /// </summary>
+    public static object? Union(object? first, object? second)
+    {
+        checked
+        {
+            return (first, second) switch
+            {
+                (null, null) => null,
+                (object x, null) => x,
+                (null, object y) => y,
+                (uint x, uint y) => (x == y) ? x : new Range1i(new[] { (int)x, (int)y }),
+
+                (uint x, IList<byte> ys) => ((Range1i)new Range1b(ys)).ExtendedBy((int)x),
+                (uint x, IList<short> ys) => ((Range1i)new Range1s(ys)).ExtendedBy((int)x),
+                (uint x, IList<int> ys) => new Range1i(ys).ExtendedBy((int)x),
+
+                (IList<byte> xs, uint y) => ((Range1i)new Range1b(xs)).ExtendedBy((int)y),
+                (IList<short> xs, uint y) => ((Range1i)new Range1s(xs)).ExtendedBy((int)y),
+                (IList<int> xs, uint y) => new Range1i(xs).ExtendedBy((int)y),
+
+                (IList<byte> xs, IList<byte> ys) => (Range1i)new Range1b(xs.Concat(ys)),
+                (IList<byte> xs, IList<short> ys) => (Range1i)new Range1s(xs.Select(x => (short)x).Concat(ys)),
+                (IList<byte> xs, IList<int> ys) => new Range1i(xs.Select(x => (int)x).Concat(ys)),
+
+                (IList<short> xs, IList<byte> ys) => (Range1i)new Range1s(xs.Concat(ys.Select(x => (short)x))),
+                (IList<short> xs, IList<short> ys) => (Range1i)new Range1s(xs.Concat(ys)),
+                (IList<short> xs, IList<int> ys) => new Range1i(xs.Select(x => (int)x).Concat(ys)),
+
+                (IList<int> xs, IList<byte> ys) => new Range1i(xs.Concat(ys.Select(x => (int)x))),
+                (IList<int> xs, IList<short> ys) => new Range1i(xs.Concat(ys.Select(x => (int)x))),
+                (IList<int> xs, IList<int> ys) => new Range1i(xs.Concat(ys)),
+
+                _ => throw new Exception(
+                    $"Unexpected part indices types {first.GetType().FullName} and {second.GetType().FullName}. " +
+                    $"Error 2f0672f5-8c6b-400b-8172-e83a30d70c28"
+                    )
+            };
+        }
+    }
+
+}
diff --git a/src/Aardvark.Data.Points.Base/Storage.cs b/src/Aardvark.Data.Points.Base/Storage.cs
@@ -26,6 +26,21 @@ namespace Aardvark.Data.Points
     /// </summary>
     public class Storage : IDisposable
     {
+        private static readonly HashSet<Storage> s_storages = new();
+
+        /// <summary>
+        /// No storage. All functions are NOP.
+        /// </summary>
+        public static readonly Storage None = new(
+            add: (_, _, _) => { },
+            get: _ => null,
+            getSlice: (_, _, _) => null,
+            remove: _ => { },
+            dispose: () => { },
+            flush: () => { },
+            cache: null
+            );
+
         private bool m_isDisposed = false;
 
         /// <summary>add(key, value, create)</summary>
@@ -138,7 +153,5 @@ private static void Unregister(Storage storage)
                 //}
             }
         }
-
-        private static readonly HashSet<Storage> s_storages = new();
     }
 }
diff --git a/src/Aardvark.Data.Points.Ply/Aardvark.Data.Points.Ply.csproj b/src/Aardvark.Data.Points.Ply/Aardvark.Data.Points.Ply.csproj
@@ -2,7 +2,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <LangVersion>latest</LangVersion>
+    <LangVersion>10</LangVersion>
     <Nullable>enable</Nullable>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <RepositoryRoot>..\..</RepositoryRoot>

diff --git a/src/Aardvark.Data.Points.Ply/PlyImport.cs b/src/Aardvark.Data.Points.Ply/PlyImport.cs
@@ -59,7 +59,9 @@ public static IEnumerable<Chunk> Chunks(string filename, ParseConfig config)
         /// <summary>
         /// Parses PLY (.ply) file.
         /// </summary>
+#pragma warning disable IDE0060 // Remove unused parameter
         public static IEnumerable<Chunk> Chunks(this Stream stream, long streamLengthInBytes, ParseConfig config)
+#pragma warning restore IDE0060 // Remove unused parameter
             => PlyParser.Parse(stream, config.MaxChunkPointCount, config.Verbose ? (s => Report.Line(s)) : null).Chunks();
 
         /// <summary>

diff --git a/src/Aardvark.Geometry.PointSet/Import/ImportChunks.cs b/src/Aardvark.Geometry.PointSet/Import/ImportChunks.cs
@@ -171,12 +171,12 @@ Chunk map(Chunk x, CancellationToken ct)
             // create LOD data
             if (config.Verbose) Report.BeginTimed("generate lod");
             final = final.GenerateLod(config.WithRandomKey().WithProgressCallback(x => config.ProgressCallback(0.66 + x * 0.34)));
-            if (config.Storage?.GetPointCloudNode(final.Root.Value.Id) == null) throw new InvalidOperationException("Invariant 4d633e55-bf84-45d7-b9c3-c534a799242e.");
+            if (final.Root.Value != null && final.Root.Value.Id != Guid.Empty && config.Storage?.GetPointCloudNode(final.Root.Value.Id) == null) throw new InvalidOperationException("Invariant 4d633e55-bf84-45d7-b9c3-c534a799242e.");
             if (config.Verbose) Report.End();
 
             // create final point set with specified key (or random key when no key is specified)
             var key = config.Key ?? Guid.NewGuid().ToString();
-            final = new PointSet(config.Storage ?? throw new Exception($"No storage specified. Error 5b4ebfec-d418-4ddc-9c2f-646d270cf78c."), key, final.Root.Value.Id, config.OctreeSplitLimit);
+            final = new PointSet(config.Storage ?? throw new Exception($"No storage specified. Error 5b4ebfec-d418-4ddc-9c2f-646d270cf78c."), key, final.Root?.Value?.Id ?? Guid.Empty, config.OctreeSplitLimit);
             config.Storage.Add(key, final);
 
             return final;

diff --git a/src/Aardvark.Geometry.PointSet/Import/MapReduce.cs b/src/Aardvark.Geometry.PointSet/Import/MapReduce.cs
@@ -88,6 +88,9 @@ public static PointSet MapReduce(this IEnumerable<Chunk> chunks, ImportConfig co
             var parts = new HashSet<PointSet>(pointsets);
             var final = pointsets.MapReduceParallel((first, second, ct2) =>
             {
+                if (first .Root.Value == null) throw new Exception($"Expected first octree not to be null. Error c1ac4063-efaa-4c92-879c-7925509adee4.");
+                if (second.Root.Value == null) throw new Exception($"Expected second octree not to be null. Error 65289b58-2c36-46dd-91a6-81b42554d625.");
+
                 lock (parts)
                 {
                     if (!parts.Remove(first)) throw new InvalidOperationException("map reduce error");
@@ -123,6 +126,8 @@ public static PointSet MapReduce(this IEnumerable<Chunk> chunks, ImportConfig co
                     Interlocked.Increment(ref doneCount);
                 }
 
+                if (merged.Root.Value == null) throw new Exception($"Expected merged octree not to be null. Error b49cf692-2c1f-4067-b4cc-2960783fe141.");
+
                 //Console.WriteLine($"[MERGE CALLBACK][{id}] {(first.PointCount + second.PointCount) / (double)totalPointsToMerge,7:N3}");
 
                 lock (parts)