Skip to content

Commit

Permalink
[e57] fix parsing of classifications that would be encoded with 0 bit…
Browse files Browse the repository at this point in the history 
…s per value (see e57 spec 9.7.4.2 (3))
  • Loading branch information
@stefanmaierhofer
stefanmaierhofer committed Aug 20, 2024
1 parent 342f6da commit d8d16ab
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 20 deletions.
17 changes: 9 additions & 8 deletions src/Aardvark.Algodat.Tests/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

#pragma warning disable IDE0051 // Remove unused private members
#pragma warning disable IDE0059 // Unnecessary assignment of a value
#pragma warning disable IDE0130 // Namespace does not match folder structure

#if !NETCOREAPP
using System.ComponentModel;
Expand Down Expand Up @@ -2731,7 +2732,7 @@ static Task Parts_Test_20231006()
}
static async Task Parts_Test_20231006_Merge()
{
V3d[] randomPoints(int n, Box3d bb)
static V3d[] randomPoints(int n, Box3d bb)
{
var size = bb.Size;
var ps = new V3d[n];
Expand Down Expand Up @@ -2759,19 +2760,19 @@ V3d[] randomPoints(int n, Box3d bb)
var root = ps.Root.Value;
}

#pragma warning disable CS1998
//#pragma warning disable CS1998

public static async Task Main(string[] _)
{
await Task.CompletedTask; // avoid warning if no async methods are called here ...

Test_Import_Regression();
//Test_Import_Regression();

//await CreateStore(
// @"W:\Datasets\Vgm\Data\2024-04-04_bugreport\Bestand.e57",
// @"T:\tmp\issue72_Bestand.e57",
// minDist: 0.005
// );
await CreateStore(
@"E:\Villa Vaduz gesamt.e57",
@"T:\tmp\Villa Vaduz gesamt.e57",
minDist: 0.005
);

//{
// var chunks = E57.Chunks(@"W:\Datasets\Vgm\Data\2024-04-30_bugreport\F_240205.e57", ParseConfig.Default);
Expand Down
70 changes: 61 additions & 9 deletions src/Aardvark.Data.E57/ASTM_E57.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2023. Aardvark Platform Team. http://github.com/aardvark-platform.
Copyright (C) 2006-2024. Aardvark Platform Team. http://github.com/aardvark-platform.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
Expand All @@ -11,6 +11,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using Aardvark.Base;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
Expand All @@ -21,7 +22,6 @@ You should have received a copy of the GNU Affero General Public License
using System.Runtime.CompilerServices;
using System.Text;
using System.Xml.Linq;
using Aardvark.Base;

namespace Aardvark.Data.E57
{
Expand Down Expand Up @@ -522,16 +522,25 @@ bool verbose
Report.Line($"[E57CompressedVector] #bytestreams = {ByteStreamsCount,10}");
}

var recordsLeftToConsumePerByteStream = new long[ByteStreamsCount].Set(RecordCount);
var bitpackerPerByteStream = Prototype.Children.Map(x => new BitPacker(((IBitPack)x).NumberOfBitsForBitPack));

// [Spec 9.7.4.2]
// (3) If N = 0, a bytestream shall not be used to encode that
// element in the binary section.
var ignoreByteStream = bitpackerPerByteStream.Map(x => x.BitsPerValue == 0);

var recordsLeftToConsumePerByteStream = new long[ByteStreamsCount].Set(RecordCount);
for (var i = 0; i < ByteStreamsCount; i++) if (ignoreByteStream[i]) recordsLeftToConsumePerByteStream[i] = 0L;

var bytesLeftToConsume = compressedVectorHeader.SectionLength - 32;

bool has(PointPropertySemantics sem) => sem2idx.ContainsKey(sem);

if (compressedVectorHeader.DataStartOffset.Value == 0) throw new Exception($"Unexpected compressedVectorHeader.DataStartOffset (0).");
if (compressedVectorHeader.IndexStartOffset.Value != 0) throw new Exception($"Unexpected compressedVectorHeader.IndexStartOffset ({compressedVectorHeader.IndexStartOffset})");

var data = new ValueBufferSet(sem2idx.Keys);
var actualKeys = sem2idx.Keys.Where(k => !ignoreByteStream[sem2idx[k]]).ToArray();
var data = new ValueBufferSet(actualKeys);

var offset = (E57LogicalOffset)compressedVectorHeader.DataStartOffset;
var verboseDetail = false; // verbose
Expand Down Expand Up @@ -566,7 +575,9 @@ bool verbose
bytestreamBuffers[i] = ReadLogicalBytes(m_stream, offset, count);
offset += count;

buffers[i] = UnpackByteStream(bytestreamBuffers[i], bitpackerPerByteStream[i], (IBitPack)Prototype.Children[i]);
var bitpacker = (IBitPack)Prototype.Children[i];
if (ignoreByteStream[i]) continue; // [Spec 9.7.4.2 (3)]
buffers[i] = UnpackByteStream(bytestreamBuffers[i], bitpackerPerByteStream[i], bitpacker);

recordsLeftToConsumePerByteStream[i] -= buffers[i].Length;
}
Expand All @@ -580,7 +591,9 @@ bool verbose

foreach (var kv in sem2idx)
{
var (sem, raw) = (kv.Key, buffers[kv.Value]);
var i = kv.Value;
if (ignoreByteStream[i]) continue; // [Spec 9.7.4.2 (3)]
var (sem, raw) = (kv.Key, buffers[i]);
switch (sem, raw)
{
case (PointPropertySemantics.CartesianX, double[] xs) : data.Append(sem, xs); break;
Expand Down Expand Up @@ -712,7 +725,10 @@ int[] remapWithLimits(float[] xs)
}

if (data.CountMin >= maxChunkPointCount)
yield return data.Consume(data.CountMin);
{
var partialResult = data.Consume(data.CountMin);
yield return partialResult;
}
}

// move to next packet
Expand Down Expand Up @@ -742,7 +758,10 @@ int[] remapWithLimits(float[] xs)
}

if (data.CountMin > 0)
yield return data.Consume(data.CountMin);
{
var partialResult = data.Consume(data.CountMin);
yield return partialResult;
}


#region Helpers
Expand Down Expand Up @@ -1248,8 +1267,10 @@ ImmutableHashSet<PointPropertySemantics> exclude
{
var filteredSem2Index = Sem2Index.Where(kv => !exclude.Contains(kv.Key)).ToImmutableDictionary();
var chunks = Points.ReadDataFull(maxChunkPointCount, filteredSem2Index, IntensityLimits, verbose);
foreach (var chunk in chunks)
foreach (var _chunk in chunks)
{
var chunk = _chunk;

V3d[] ps;
if (chunk.ContainsKey(PointPropertySemantics.CartesianX))
{
Expand Down Expand Up @@ -1296,6 +1317,37 @@ ImmutableHashSet<PointPropertySemantics> exclude
ps = ps.Map(p => Pose.Rotation.Transform(p) + Pose.Translation);
}

// add properties that were skipped according to [Spec 9.7.4.2 (3)]
foreach (var kv in Sem2Index)
{
var semantic = kv.Key;
var i = kv.Value;
var prototype = Points.Prototype.Children[i];

if (prototype is E57Integer x0 && x0.NumberOfBitsForBitPack == 0)
{
if (semantic == PointPropertySemantics.Classification)
{
if (x0.Value >= int.MinValue && x0.Value <= int.MaxValue)
{
chunk = chunk.Add(PointPropertySemantics.Classification, new int[ps.Length].Set((int)x0.Value));
}
else
{
throw new Exception($"Classification value {x0.Value} is outside of supported range [{int.MinValue}, {int.MaxValue}]. Error 95d79844-c3d0-46c3-bea9-3ce3307e9f02.");
}
}
else
{
throw new Exception($"Semantic {semantic} with 0 bits per value not supported. Error eef7daca-7f90-4f8a-a714-b1d17b18b25f.");
}
}
else if (prototype is E57ScaledInteger x1 && x1.NumberOfBitsForBitPack == 0)
{
throw new Exception($"Semantic {semantic} with 0 bits per value not supported. Error 9f1f2529-25c5-4bd6-a2f2-527aa0c7e369.");
}
}

yield return (ps, chunk);
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/Aardvark.Data.E57/BitPack.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2023. Aardvark Platform Team. http://github.com/aardvark-platform.
Copyright (C) 2006-2024. Aardvark Platform Team. http://github.com/aardvark-platform.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
Expand Down
2 changes: 1 addition & 1 deletion src/Aardvark.Data.E57/ImportE57.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright (C) 2006-2023. Aardvark Platform Team. http://github.com/aardvark-platform.
Copyright (C) 2006-2024. Aardvark Platform Team. http://github.com/aardvark-platform.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
Expand Down
15 changes: 14 additions & 1 deletion src/Aardvark.Data.E57/Utils.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,17 @@
using Aardvark.Base;
/*
Copyright (C) 2006-2024. Aardvark Platform Team. http://github.com/aardvark-platform.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using Aardvark.Base;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
Expand Down

0 comments on commit d8d16ab

Please sign in to comment.