diff --git a/external/untwine/api/QgisUntwine_win.cpp b/external/untwine/api/QgisUntwine_win.cpp index 79ab868b09a3..d837a35be1ca 100644 --- a/external/untwine/api/QgisUntwine_win.cpp +++ b/external/untwine/api/QgisUntwine_win.cpp @@ -28,7 +28,7 @@ bool QgisUntwine::start(Options& options) cmdline += "--" + op.first + " \"" + op.second + "\" "; PROCESS_INFORMATION processInfo; - STARTUPINFOA startupInfo; + STARTUPINFO startupInfo; ZeroMemory(&processInfo, sizeof(PROCESS_INFORMATION)); ZeroMemory(&startupInfo, sizeof(STARTUPINFO)); diff --git a/external/untwine/bu/BuPyramid.cpp b/external/untwine/bu/BuPyramid.cpp index 72ef674aed83..d41028d59e37 100644 --- a/external/untwine/bu/BuPyramid.cpp +++ b/external/untwine/bu/BuPyramid.cpp @@ -14,6 +14,8 @@ #include "../untwine/Common.hpp" #include "../untwine/ProgressWriter.hpp" +#include //untwine/os + namespace untwine { namespace bu @@ -53,7 +55,7 @@ void BuPyramid::getInputFiles() return std::make_pair(true, VoxelKey(x, y, z, level)); }; - std::vector files = directoryList(m_b.opts.tempDir); + std::vector files = os::directoryList(m_b.opts.tempDir); VoxelKey root; for (std::string file : files) diff --git a/external/untwine/bu/CopcSupport.cpp b/external/untwine/bu/CopcSupport.cpp index 46d7d61e9659..5ddc2e8f51ce 100644 --- a/external/untwine/bu/CopcSupport.cpp +++ b/external/untwine/bu/CopcSupport.cpp @@ -25,6 +25,8 @@ #include "../untwine/Common.hpp" #include "../untwine/FileDimInfo.hpp" +#include // untwine/os + namespace untwine { namespace bu @@ -39,7 +41,7 @@ CopcSupport::CopcSupport(const BaseInfo& b) : m_b(b), else m_wktVlr = b.srs.getWKT(); - m_f.open(toNative(b.opts.outputName), std::ios::out | std::ios::binary); + m_f.open(os::toNative(b.opts.outputName), std::ios::out | std::ios::binary); m_header.global_encoding = m_b.globalEncoding; m_header.global_encoding |= (1 << 4); // Set for WKT diff --git a/external/untwine/bu/FileInfo.hpp b/external/untwine/bu/FileInfo.hpp index 68ec6c3d3a4d..44b23683d5fc 100644 --- a/external/untwine/bu/FileInfo.hpp +++ b/external/untwine/bu/FileInfo.hpp @@ -14,7 +14,7 @@ #include -#include "../untwine/Common.hpp" +#include // untwine/os namespace untwine { @@ -44,16 +44,16 @@ class FileInfo char *address() const { return reinterpret_cast(m_ctx.addr()); } - MapContext context() const + untwine::os::MapContext context() const { return m_ctx; } - void setContext(const MapContext& ctx) + void setContext(const untwine::os::MapContext& ctx) { m_ctx = ctx; } private: std::string m_filename; int m_numPoints; int m_start; - MapContext m_ctx; + untwine::os::MapContext m_ctx; }; using FileInfoList = std::list; diff --git a/external/untwine/bu/PointAccessor.hpp b/external/untwine/bu/PointAccessor.hpp index ee6f6b51671b..4ec814438941 100644 --- a/external/untwine/bu/PointAccessor.hpp +++ b/external/untwine/bu/PointAccessor.hpp @@ -15,6 +15,8 @@ #include "../untwine/Common.hpp" #include "../untwine/Point.hpp" +#include // untwine/os + #include "FileInfo.hpp" namespace untwine @@ -31,13 +33,13 @@ class PointAccessor ~PointAccessor() { for (FileInfo *fi : m_fileInfos) - unmapFile(fi->context()); + os::unmapFile(fi->context()); } void read(FileInfo& fi) { std::string filename = m_b.opts.tempDir + "/" + fi.filename(); - auto ctx = mapFile(filename, true, 0, fi.numPoints() * m_b.pointSize); + auto ctx = os::mapFile(filename, true, 0, fi.numPoints() * m_b.pointSize); if (ctx.m_addr == nullptr) throw FatalError(filename + ": " + ctx.m_error); fi.setContext(ctx); diff --git a/external/untwine/bu/Processor.cpp b/external/untwine/bu/Processor.cpp index 7c5eef06cb81..c7e7d73868d1 100644 --- a/external/untwine/bu/Processor.cpp +++ b/external/untwine/bu/Processor.cpp @@ -28,6 +28,8 @@ #include "Processor.hpp" #include "PyramidManager.hpp" +#include // untwine/os + namespace untwine { namespace bu @@ -50,14 +52,12 @@ void Processor::run() } catch (const std::exception& ex) { - std::cerr << "Exception: " << ex.what() << "\n"; m_manager.queueWithError(m_vi.octant(), ex.what()); return; } catch (...) { std::string msg = std::string("Unexpected error processing ") + m_vi.key().toString() + "."; - std::cerr << "Exception: " << msg << "\n"; m_manager.queueWithError(m_vi.octant(), msg); return; } @@ -245,7 +245,7 @@ void Processor::writeBinOutput(Index& index) // pass. std::string filename = m_vi.key().toString() + ".bin"; std::string fullFilename = m_b.opts.tempDir + "/" + filename; - std::ofstream out(toNative(fullFilename), std::ios::binary | std::ios::trunc); + std::ofstream out(os::toNative(fullFilename), std::ios::binary | std::ios::trunc); if (!out) throw FatalError("Couldn't open '" + fullFilename + "' for output."); for (size_t i = 0; i < index.size(); ++i) @@ -482,13 +482,13 @@ void Processor::createChunk(const VoxelKey& key, pdal::PointViewPtr view) uint64_t location = m_manager.newChunk(key, chunk.size(), (uint32_t)view->size()); - std::ofstream out(toNative(m_b.opts.outputName), + std::ofstream out(os::toNative(m_b.opts.outputName), std::ios::out | std::ios::in | std::ios::binary); out.seekp(std::ofstream::pos_type(location)); out.write(reinterpret_cast(chunk.data()), chunk.size()); out.close(); if (!out) - throw FatalError("Failure writing to '" + m_b.opts.outputName + "'."); + throw FatalError("Failure writing to file '" + m_b.opts.outputName + "'."); } void Processor::fillPointBuf(pdal::PointRef& point, std::vector& buf, diff --git a/external/untwine/bu/PyramidManager.cpp b/external/untwine/bu/PyramidManager.cpp index 36748c17848c..54345be7864c 100644 --- a/external/untwine/bu/PyramidManager.cpp +++ b/external/untwine/bu/PyramidManager.cpp @@ -79,7 +79,8 @@ void PyramidManager::run() if (m_error.size()) { - std::cerr << "Exception: " << m_error << "\n"; + lock.unlock(); + m_pool.join(); throw FatalError(m_error); } } diff --git a/external/untwine/epf/Epf.cpp b/external/untwine/epf/Epf.cpp index 3d98a8eb32a9..057f661450e1 100644 --- a/external/untwine/epf/Epf.cpp +++ b/external/untwine/epf/Epf.cpp @@ -33,6 +33,7 @@ #include #include +#include // untwine/os namespace untwine { @@ -265,7 +266,7 @@ void Epf::createFileInfos(const StringList& input, std::vector& fileIn { if (FileUtils::isDirectory(filename)) { - StringList dirfiles = directoryList(filename); + StringList dirfiles = os::directoryList(filename); filenames.insert(filenames.end(), dirfiles.begin(), dirfiles.end()); } else @@ -281,12 +282,18 @@ void Epf::createFileInfos(const StringList& input, std::vector& fileIn std::string driver = factory.inferReaderDriver(filename); if (driver.empty()) throw FatalError("Can't infer reader for '" + filename + "'."); + // Use LAS reader for COPC files. + if (driver == "readers.copc") + driver = "readers.las"; Stage *s = factory.createStage(driver); pdal::Options opts; opts.add("filename", filename); if (driver == "readers.las") + { opts.add("nosrs", m_b.opts.no_srs); + opts.add("use_eb_vlr", "true"); + } s->setOptions(opts); FileInfo fi; @@ -437,8 +444,8 @@ std::vector Epf::processLas(pdal::LasReader& r, FileInfo fi) fi.numPoints = h.pointCount(); m_b.scale[0] = (std::max)(m_b.scale[0], h.scaleX()); - m_b.scale[1] = (std::max)(m_b.scale[0], h.scaleY()); - m_b.scale[2] = (std::max)(m_b.scale[0], h.scaleZ()); + m_b.scale[1] = (std::max)(m_b.scale[1], h.scaleY()); + m_b.scale[2] = (std::max)(m_b.scale[2], h.scaleZ()); fi.offsets[0] = h.offsetX(); fi.offsets[1] = h.offsetY(); diff --git a/external/untwine/epf/FileProcessor.cpp b/external/untwine/epf/FileProcessor.cpp index e9c977c24f43..b6454b9eadc4 100644 --- a/external/untwine/epf/FileProcessor.cpp +++ b/external/untwine/epf/FileProcessor.cpp @@ -134,11 +134,13 @@ void FileProcessor::run() opts.add("filename", m_fi.filename); opts.add("count", m_fi.numPoints); if (m_fi.driver == "readers.las") + { opts.add("nosrs", m_fi.no_srs); + opts.add("use_eb_vlr", "true"); #ifdef PDAL_LAS_START - if (m_fi.driver == "readers.las") opts.add("start", m_fi.start); #endif + } pdal::StageFactory factory; pdal::Stage *s = factory.createStage(m_fi.driver); diff --git a/external/untwine/epf/Reprocessor.cpp b/external/untwine/epf/Reprocessor.cpp index 9ea9b5b33859..c49d0cc616ed 100644 --- a/external/untwine/epf/Reprocessor.cpp +++ b/external/untwine/epf/Reprocessor.cpp @@ -19,6 +19,8 @@ #include "Reprocessor.hpp" #include "../untwine/Common.hpp" +#include // untwine/os + namespace untwine { namespace epf @@ -51,7 +53,7 @@ Reprocessor::Reprocessor(const VoxelKey& k, int numPoints, int pointSize, void Reprocessor::run() { - auto ctx = mapFile(m_filename, true, 0, m_fileSize); + auto ctx = os::mapFile(m_filename, true, 0, m_fileSize); if (ctx.addr() == nullptr) { std::cerr << "FATAL: " + m_filename + ": " + ctx.what(); @@ -69,7 +71,7 @@ void Reprocessor::run() cell->advance(); pos += m_pointSize; } - unmapFile(ctx); + os::unmapFile(ctx); pdal::FileUtils::deleteFile(m_filename); } diff --git a/external/untwine/epf/Writer.cpp b/external/untwine/epf/Writer.cpp index ef1b7fd43507..3e255841bf9c 100644 --- a/external/untwine/epf/Writer.cpp +++ b/external/untwine/epf/Writer.cpp @@ -20,6 +20,8 @@ #include "../untwine/Common.hpp" #include "../untwine/VoxelKey.hpp" +#include // untwine/os + using namespace pdal; namespace untwine @@ -144,14 +146,14 @@ void Writer::run() // Open the file. Write the data. Stick the buffer back on the cache. // Remove the key from the active key list. - std::ofstream out(toNative(path(wd.key)), std::ios::app | std::ios::binary); + std::ofstream out(os::toNative(path(wd.key)), std::ios::app | std::ios::binary); out.write(reinterpret_cast(wd.data->data()), wd.dataSize); out.close(); std::lock_guard lock(m_mutex); if (!out) { - m_pool.setError("Failure writing to '" + path(wd.key) + "'."); + m_pool.setError("Failure writing to file '" + path(wd.key) + "'."); m_stop = true; } else diff --git a/external/untwine/untwine/Common.cpp b/external/untwine/untwine/Common.cpp deleted file mode 100644 index 5acf98e5ee8f..000000000000 --- a/external/untwine/untwine/Common.cpp +++ /dev/null @@ -1,179 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2020, Hobu Inc. - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following - * conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided - * with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY - * OF SUCH DAMAGE. - ****************************************************************************/ - -#include "Common.hpp" - -#ifdef _WIN32 -#include -#endif - -#include -#include - -namespace untwine -{ - -// This is here so that things will work with older version of PDAL. - -MapContext mapFile(const std::string& filename, bool readOnly, size_t pos, size_t size) -{ - MapContext ctx; - - if (!readOnly) - { - ctx.m_error = "readOnly must be true."; - return ctx; - } - -#if defined(_MSC_VER) // Windows - ctx.m_fd = ::_wopen(toNative(filename).data(), readOnly ? _O_RDONLY : _O_RDWR); -#elif defined(_WIN32) // MinGW - ctx.m_fd = ::_open(toNative(filename).data(), readOnly ? _O_RDONLY : _O_RDWR); -#else // Everyone else - ctx.m_fd = ::open(filename.data(), readOnly ? O_RDONLY : O_RDWR); -#endif - - if (ctx.m_fd == -1) - { - ctx.m_error = "Mapped file couldn't be opened."; - return ctx; - } - ctx.m_size = size; - -#ifndef _WIN32 - ctx.m_addr = ::mmap(0, size, PROT_READ, MAP_SHARED, ctx.m_fd, (off_t)pos); - if (ctx.m_addr == MAP_FAILED) - { - ctx.m_addr = nullptr; - ctx.m_error = "Couldn't map file"; - } -#else - ctx.m_handle = CreateFileMapping((HANDLE)_get_osfhandle(ctx.m_fd), - NULL, PAGE_READONLY, 0, 0, NULL); - uint32_t low = pos & 0xFFFFFFFF; - uint32_t high = (pos >> 8); - ctx.m_addr = MapViewOfFile(ctx.m_handle, FILE_MAP_READ, high, low, - ctx.m_size); - if (ctx.m_addr == nullptr) - ctx.m_error = "Couldn't map file"; -#endif - - return ctx; -} - -MapContext unmapFile(MapContext ctx) -{ -#ifndef _WIN32 - if (::munmap(ctx.m_addr, ctx.m_size) == -1) - ctx.m_error = "Couldn't unmap file."; - else - { - ctx.m_addr = nullptr; - ctx.m_size = 0; - ctx.m_error = ""; - } - ::close(ctx.m_fd); -#else - if (UnmapViewOfFile(ctx.m_addr) == 0) - ctx.m_error = "Couldn't unmap file."; - else - { - ctx.m_addr = nullptr; - ctx.m_size = 0; - ctx.m_error = ""; - } - CloseHandle(ctx.m_handle); - ::_close(ctx.m_fd); -#endif - return ctx; -} - -// PDAL's directoryList had a bug, so we've imported a working -// version here so that we can still use older PDAL releases. - -#ifndef __APPLE_CC__ -std::vector directoryList(const std::string& dir) -{ - namespace fs = std::filesystem; - - std::vector files; - - try - { - fs::directory_iterator it(untwine::toNative(dir)); - fs::directory_iterator end; - while (it != end) - { - files.push_back(it->path().string()); - it++; - } - } - catch (fs::filesystem_error&) - { - files.clear(); - } - return files; -} -#else - -#include - -// Provide simple opendir/readdir solution for OSX because directory_iterator is -// not available until OSX 10.15 -std::vector directoryList(const std::string& dir) -{ - std::vector files; - - DIR *dpdf = opendir(dir.c_str()); - if (dpdf) - { - while (true) - { - struct dirent *epdf = readdir(dpdf); - if (!epdf) - break; - - std::string name = untwine::fromNative(epdf->d_name); - // Skip paths - if (!pdal::Utils::iequals(name, ".") && - !pdal::Utils::iequals(name, "..")) - { - // we expect the path + name - files.push_back(dir + "/" + untwine::fromNative(epdf->d_name)); - } - } - closedir(dpdf); - } - return files; -} -#endif - -} // namespace untwine - diff --git a/external/untwine/untwine/Common.hpp b/external/untwine/untwine/Common.hpp index 0fb7776a9796..0dabc061c26d 100644 --- a/external/untwine/untwine/Common.hpp +++ b/external/untwine/untwine/Common.hpp @@ -1,13 +1,5 @@ #pragma once -#ifdef _WIN32 -#include -#include -#else -#include -#include -#endif - #include #include #include @@ -18,6 +10,7 @@ #include #include +#include "FatalError.hpp" #include "FileDimInfo.hpp" namespace untwine @@ -29,13 +22,6 @@ const int CellCount = 128; using PointCount = uint64_t; using StringList = std::vector; -class FatalError : public std::runtime_error -{ -public: - inline FatalError(std::string const& msg) : std::runtime_error(msg) - {} -}; - struct Options { std::string outputName; @@ -126,7 +112,7 @@ inline bool isExtraDim(const std::string& name) using namespace pdal; using D = Dimension::Id; - static const std::array lasDims + static const std::array lasDims { D::X, D::Y, @@ -134,7 +120,6 @@ inline bool isExtraDim(const std::string& name) D::Intensity, D::ReturnNumber, D::NumberOfReturns, - D::ReturnNumber, D::Classification, D::UserData, D::ScanAngleRank, @@ -153,97 +138,4 @@ inline bool isExtraDim(const std::string& name) return (name != UntwineBitsDimName); } -// We check both _WIN32 and _MSC_VER to deal with MinGW, which doesn't support the special -// Windows wide character interfaces for streams. -#if defined(_WIN32) && defined(_MSC_VER) -inline std::wstring toNative(const std::string& in) -{ - if (in.empty()) - return std::wstring(); - - int len = MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), nullptr, 0); - std::wstring out(len, 0); - if (MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), out.data(), len) == 0) - { - char buf[200] {}; - len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); - throw FatalError("Can't convert UTF8 to UTF16: " + std::string(buf, len)); - } - return out; -} - -inline std::string fromNative(const std::wstring& in) -{ - if (in.empty()) - return std::string(); - - int len = WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), nullptr, - 0, nullptr, nullptr); - std::string out(len, 0); - if (WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), out.data(), - len, nullptr, nullptr) == 0) - { - char buf[200] {}; - len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); - throw FatalError("Can't convert UTF16 to UTF8: " + std::string(buf, len)); - } - return out; -} -#else -inline std::string toNative(const std::string& in) -{ - return in; -} - -inline std::string fromNative(const std::string& in) -{ - return in; -} -#endif - -//ABELL - This exists here because older version of PDAL don't have it and the QGIS -// crew wanted things to work with older versions of PDAL. -/** - Context info for mapping a file. -*/ -struct MapContext -{ -public: - MapContext() : m_fd(-1), m_addr(nullptr) - {} - - void *addr() const - { return m_addr; } - std::string what() const - { return m_error; } - - int m_fd; - size_t m_size; - void *m_addr; - std::string m_error; -#ifdef _WIN32 - HANDLE m_handle; -#endif -}; - -/** - Map a file to memory. - \param filename Filename to map. - \param readOnly Must be true at this time. - \param pos Starting position of file to map. - \param size Number of bytes in file to map. - \return MapContext. addr() gets the mapped address. what() gets - any error message. addr() returns nullptr on error. -*/ -MapContext mapFile(const std::string& filename, bool readOnly, size_t pos, size_t size); - -/** - Unmap a previously mapped file. - \param ctx Previously returned MapContext - \return MapContext indicating current state of the file mapping. -*/ -MapContext unmapFile(MapContext ctx); - -std::vector directoryList(const std::string& dir); - } // namespace untwine diff --git a/external/untwine/untwine/FatalError.hpp b/external/untwine/untwine/FatalError.hpp new file mode 100644 index 000000000000..916ea82ccb50 --- /dev/null +++ b/external/untwine/untwine/FatalError.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace untwine +{ + +class FatalError : public std::runtime_error +{ +public: + inline FatalError(std::string const& msg) : std::runtime_error(msg) + {} +}; + +} // namespace untwine diff --git a/external/untwine/untwine/ProgressWriter.cpp b/external/untwine/untwine/ProgressWriter.cpp index f45c9c4e9016..ee3fd47ac7cd 100644 --- a/external/untwine/untwine/ProgressWriter.cpp +++ b/external/untwine/untwine/ProgressWriter.cpp @@ -2,13 +2,8 @@ #include #include -#ifndef _WIN32 -#include -#else -#include -#endif - #include "ProgressWriter.hpp" +#include "progress.hpp" namespace untwine { @@ -71,27 +66,8 @@ void ProgressWriter::writeMessage(uint32_t percent, const std::string& message) return; const int32_t msgId = 1000; -#ifndef _WIN32 - bool err = false; - err = (::write(m_fd, &msgId, sizeof(msgId)) == -1); - err |= (::write(m_fd, &percent, sizeof(percent)) == -1); - uint32_t ssize = (uint32_t)message.size(); - err |= (::write(m_fd, &ssize, sizeof(ssize)) == -1); - err |= (::write(m_fd, message.data(), ssize) == -1); - if (err) - { - ::close(m_fd); + if (!os::writeMessage(m_fd, msgId, percent, message)) m_fd = -1; - } -#else - DWORD numWritten; - HANDLE h = reinterpret_cast((intptr_t)m_fd); - WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL); - WriteFile(h, &percent, sizeof(percent), &numWritten, NULL); - uint32_t ssize = (uint32_t)message.size(); - WriteFile(h, &ssize, sizeof(ssize), &numWritten, NULL); - WriteFile(h, message.data(), ssize, &numWritten, NULL); -#endif } void ProgressWriter::writeErrorMessage(const std::string& message) @@ -103,25 +79,8 @@ void ProgressWriter::writeErrorMessage(const std::string& message) } const int32_t msgId = 1001; -#ifndef _WIN32 - bool err = false; - err = (::write(m_fd, &msgId, sizeof(msgId)) == -1); - uint32_t ssize = (uint32_t)message.size(); - err |= (::write(m_fd, &ssize, sizeof(ssize)) == -1); - err |= (::write(m_fd, message.data(), ssize) == -1); - if (err) - { - ::close(m_fd); + if (!os::writeErrorMessage(m_fd, msgId, message)) m_fd = -1; - } -#else - DWORD numWritten; - HANDLE h = reinterpret_cast((intptr_t)m_fd); - WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL); - uint32_t ssize = (uint32_t)message.size(); - WriteFile(h, &ssize, sizeof(ssize), &numWritten, NULL); - WriteFile(h, message.data(), ssize, &numWritten, NULL); -#endif } // Determine the point increment and reset the counters. diff --git a/external/untwine/untwine/Untwine.cpp b/external/untwine/untwine/Untwine.cpp index ddd5a0c5bc66..e398bedda0d3 100644 --- a/external/untwine/untwine/Untwine.cpp +++ b/external/untwine/untwine/Untwine.cpp @@ -23,6 +23,9 @@ #include "../epf/Epf.hpp" #include "../bu/BuPyramid.hpp" +#include // untwine/os +#include // untwine/os + namespace untwine { @@ -72,7 +75,7 @@ bool handleOptions(pdal::StringList& arglist, Options& options) std::cout << "untwine version (" << UNTWINE_VERSION << ")\n"; if (help) { - std::cout << "Usage: untwine [output file/directory] \n"; + std::cout << "Usage: untwine output file \n"; programArgs.dump(std::cout, 2, 80); } if (help || version) @@ -80,6 +83,14 @@ bool handleOptions(pdal::StringList& arglist, Options& options) programArgs.parse(arglist); + // Make sure the output file can be opened so that we can provide an early error if + // there's a problem. + std::ofstream tmp(os::toNative(options.outputName), std::ios::out | std::ios::binary); + if (!tmp) + throw FatalError("Can't open file '" + options.outputName + "' for output"); + tmp.close(); + pdal::FileUtils::deleteFile(options.outputName); + if (!tempArg->set()) { options.tempDir = options.outputName + "_tmp"; @@ -114,7 +125,7 @@ void cleanup(const std::string& dir, bool rmdir) std::regex re("[0-9]+-[0-9]+-[0-9]+-[0-9]+.bin"); std::smatch sm; - const std::vector& files = directoryList(dir); + const std::vector& files = os::directoryList(dir); for (const std::string& f : files) if (std::regex_match(f, sm, re)) pdal::FileUtils::deleteFile(dir + "/" + f); @@ -136,7 +147,7 @@ int main(int argc, char *argv[]) argv++; argc--; while (argc--) - arglist.push_back(untwine::fromNative(*argv++)); + arglist.push_back(untwine::os::fromNative(*argv++)); using namespace untwine; diff --git a/external/untwine/untwine/generic/dirlist.hpp b/external/untwine/untwine/generic/dirlist.hpp new file mode 100644 index 000000000000..cc4bf903401f --- /dev/null +++ b/external/untwine/untwine/generic/dirlist.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include + +namespace untwine +{ +namespace os +{ + +// PDAL's directoryList had a bug, so we've imported a working +// version here so that we can still use older PDAL releases. + +inline std::vector directoryList(const std::string& dir) +{ + namespace fs = std::filesystem; + + std::vector files; + + try + { + fs::directory_iterator it(toNative(dir)); + fs::directory_iterator end; + while (it != end) + { + files.push_back(fromNative(it->path())); + it++; + } + } + catch (fs::filesystem_error&) + { + files.clear(); + } + return files; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/generic/mapfile.hpp b/external/untwine/untwine/generic/mapfile.hpp new file mode 100644 index 000000000000..3488710a871e --- /dev/null +++ b/external/untwine/untwine/generic/mapfile.hpp @@ -0,0 +1,79 @@ + +#pragma once + +#include +#include +#include + +namespace untwine +{ +namespace os +{ + +//ABELL - This exists here because older version of PDAL don't have it and the QGIS +// crew wanted things to work with older versions of PDAL. +/** + Context info for mapping a file. +*/ +struct MapContext +{ +public: + MapContext() : m_fd(-1), m_addr(nullptr) + {} + + void *addr() const + { return m_addr; } + std::string what() const + { return m_error; } + + int m_fd; + size_t m_size; + void *m_addr; + std::string m_error; +}; + +inline MapContext mapFile(const std::string& filename, bool readOnly, size_t pos, size_t size) +{ + MapContext ctx; + + if (!readOnly) + { + ctx.m_error = "readOnly must be true."; + return ctx; + } + + ctx.m_fd = ::open(filename.data(), readOnly ? O_RDONLY : O_RDWR); + + if (ctx.m_fd == -1) + { + ctx.m_error = "Mapped file couldn't be opened."; + return ctx; + } + ctx.m_size = size; + + ctx.m_addr = ::mmap(0, size, PROT_READ, MAP_SHARED, ctx.m_fd, (off_t)pos); + if (ctx.m_addr == MAP_FAILED) + { + ctx.m_addr = nullptr; + ctx.m_error = "Couldn't map file"; + } + + return ctx; +} + +inline MapContext unmapFile(MapContext ctx) +{ + if (::munmap(ctx.m_addr, ctx.m_size) == -1) + ctx.m_error = "Couldn't unmap file."; + else + { + ctx.m_addr = nullptr; + ctx.m_size = 0; + ctx.m_error = ""; + } + ::close(ctx.m_fd); + return ctx; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/generic/progress.hpp b/external/untwine/untwine/generic/progress.hpp new file mode 100644 index 000000000000..e7d8a0847681 --- /dev/null +++ b/external/untwine/untwine/generic/progress.hpp @@ -0,0 +1,36 @@ +#include +#include +#include + +namespace untwine +{ +namespace os +{ + +inline bool writeMessage(int fd, int32_t msgId, uint32_t percent, const std::string& message) +{ + bool err = false; + err = (::write(fd, &msgId, sizeof(msgId)) == -1); + err |= (::write(fd, &percent, sizeof(percent)) == -1); + uint32_t ssize = (uint32_t)message.size(); + err |= (::write(fd, &ssize, sizeof(ssize)) == -1); + err |= (::write(fd, message.data(), ssize) == -1); + if (err) + ::close(fd); + return err; +} + +inline bool writeErrorMessage(int fd, int32_t msgId, const std::string& message) +{ + bool err = false; + err = (::write(fd, &msgId, sizeof(msgId)) == -1); + uint32_t ssize = (uint32_t)message.size(); + err |= (::write(fd, &ssize, sizeof(ssize)) == -1); + err |= (::write(fd, message.data(), ssize) == -1); + if (err) + ::close(fd); + return err; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/generic/stringconv.hpp b/external/untwine/untwine/generic/stringconv.hpp new file mode 100644 index 000000000000..e364f5ea7fad --- /dev/null +++ b/external/untwine/untwine/generic/stringconv.hpp @@ -0,0 +1,19 @@ +#pragma once + +namespace untwine +{ +namespace os +{ + +inline std::string toNative(const std::string& in) +{ + return in; +} + +inline std::string fromNative(const std::string& in) +{ + return in; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/mingw/dirlist.hpp b/external/untwine/untwine/mingw/dirlist.hpp new file mode 100644 index 000000000000..d22668980c15 --- /dev/null +++ b/external/untwine/untwine/mingw/dirlist.hpp @@ -0,0 +1 @@ +#include "../generic/dirlist.hpp" diff --git a/external/untwine/untwine/mingw/mapfile.h b/external/untwine/untwine/mingw/mapfile.h new file mode 100644 index 000000000000..6d9362643acf --- /dev/null +++ b/external/untwine/untwine/mingw/mapfile.h @@ -0,0 +1,80 @@ +#pragma once + +#include + +namespace untwine +{ +namespace os +{ + +//ABELL - This exists here because older version of PDAL don't have it and the QGIS +// crew wanted things to work with older versions of PDAL. +/** + Context info for mapping a file. +*/ +struct MapContext +{ +public: + MapContext() : m_fd(-1), m_addr(nullptr) + {} + + void *addr() const + { return m_addr; } + std::string what() const + { return m_error; } + + int m_fd; + size_t m_size; + void *m_addr; + std::string m_error; + HANDLE m_handle; +}; + +MapContext mapFile(const std::string& filename, bool readOnly, size_t pos, size_t size) +{ + MapContext ctx; + + if (!readOnly) + { + ctx.m_error = "readOnly must be true."; + return ctx; + } + + ctx.m_fd = ::_open(toNative(filename).data(), readOnly ? _O_RDONLY : _O_RDWR); + + if (ctx.m_fd == -1) + { + ctx.m_error = "Mapped file couldn't be opened."; + return ctx; + } + ctx.m_size = size; + + ctx.m_handle = CreateFileMapping((HANDLE)_get_osfhandle(ctx.m_fd), + NULL, PAGE_READONLY, 0, 0, NULL); + uint32_t low = pos & 0xFFFFFFFF; + uint32_t high = (pos >> 8); + ctx.m_addr = MapViewOfFile(ctx.m_handle, FILE_MAP_READ, high, low, + ctx.m_size); + if (ctx.m_addr == nullptr) + ctx.m_error = "Couldn't map file"; + + return ctx; +} + +MapContext unmapFile(MapContext ctx) +{ + if (UnmapViewOfFile(ctx.m_addr) == 0) + ctx.m_error = "Couldn't unmap file."; + else + { + ctx.m_addr = nullptr; + ctx.m_size = 0; + ctx.m_error = ""; + } + CloseHandle(ctx.m_handle); + ::_close(ctx.m_fd); + return ctx; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/mingw/progress.hpp b/external/untwine/untwine/mingw/progress.hpp new file mode 100644 index 000000000000..d3c1691b78d2 --- /dev/null +++ b/external/untwine/untwine/mingw/progress.hpp @@ -0,0 +1 @@ +#include "../generic/progress.hpp" diff --git a/external/untwine/untwine/mingw/stringconv.hpp b/external/untwine/untwine/mingw/stringconv.hpp new file mode 100644 index 000000000000..0c976a0068a1 --- /dev/null +++ b/external/untwine/untwine/mingw/stringconv.hpp @@ -0,0 +1 @@ +#include "../generic/stringconv.hpp" diff --git a/external/untwine/untwine/osx/dirlist.hpp b/external/untwine/untwine/osx/dirlist.hpp new file mode 100644 index 000000000000..9a540965248a --- /dev/null +++ b/external/untwine/untwine/osx/dirlist.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include + +namespace untwine +{ +namespace os +{ + +// Provide simple opendir/readdir solution for OSX because directory_iterator is +// not available until OSX 10.15 +inline std::vector directoryList(const std::string& dir) +{ + std::vector files; + + DIR *dpdf = opendir(dir.c_str()); + if (dpdf) + { + while (true) + { + struct dirent *epdf = readdir(dpdf); + if (!epdf) + break; + + std::string name = fromNative(epdf->d_name); + // Skip paths + if (!pdal::Utils::iequals(name, ".") && + !pdal::Utils::iequals(name, "..")) + { + // we expect the path + name + files.push_back(dir + "/" + fromNative(epdf->d_name)); + } + } + closedir(dpdf); + } + return files; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/osx/mapfile.hpp b/external/untwine/untwine/osx/mapfile.hpp new file mode 100644 index 000000000000..66d54fd963ec --- /dev/null +++ b/external/untwine/untwine/osx/mapfile.hpp @@ -0,0 +1 @@ +#include "../generic/mapfile.hpp" diff --git a/external/untwine/untwine/osx/progress.hpp b/external/untwine/untwine/osx/progress.hpp new file mode 100644 index 000000000000..d3c1691b78d2 --- /dev/null +++ b/external/untwine/untwine/osx/progress.hpp @@ -0,0 +1 @@ +#include "../generic/progress.hpp" diff --git a/external/untwine/untwine/osx/stringconv.hpp b/external/untwine/untwine/osx/stringconv.hpp new file mode 100644 index 000000000000..0c976a0068a1 --- /dev/null +++ b/external/untwine/untwine/osx/stringconv.hpp @@ -0,0 +1 @@ +#include "../generic/stringconv.hpp" diff --git a/external/untwine/untwine/windows/dirlist.hpp b/external/untwine/untwine/windows/dirlist.hpp new file mode 100644 index 000000000000..d22668980c15 --- /dev/null +++ b/external/untwine/untwine/windows/dirlist.hpp @@ -0,0 +1 @@ +#include "../generic/dirlist.hpp" diff --git a/external/untwine/untwine/windows/mapfile.hpp b/external/untwine/untwine/windows/mapfile.hpp new file mode 100644 index 000000000000..abd7dcfbc08a --- /dev/null +++ b/external/untwine/untwine/windows/mapfile.hpp @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include + +#include + +namespace untwine +{ +namespace os +{ + +//ABELL - This exists here because older version of PDAL don't have it and the QGIS +// crew wanted things to work with older versions of PDAL. +/** + Context info for mapping a file. +*/ +struct MapContext +{ +public: + MapContext() : m_fd(-1), m_addr(nullptr) + {} + + void *addr() const + { return m_addr; } + std::string what() const + { return m_error; } + + int m_fd; + size_t m_size; + void *m_addr; + std::string m_error; + HANDLE m_handle; +}; + +inline MapContext mapFile(const std::string& filename, bool readOnly, size_t pos, size_t size) +{ + MapContext ctx; + + if (!readOnly) + { + ctx.m_error = "readOnly must be true."; + return ctx; + } + + ctx.m_fd = ::_wopen(toNative(filename).data(), readOnly ? _O_RDONLY : _O_RDWR); + + if (ctx.m_fd == -1) + { + ctx.m_error = "Mapped file couldn't be opened."; + return ctx; + } + ctx.m_size = size; + + ctx.m_handle = CreateFileMapping((HANDLE)_get_osfhandle(ctx.m_fd), + NULL, PAGE_READONLY, 0, 0, NULL); + uint32_t low = pos & 0xFFFFFFFF; + uint32_t high = (pos >> 8); + ctx.m_addr = MapViewOfFile(ctx.m_handle, FILE_MAP_READ, high, low, + ctx.m_size); + if (ctx.m_addr == nullptr) + ctx.m_error = "Couldn't map file"; + + return ctx; +} + +inline MapContext unmapFile(MapContext ctx) +{ + if (UnmapViewOfFile(ctx.m_addr) == 0) + ctx.m_error = "Couldn't unmap file."; + else + { + ctx.m_addr = nullptr; + ctx.m_size = 0; + ctx.m_error = ""; + } + CloseHandle(ctx.m_handle); + ::_close(ctx.m_fd); + return ctx; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/windows/progress.hpp b/external/untwine/untwine/windows/progress.hpp new file mode 100644 index 000000000000..5550cef9a44d --- /dev/null +++ b/external/untwine/untwine/windows/progress.hpp @@ -0,0 +1,32 @@ +#include + +namespace untwine +{ +namespace os +{ + +inline bool writeMessage(int fd, int32_t msgId, uint32_t percent, const std::string& message) +{ + DWORD numWritten; + HANDLE h = reinterpret_cast((intptr_t)fd); + WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL); + WriteFile(h, &percent, sizeof(percent), &numWritten, NULL); + uint32_t ssize = (uint32_t)message.size(); + WriteFile(h, &ssize, sizeof(ssize), &numWritten, NULL); + WriteFile(h, message.data(), ssize, &numWritten, NULL); + return true; +} + +inline bool writeErrorMessage(int fd, int32_t msgId, const std::string& message) +{ + DWORD numWritten; + HANDLE h = reinterpret_cast((intptr_t)fd); + WriteFile(h, &msgId, sizeof(msgId), &numWritten, NULL); + uint32_t ssize = (uint32_t)message.size(); + WriteFile(h, &ssize, sizeof(ssize), &numWritten, NULL); + WriteFile(h, message.data(), ssize, &numWritten, NULL); + return true; +} + +} // namespace os +} // namespace untwine diff --git a/external/untwine/untwine/windows/stringconv.hpp b/external/untwine/untwine/windows/stringconv.hpp new file mode 100644 index 000000000000..0d072fb3afd5 --- /dev/null +++ b/external/untwine/untwine/windows/stringconv.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include + +#include + +namespace untwine +{ +namespace os +{ + +inline std::wstring toNative(const std::string& in) +{ + if (in.empty()) + return std::wstring(); + + int len = MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), nullptr, 0); + std::wstring out(len, 0); + if (MultiByteToWideChar(CP_UTF8, 0, in.data(), in.length(), out.data(), len) == 0) + { + char buf[200] {}; + len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); + throw FatalError("Can't convert UTF8 to UTF16: " + std::string(buf, len)); + } + return out; +} + +inline std::string fromNative(const std::wstring& in) +{ + if (in.empty()) + return std::string(); + + int len = WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), nullptr, + 0, nullptr, nullptr); + std::string out(len, 0); + if (WideCharToMultiByte(CP_UTF8, 0, in.data(), in.length(), out.data(), + len, nullptr, nullptr) == 0) + { + char buf[200] {}; + len = FormatMessageA(0, 0, GetLastError(), 0, buf, 199, 0); + throw FatalError("Can't convert UTF16 to UTF8: " + std::string(buf, len)); + } + return out; +} + +} // namespace os +} // namespace untwine diff --git a/src/providers/pdal/CMakeLists.txt b/src/providers/pdal/CMakeLists.txt index 936958fb1a1e..d4082adc29bb 100644 --- a/src/providers/pdal/CMakeLists.txt +++ b/src/providers/pdal/CMakeLists.txt @@ -46,7 +46,6 @@ set(UNTWINE_SRCS ${CMAKE_SOURCE_DIR}/external/untwine/epf/Reprocessor.cpp ${CMAKE_SOURCE_DIR}/external/untwine/epf/Writer.cpp - ${CMAKE_SOURCE_DIR}/external/untwine/untwine/Common.cpp ${CMAKE_SOURCE_DIR}/external/untwine/untwine/ProgressWriter.cpp ${CMAKE_SOURCE_DIR}/external/untwine/untwine/ThreadPool.cpp ${CMAKE_SOURCE_DIR}/external/untwine/untwine/Untwine.cpp @@ -73,6 +72,7 @@ set(UNTWINE_HDRS ${CMAKE_SOURCE_DIR}/external/untwine/epf/Writer.hpp ${CMAKE_SOURCE_DIR}/external/untwine/untwine/Common.hpp + ${CMAKE_SOURCE_DIR}/external/untwine/untwine/FatalError.hpp ${CMAKE_SOURCE_DIR}/external/untwine/untwine/FileDimInfo.hpp ${CMAKE_SOURCE_DIR}/external/untwine/untwine/GridKey.hpp ${CMAKE_SOURCE_DIR}/external/untwine/untwine/Point.hpp @@ -81,6 +81,20 @@ set(UNTWINE_HDRS ${CMAKE_SOURCE_DIR}/external/untwine/untwine/VoxelKey.hpp ) +# MINGW must come before WIN32 test +# APPLE must come before UNIX test +if (MINGW) + set(UNTWINE_OS_DIR ${CMAKE_SOURCE_DIR}/external/untwine/untwine/mingw) +elseif (WIN32) + set(UNTWINE_OS_DIR ${CMAKE_SOURCE_DIR}/external/untwine/untwine/windows) +elseif (APPLE) + set(UNTWINE_OS_DIR ${CMAKE_SOURCE_DIR}/external/untwine/untwine/osx) +elseif (UNIX) + set(UNTWINE_OS_DIR ${CMAKE_SOURCE_DIR}/external/untwine/untwine/generic) +else() + message(FATAL_ERROR "OS not supported") +endif() + configure_file(${CMAKE_SOURCE_DIR}/external/untwine/untwine/Config.hpp.in ${CMAKE_BINARY_DIR}/untwine/Config.hpp) set(UNTWINE_INCLUDE_DIRS @@ -156,7 +170,7 @@ target_link_libraries (untwine ${PDAL_LIBRARIES} Threads::Threads ) -target_include_directories(untwine PRIVATE ${UNTWINE_INCLUDE_DIRS}) +target_include_directories(untwine PRIVATE ${UNTWINE_OS_DIR} ${UNTWINE_INCLUDE_DIRS}) if (LazPerf_FOUND) # Use system laz-perf