From 6cb77ce50eaf1f8b555ea6c4409698621081fb44 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Mon, 6 Dec 2021 22:55:14 +0300 Subject: [PATCH 01/11] Working on reading PBO pack configuration from pbo.json file and/or prefix files --- pbom/model/CMakeLists.txt | 4 + pbom/model/binarysourceutils.h | 30 +++ pbom/model/interactionparcel.cpp | 18 +- .../task/__test__/packconfiguration_test.cpp | 156 ++++++++++++++ pbom/model/task/__test__/packoptions_test.cpp | 48 +++++ pbom/model/task/packconfiguration.cpp | 134 ++++++++++++ pbom/model/task/packconfiguration.h | 41 ++++ pbom/model/task/packoptions.cpp | 47 ++++ pbom/model/task/packoptions.h | 49 +++++ pbom/util/CMakeLists.txt | 2 + pbom/util/__test__/json_test.cpp | 202 ++++++++++++++++++ pbom/util/json.cpp | 48 +++++ pbom/util/json.h | 166 ++++++++++++++ 13 files changed, 929 insertions(+), 16 deletions(-) create mode 100644 pbom/model/binarysourceutils.h create mode 100644 pbom/model/task/__test__/packconfiguration_test.cpp create mode 100644 pbom/model/task/__test__/packoptions_test.cpp create mode 100644 pbom/model/task/packconfiguration.cpp create mode 100644 pbom/model/task/packconfiguration.h create mode 100644 pbom/model/task/packoptions.cpp create mode 100644 pbom/model/task/packoptions.h create mode 100644 pbom/util/__test__/json_test.cpp create mode 100644 pbom/util/json.cpp create mode 100644 pbom/util/json.h diff --git a/pbom/model/CMakeLists.txt b/pbom/model/CMakeLists.txt index 471dad9..99aeba7 100644 --- a/pbom/model/CMakeLists.txt +++ b/pbom/model/CMakeLists.txt @@ -1,4 +1,6 @@ list(APPEND PROJECT_SOURCES + "model/task/packconfiguration.cpp" + "model/task/packoptions.cpp" "model/task/packtask.cpp" "model/task/packwindowmodel.cpp" "model/task/task.h" @@ -12,6 +14,8 @@ list(APPEND PROJECT_SOURCES set(PROJECT_SOURCES ${PROJECT_SOURCES} PARENT_SCOPE) list(APPEND TEST_SOURCES + "model/task/__test__/packconfiguration_test.cpp" + "model/task/__test__/packoptions_test.cpp" "model/__test__/conflictsparcel_test.cpp" "model/__test__/interactionparcel_test.cpp") diff --git a/pbom/model/binarysourceutils.h b/pbom/model/binarysourceutils.h new file mode 100644 index 0000000..faae641 --- /dev/null +++ b/pbom/model/binarysourceutils.h @@ -0,0 +1,30 @@ +#pragma once + +#include "exception.h" +#include "domain/binarysource.h" +#include "io/bs/pbobinarysource.h" +#include "io/bs/fsrawbinarysource.h" +#include "io/bs/fslzhbinarysource.h" + +namespace pboman3::model { + using namespace domain; + using namespace io; + + inline void ChangeBinarySourceCompressionMode(QSharedPointer& bs, bool compress) { + if (dynamic_cast(bs.get())) { + throw InvalidOperationException("Can't query compression status"); + } + + if (compress) { + if (dynamic_cast(bs.get())) { + bs = QSharedPointer(new FsLzhBinarySource(bs->path())); + bs->open(); + } + } else { + if (dynamic_cast(bs.get())) { + bs = QSharedPointer(new FsRawBinarySource(bs->path())); + bs->open(); + } + } + } +} diff --git a/pbom/model/interactionparcel.cpp b/pbom/model/interactionparcel.cpp index 2c9a438..c5ecf86 100644 --- a/pbom/model/interactionparcel.cpp +++ b/pbom/model/interactionparcel.cpp @@ -1,7 +1,7 @@ #include "interactionparcel.h" #include #include -#include "exception.h" +#include "binarysourceutils.h" namespace pboman3::model { const QSharedPointer& NodeDescriptor::binarySource() const { @@ -13,21 +13,7 @@ namespace pboman3::model { } void NodeDescriptor::setCompressed(bool compressed) { - if (dynamic_cast(binarySource_.get())) { - throw InvalidOperationException("Can't query compression status"); - } - - if (compressed) { - if (dynamic_cast(binarySource_.get())) { - binarySource_ = QSharedPointer(new FsLzhBinarySource(binarySource_->path())); - binarySource_->open(); - } - } else { - if (dynamic_cast(binarySource_.get())) { - binarySource_ = QSharedPointer(new FsRawBinarySource(binarySource_->path())); - binarySource_->open(); - } - } + ChangeBinarySourceCompressionMode(binarySource_, compressed); } QByteArray NodeDescriptors::serialize(const NodeDescriptors& data) { diff --git a/pbom/model/task/__test__/packconfiguration_test.cpp b/pbom/model/task/__test__/packconfiguration_test.cpp new file mode 100644 index 0000000..cc23e51 --- /dev/null +++ b/pbom/model/task/__test__/packconfiguration_test.cpp @@ -0,0 +1,156 @@ +#include "model/task/packconfiguration.h" +#include +#include +#include "domain/pbodocument.h" +#include "io/bs/fslzhbinarysource.h" +#include "io/bs/fsrawbinarysource.h" + +namespace pboman3::model::task::test { + using namespace domain; + + TEST(PackConfigurationTest, Apply_Removes_PboJson_Node) { + QTemporaryFile json; + json.open(); + json.write(QByteArray("{}")); + json.close(); + + PboDocument document("file.pbo"); + document.root()->createHierarchy(PboPath({"f1.txt"})); + PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); + pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_EQ(document.root()->count(), 1); //pbo.json removed + ASSERT_TRUE(document.root()->get(PboPath({"f1.txt"}))); //but others are in places + } + + TEST(PackConfigurationTest, Apply_Sets_Headers_From_PboJson) { + QTemporaryFile json; + json.open(); + json.write(QByteArray( + "{\"headers\":[{\"name\":\"p1\", \"value\":\"v1\"}, {\"name\":\"p2\", \"value\":\"v2\"}]}")); + json.close(); + + PboDocument document("file.pbo"); + PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); + pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_EQ(document.headers()->count(), 2); + ASSERT_EQ(document.headers()->at(0)->name(), "p1"); + ASSERT_EQ(document.headers()->at(0)->value(), "v1"); + ASSERT_EQ(document.headers()->at(1)->name(), "p2"); + ASSERT_EQ(document.headers()->at(1)->value(), "v2"); + } + + TEST(PackConfigurationTest, Apply_Compresses_Only_Included_Files) { + QTemporaryFile json; + json.open(); + json.write(QByteArray("{\"compress\":{\"include\":[\".+\\.txt$\"]}}")); + json.close(); + + PboDocument document("file.pbo"); + PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); + pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + PboNode* node1 = document.root()->createHierarchy(PboPath({"file.txt"})); + node1->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* node2 = document.root()->createHierarchy(PboPath({"file2.txt"})); + node2->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* node3 = document.root()->createHierarchy(PboPath({"file3.jpg"})); + node3->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_TRUE(dynamic_cast(node1->binarySource.get())); + ASSERT_TRUE(dynamic_cast(node2->binarySource.get())); + ASSERT_TRUE(dynamic_cast(node3->binarySource.get())); + } + + TEST(PackConfigurationTest, Apply_Not_Compresses_Inlcuded_But_Excluded_Files) { + QTemporaryFile json; + json.open(); + json.write(QByteArray("{\"compress\":{\"include\":[\"\\.txt$\"], \"exclude\":[\"^file3\\.\"]}}")); + json.close(); + + PboDocument document("file.pbo"); + PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); + pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + PboNode* node1 = document.root()->createHierarchy(PboPath({"file.txt"})); + node1->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* node2 = document.root()->createHierarchy(PboPath({"file2.txt"})); + node2->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* node3 = document.root()->createHierarchy(PboPath({"file3.txt"})); + node3->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_TRUE(dynamic_cast(node1->binarySource.get())); + ASSERT_TRUE(dynamic_cast(node2->binarySource.get())); + ASSERT_TRUE(dynamic_cast(node3->binarySource.get())); + } + + struct PackConfigurationJsonIssuesParam { + QString json; + QString expectedMessage; + }; + + class PackConfigurationJsonIssuesTest : public testing::TestWithParam { + }; + + TEST_P(PackConfigurationJsonIssuesTest, Apply_Throws_If_PboJson_Has_Issues) { + QTemporaryFile json; + json.open(); + json.write(GetParam().json.toUtf8()); + json.close(); + + PboDocument document("file.pbo"); + PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); + pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + const PackConfiguration packConfiguration(&document); + try { + packConfiguration.apply(); + FAIL() << "Should not have reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), GetParam().expectedMessage); + } + } + + INSTANTIATE_TEST_SUITE_P(PackConfigurationTest, PackConfigurationJsonIssuesTest, testing::Values( + PackConfigurationJsonIssuesParam{"", "illegal value at offset 0"}, + PackConfigurationJsonIssuesParam{"[]", "The json must contain an object"}, + PackConfigurationJsonIssuesParam{"ghkjk", "illegal value at offset 1"}, + PackConfigurationJsonIssuesParam{"{\"compress\":{\"include\":[\"[[\"]}}", + "The regular expression \"[[\" is invalid: missing terminating ] for character class" + } + )); + + TEST(PackConfigurationTest, Apply_Throws_If_Json_Is_Not_Object) { + QTemporaryFile json; + json.open(); + json.write(QByteArray( + "{\"headers\":[{\"name\":\"p1\", \"value\":\"v1\"}, {\"name\":\"p2\", \"value\":\"v2\"}]}")); + json.close(); + + PboDocument document("file.pbo"); + PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); + pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_EQ(document.headers()->count(), 2); + ASSERT_EQ(document.headers()->at(0)->name(), "p1"); + ASSERT_EQ(document.headers()->at(0)->value(), "v1"); + ASSERT_EQ(document.headers()->at(1)->name(), "p2"); + ASSERT_EQ(document.headers()->at(1)->value(), "v2"); + } +} diff --git a/pbom/model/task/__test__/packoptions_test.cpp b/pbom/model/task/__test__/packoptions_test.cpp new file mode 100644 index 0000000..fc0a627 --- /dev/null +++ b/pbom/model/task/__test__/packoptions_test.cpp @@ -0,0 +1,48 @@ +#include "model/task/packoptions.h" +#include +#include + +namespace pboman3::model::task::test { + TEST(PackOptionsTest, Settle_Reads_Json_With_All_Fields) { + const QJsonDocument json = QJsonDocument::fromJson( + "{\"headers\":[{\"name\":\"n1\",\"value\":\"v1\"},{\"name\":\"n2\",\"value\":\"v2\"}], \"compress\":{\"include\":[\"i1\",\"i2\"],\"exclude\":[\"e1\",\"e2\"]}}"); + + PackOptions config; + config.settle(json.object(), ""); + + ASSERT_EQ(config.headers().count(), 2); + ASSERT_EQ(config.headers().at(0).name(), "n1"); + ASSERT_EQ(config.headers().at(0).value(), "v1"); + ASSERT_EQ(config.headers().at(1).name(), "n2"); + ASSERT_EQ(config.headers().at(1).value(), "v2"); + + ASSERT_EQ(config.compress().include().count(), 2); + ASSERT_EQ(config.compress().include().at(0), "i1"); + ASSERT_EQ(config.compress().include().at(1), "i2"); + ASSERT_EQ(config.compress().exclude().count(), 2); + ASSERT_EQ(config.compress().exclude().at(0), "e1"); + ASSERT_EQ(config.compress().exclude().at(1), "e2"); + } + + TEST(PackOptionsTest, Settle_Reads_Empty_Configuration) { + const QJsonDocument json = QJsonDocument::fromJson("{}"); + + PackOptions config; + config.settle(json.object(), ""); + + ASSERT_EQ(config.headers().count(), 0); + ASSERT_EQ(config.compress().include().count(), 0); + ASSERT_EQ(config.compress().exclude().count(), 0); + } + + TEST(PackOptionsTest, Settle_Reads_Empty_Compression) { + const QJsonDocument json = QJsonDocument::fromJson("{\"compression\":{}}"); + + PackOptions config; + config.settle(json.object(), ""); + + ASSERT_EQ(config.headers().count(), 0); + ASSERT_EQ(config.compress().include().count(), 0); + ASSERT_EQ(config.compress().exclude().count(), 0); + } +} diff --git a/pbom/model/task/packconfiguration.cpp b/pbom/model/task/packconfiguration.cpp new file mode 100644 index 0000000..0d6c071 --- /dev/null +++ b/pbom/model/task/packconfiguration.cpp @@ -0,0 +1,134 @@ +#include "packconfiguration.h" +#include +#include +#include +#include "domain/documentheaderstransaction.h" +#include "domain/func.h" +#include "io/diskaccessexception.h" +#include "model/binarysourceutils.h" + +namespace pboman3::model::task { + PackConfiguration::PackConfiguration(PboDocument* document) + : document_(document) { + } + + void PackConfiguration::apply() const { + PboNode* pboJson = FindDirectChild(document_->root(), "pbo.json"); + if (pboJson) { + const PackOptions packOptions = readPackOptions(pboJson); + applyDocumentHeaders(packOptions); + const CompressionRules compressionRules = buildCompressionRules(packOptions); + applyDocumentCompressionRules(document_->root(), compressionRules); + } + if (!pboJson) { + applyFileContentAsHeader("prefix"); + applyFileContentAsHeader("product"); + applyFileContentAsHeader("version"); + } + if (pboJson) + pboJson->removeFromHierarchy(); + } + + void PackConfiguration::applyDocumentHeaders(const PackOptions& options) const { + if (options.headers().isEmpty()) + return; + + const QSharedPointer tran = document_->headers()->beginTransaction(); + for (const PackHeader& header : options.headers()) { + tran->add(header.name(), header.value()); + } + + tran->commit(); + } + + void PackConfiguration::applyDocumentCompressionRules(PboNode* node, const CompressionRules& rules) const { + if (node->nodeType() == PboNodeType::File) { + if (shouldCompress(node, rules)) { + ChangeBinarySourceCompressionMode(node->binarySource, true); + } + } else { + for (PboNode* child : *node) { + applyDocumentCompressionRules(child, rules); + } + } + } + + bool PackConfiguration::shouldCompress(const PboNode* node, const CompressionRules& rules) { + const QString path = node->makePath().toString(); + + bool include = false; + for (const QRegularExpression& rule : rules.include) { + if (rule.match(path).hasMatch()) { + include = true; + break; + } + } + if (include) { + for (const QRegularExpression& rule : rules.exclude) { + if (rule.match(path).hasMatch()) { + include = false; + break; + } + } + } + return include; + } + + PackConfiguration::CompressionRules PackConfiguration::buildCompressionRules(const PackOptions& options) { + CompressionRules rules; + convertToCompressionRules(options.compress().include(), &rules.include); + convertToCompressionRules(options.compress().exclude(), &rules.exclude); + return rules; + } + + void PackConfiguration::convertToCompressionRules(const QList& source, QList* dest) { + dest->reserve(source.count()); + + for (const QString& rule : source) { + QRegularExpression reg( + rule, QRegularExpression::CaseInsensitiveOption | QRegularExpression::DontCaptureOption); + if (!reg.isValid()) { + throw JsonStructureException( + "The regular expression \"" + reg.pattern() + "\" is invalid: " + reg.errorString()); + } + dest->append(std::move(reg)); + } + } + + PackOptions PackConfiguration::readPackOptions(const PboNode* node) { + const QByteArray data = readNodeContent(node); + QJsonParseError err; + const QJsonDocument json = QJsonDocument::fromJson(data, &err); + if (json.isNull()) { + throw JsonStructureException(err.errorString() + " at offset " + QString::number(err.offset)); + } + if (!json.isObject()) { + throw JsonStructureException("The json must contain an object"); + } + + PackOptions opts; + opts.settle(json.object(), ""); + return opts; + } + + QByteArray PackConfiguration::readNodeContent(const PboNode* node) { + QFile f(node->binarySource->path()); + if (!f.open(QIODeviceBase::ReadOnly)) { + throw io::DiskAccessException("Could not read the file", f.fileName()); + } + QByteArray data = f.readAll(); + + return data; + } + + void PackConfiguration::applyFileContentAsHeader(const QString& prefix) const { + PboNode* node = FindDirectChild(document_->root(), "$" + prefix + "$"); + if (node) { + const QByteArray data = readNodeContent(node); + const QSharedPointer tran = document_->headers()->beginTransaction(); + tran->add(prefix, data); + tran->commit(); + node->removeFromHierarchy(); + } + } +} diff --git a/pbom/model/task/packconfiguration.h b/pbom/model/task/packconfiguration.h new file mode 100644 index 0000000..cf58510 --- /dev/null +++ b/pbom/model/task/packconfiguration.h @@ -0,0 +1,41 @@ +#pragma once + +#include "domain/pbodocument.h" +#include "packoptions.h" + +namespace pboman3::model::task { + using namespace domain; + + class PackConfiguration { + public: + explicit PackConfiguration(PboDocument* document); + + void apply() const; + + private: + struct CompressionRules; + + PboDocument* document_; + + void applyDocumentHeaders(const PackOptions& options) const; + + void applyDocumentCompressionRules(PboNode* node, const CompressionRules& rules) const; + + static bool shouldCompress(const PboNode* node, const CompressionRules& rules); + + static CompressionRules buildCompressionRules(const PackOptions& options); + + static void convertToCompressionRules(const QList& source, QList* dest); + + static PackOptions readPackOptions(const PboNode* node); + + static QByteArray readNodeContent(const PboNode* node); + + struct CompressionRules { + QList include; + QList exclude; + }; + + void applyFileContentAsHeader(const QString& prefix) const; + }; +} diff --git a/pbom/model/task/packoptions.cpp b/pbom/model/task/packoptions.cpp new file mode 100644 index 0000000..a8d827b --- /dev/null +++ b/pbom/model/task/packoptions.cpp @@ -0,0 +1,47 @@ +#include "packoptions.h" +#include +#include +#include + +namespace pboman3::model::task { + const QList& CompressOptions::include() const { + return include_; + } + + const QList& CompressOptions::exclude() const { + return exclude_; + } + + void CompressOptions::inflate(const QString& path, const QJsonObject& json) { + include_ = JsonArray>().settle(json, path, "include", JsonMandatory::No).data(); + exclude_ = JsonArray>().settle(json, path, "exclude", JsonMandatory::No).data(); + } + + + const QString& PackHeader::name() const { + return name_; + } + + const QString& PackHeader::value() const { + return value_; + } + + void PackHeader::inflate(const QString& path, const QJsonObject& json) { + name_ = JsonValue().settle(json, path, "name").value(); + value_ = JsonValue().settle(json, path, "value").value(); + } + + + const QList& PackOptions::headers() const { + return headers_; + } + + const CompressOptions& PackOptions::compress() const { + return compress_; + } + + void PackOptions::inflate(const QString& path, const QJsonObject& json) { + headers_ = JsonArray().settle(json, path, "headers", JsonMandatory::No).data(); + compress_.settle(json, path, "compress", JsonMandatory::No); + } +} diff --git a/pbom/model/task/packoptions.h b/pbom/model/task/packoptions.h new file mode 100644 index 0000000..ed77f05 --- /dev/null +++ b/pbom/model/task/packoptions.h @@ -0,0 +1,49 @@ +#pragma once + +#include "util/json.h" + +namespace pboman3::model::task { + using namespace util; + + class CompressOptions : public JsonObject { + public: + const QList& include() const; + + const QList& exclude() const; + + protected: + void inflate(const QString& path, const QJsonObject& json) override; + + private: + QList include_; + QList exclude_; + }; + + class PackHeader : public JsonObject { + public: + const QString& name() const; + + const QString& value() const; + + protected: + void inflate(const QString& path, const QJsonObject& json) override; + + private: + QString name_; + QString value_; + }; + + class PackOptions : public JsonObject { + public: + const QList& headers() const; + + const CompressOptions& compress() const; + + protected: + void inflate(const QString& path, const QJsonObject& json) override; + + private: + QList headers_; + CompressOptions compress_; + }; +} diff --git a/pbom/util/CMakeLists.txt b/pbom/util/CMakeLists.txt index 1dac322..53d8152 100644 --- a/pbom/util/CMakeLists.txt +++ b/pbom/util/CMakeLists.txt @@ -1,10 +1,12 @@ list(APPEND PROJECT_SOURCES + "util/json.cpp" "util/log.cpp" "util/util.cpp") set(PROJECT_SOURCES ${PROJECT_SOURCES} PARENT_SCOPE) list(APPEND TEST_SOURCES + "util/__test__/json_test.cpp" "util/__test__/qpointerlistiterator_test.cpp" "util/__test__/util_test.cpp") diff --git a/pbom/util/__test__/json_test.cpp b/pbom/util/__test__/json_test.cpp new file mode 100644 index 0000000..adc28a7 --- /dev/null +++ b/pbom/util/__test__/json_test.cpp @@ -0,0 +1,202 @@ +#include +#include "util/json.h" +#include + +namespace pboman3::util::test { + TEST(JsonValueTest, Settle_Reads_QJsonValue) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{\"prop\":\"value1\"}}"); + + JsonValue val1; + val1.settle(doc.object()["obj"], "obj", "prop"); + + ASSERT_EQ(val1.value(), "value1"); + } + + TEST(JsonValueTest, Settle_Produces_Error_If_Parent_Is_Not_Object) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"prop\":\"value1\"}"); + + try { + JsonValue val1; + val1.settle(doc.object()["prop"], ".prop", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".prop must be an {Object}"); + } + } + + TEST(JsonValueTest, Settle_Produces_Error_If_Parent_Key_Missing) { + const QJsonDocument doc = QJsonDocument::fromJson("{}"); + + try { + JsonValue val1; + val1.settle(doc.object(), ".", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ". must contain the key \"prop\""); + } + } + + TEST(JsonValueTest, Settle_Does_Nothing_If_Optional_Parent_Key_Missing) { + const QJsonDocument doc = QJsonDocument::fromJson("{}"); + + const QString val = JsonValue().settle(doc.object(), ".", "prop", JsonMandatory::No).value(); + ASSERT_TRUE(val.isEmpty()); + } + + TEST(JsonValueTest, Settle_Produces_Error_If_Key_Is_Not_String) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"prop\":1}"); + + try { + JsonValue val1; + val1.settle(doc.object(), "", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".prop must be a {String}"); + } + } + + + class MockJsonObject : public JsonObject { + public: + QString prop; + + protected: + void inflate(const QString& path, const QJsonObject& json) override { + prop = JsonValue().settle(json, path, "prop").value(); + } + }; + + TEST(JsonObjectTest, Settle_Reads_QJsonValue) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{\"prop\":\"value1\"}}"); + + MockJsonObject obj; + obj.settle(doc.object(), ".obj", "obj"); + + ASSERT_EQ(obj.prop, "value1"); + } + + TEST(JsonObjectTest, Settle_Produces_Error_If_Parent_Is_Not_Object) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":111}"); + + try { + MockJsonObject obj; + obj.settle(doc.object()["obj"], ".obj", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".obj must be an {Object}"); + } + } + + TEST(JsonObjectTest, Settle_Produces_Error_If_Parent_Key_Missing) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{}}"); + + try { + MockJsonObject obj; + obj.settle(doc.object()["obj"], ".obj", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".obj must contain the key \"prop\""); + } + } + + TEST(JsonObjectTest, Settle_Does_Nothing_If_Optional_Parent_Key_Missing) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{}}"); + + MockJsonObject obj; + obj.settle(doc.object()["obj"], ".obj", "prop", JsonMandatory::No); + + ASSERT_TRUE(obj.prop.isEmpty()); + } + + TEST(JsonObjectTest, Settle_Produces_Error_If_Json_Is_Not_Object) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{\"prop\": 11}}"); + + try { + MockJsonObject obj; + obj.settle(doc.object()["obj"], ".obj", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".obj.prop must be an {Object}"); + } + } + + + TEST(JsonArrayTest, Settle_Reads_QJsonValue_String) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":[\"s1\",\"s2\",\"s3\"]}"); + + const QList array = JsonArray>().settle(doc.object(), ".", "obj").data(); + + ASSERT_EQ(array.count(), 3); + ASSERT_EQ(array.at(0), "s1"); + ASSERT_EQ(array.at(1), "s2"); + ASSERT_EQ(array.at(2), "s3"); + } + + TEST(JsonArrayTest, Settle_Reads_QJsonValue_Object) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":[{\"prop\":\"v1\"}, {\"prop\":\"v2\"}]}"); + + JsonArray array; + array.settle(doc.object(), ".", "obj"); + + ASSERT_EQ(array.data().count(), 2); + ASSERT_EQ(array.data().at(0).prop, "v1"); + ASSERT_EQ(array.data().at(1).prop, "v2"); + } + + TEST(JsonArrayTest, Settle_Produces_Error_If_Parent_Is_Not_Array) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":111}"); + + try { + JsonArray> array; + array.settle(doc.object()["obj"], ".obj", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".obj must be an {Array}"); + } + } + + TEST(JsonArrayTest, Settle_Produces_Error_If_Parent_Key_Missing) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{}}"); + + try { + JsonArray> array; + array.settle(doc.object()["obj"], ".obj", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".obj must contain the key \"prop\""); + } + } + + TEST(JsonArrayTest, Settle_Does_Nothing_If_Optional_Parent_Key_Missing) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{}}"); + + const QList arr = JsonArray>().settle( + doc.object()["obj"], ".obj", "prop", JsonMandatory::No).data(); + + ASSERT_EQ(arr.count(), 0); + } + + TEST(JsonArrayTest, Settle_Produces_Error_If_Json_Is_Not_Array) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":{\"prop\": 111}}"); + + try { + JsonArray> array; + array.settle(doc.object()["obj"], ".obj", "prop"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".obj.prop must be an {Array}"); + } + } + + TEST(JsonArrayTest, Settle_Produces_Error_If_Element_Is_Wrong) { + const QJsonDocument doc = QJsonDocument::fromJson("{\"obj\":[\"aa\", 11]}"); + + try { + JsonArray> array; + array.settle(doc.object(), "", "obj"); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(ex.message(), ".obj[1] must be a {String}"); + } + } +} diff --git a/pbom/util/json.cpp b/pbom/util/json.cpp new file mode 100644 index 0000000..3c226a8 --- /dev/null +++ b/pbom/util/json.cpp @@ -0,0 +1,48 @@ +#include "util/json.h" +#include + +namespace pboman3::util { + JsonStructureException::JsonStructureException(QString message) + : AppException(std::move(message)) { + } + + QDebug operator<<(QDebug debug, const JsonStructureException& ex) { + return debug << "JsonStructureException(Message=" << ex.message_ << ")"; + } + + void JsonStructureException::raise() const { + throw *this; + } + + QException* JsonStructureException::clone() const { + return new JsonStructureException(*this); + } + + void JsonObject::settle(const QJsonObject& json, const QString& path) { + inflate(path, json); + } + + void JsonObject::settle(const QJsonValue& json, const QString& path) { + if (!json.isObject()) { + throw JsonStructureException(path + " must be an {Object}"); + } + settle(json.toObject(), path); + } + + void JsonObject::settle(const QJsonObject& parent, const QString& parentPath, const QString& name, JsonMandatory mandatory) { + if (!parent.contains(name)) { + if (mandatory == JsonMandatory::No) + return; + throw JsonStructureException(parentPath + " must contain the key \"" + name + "\""); + } + + settle(parent[name], parentPath + "." + name); + } + + void JsonObject::settle(const QJsonValue& parent, const QString& parentPath, const QString& name, JsonMandatory mandatory) { + if (!parent.isObject()) { + throw JsonStructureException(parentPath + " must be an {Object}"); + } + settle(parent.toObject(), parentPath, name, mandatory); + } +} diff --git a/pbom/util/json.h b/pbom/util/json.h new file mode 100644 index 0000000..e6ce67b --- /dev/null +++ b/pbom/util/json.h @@ -0,0 +1,166 @@ +#pragma once + +#include +#include +#include "exception.h" + +namespace pboman3::util { + enum class JsonMandatory { + Yes, + No + }; + + class JsonStructureException : public AppException { + public: + JsonStructureException(QString message); + + friend QDebug operator<<(QDebug debug, const JsonStructureException& ex); + + void raise() const override; + + QException* clone() const override; + }; + + namespace json { + template + struct JsonValueTrait { + static constexpr auto typeName = ""; + + static T read(const QJsonValue& value); + }; + + template <> + struct JsonValueTrait { + static constexpr auto typeName = "{String}"; + + static QString read(const QJsonValue& value) { + return value.toString(); + } + }; + } + + template + class JsonValue { + public: + T& value() { + return value_; + } + + JsonValue& settle(const QJsonValue& value, const QString& path) { + if (!value.isString()) { + throw JsonStructureException(path + " must be a " + json::JsonValueTrait::typeName); + } + value_ = json::JsonValueTrait::read(value); + return *this; + } + + JsonValue& settle(const QJsonObject& parent, const QString& parentPath, const QString& name, + JsonMandatory mandatory = JsonMandatory::Yes) { + if (!parent.contains(name)) { + if (mandatory == JsonMandatory::No) + return *this; + throw JsonStructureException(parentPath + " must contain the key \"" + name + "\""); + } + + return settle(parent[name], parentPath + "." + name); + } + + JsonValue& settle(const QJsonValue& parent, const QString& parentPath, const QString& name, + JsonMandatory mandatory = JsonMandatory::Yes) { + if (!parent.isObject()) { + throw JsonStructureException(parentPath + " must be an {Object}"); + } + return settle(parent.toObject(), parentPath, name, mandatory); + } + + private: + T value_; + }; + + class JsonObject { + public: + virtual ~JsonObject() = default; + + void settle(const QJsonObject& json, const QString& path); + + void settle(const QJsonValue& json, const QString& path); + + void settle(const QJsonObject& parent, const QString& parentPath, const QString& name, JsonMandatory mandatory = JsonMandatory::Yes); + + void settle(const QJsonValue& parent, const QString& parentPath, const QString& name, JsonMandatory mandatory = JsonMandatory::Yes); + + protected: + virtual void inflate(const QString& path, const QJsonObject& json) = 0; + }; + + namespace json { + template + struct JsonArrayTraits { + using TypeOuter = T; + using TypeInner = T; + static TypeOuter& getValue(TypeInner& t) { return t; } + }; + + template + struct JsonArrayTraits> { + using TypeOuter = T; + using TypeInner = JsonValue; + static TypeOuter& getValue(TypeInner& t) { return t.value(); } + }; + } + + template + class JsonArray { + public: + QList::TypeOuter> data() { + return data_; + } + + JsonArray& settle(const QJsonArray& json, const QString& path) { + inflate(path, json); + return *this; + } + + JsonArray& settle(const QJsonValue& json, const QString& path) { + if (!json.isArray()) { + throw JsonStructureException(path + " must be an {Array}"); + } + return settle(json.toArray(), path); + } + + JsonArray& settle(const QJsonObject& parent, const QString& parentPath, const QString& name, + JsonMandatory mandatory = JsonMandatory::Yes) { + if (!parent.contains(name)) { + if (mandatory == JsonMandatory::No) + return *this; + throw JsonStructureException(parentPath + " must contain the key \"" + name + "\""); + } + + return settle(parent[name], parentPath + "." + name); + } + + JsonArray& settle(const QJsonValue& parent, const QString& parentPath, const QString& name, + JsonMandatory mandatory = JsonMandatory::Yes) { + if (!parent.isObject()) { + throw JsonStructureException(parentPath + " must be an {Array}"); + } + return settle(parent.toObject(), parentPath, name, mandatory); + } + + private: + QList::TypeOuter> data_; + + void inflate(const QString& path, const QJsonArray& json) { + data_.reserve(json.count()); + int i = 0; + auto it = json.begin(); + while (it != json.end()) { + typename json::JsonArrayTraits::TypeInner item; + item.settle(*it, path + "[" + QString::number(i) + "]"); + data_.append(json::JsonArrayTraits::getValue(item)); + ++it; + i++; + } + } + }; +} From 472206e840fad6529111a30ac20fecf903930603 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Tue, 7 Dec 2021 21:02:56 +0300 Subject: [PATCH 02/11] Pack command now supports reading pack configuration from pbo.json or prefix files --- .../task/__test__/packconfiguration_test.cpp | 74 ++++++++++++++++- pbom/model/task/__test__/packoptions_test.cpp | 12 +++ pbom/model/task/packconfiguration.cpp | 79 +++++++++++++++---- pbom/model/task/packconfiguration.h | 15 +++- pbom/model/task/packoptions.cpp | 3 + pbom/model/task/packtask.cpp | 13 +++ 6 files changed, 177 insertions(+), 19 deletions(-) diff --git a/pbom/model/task/__test__/packconfiguration_test.cpp b/pbom/model/task/__test__/packconfiguration_test.cpp index cc23e51..32dfbec 100644 --- a/pbom/model/task/__test__/packconfiguration_test.cpp +++ b/pbom/model/task/__test__/packconfiguration_test.cpp @@ -8,7 +8,7 @@ namespace pboman3::model::task::test { using namespace domain; - TEST(PackConfigurationTest, Apply_Removes_PboJson_Node) { + TEST(PackConfigurationTest, Apply_Removes_All_Config_Nodes) { QTemporaryFile json; json.open(); json.write(QByteArray("{}")); @@ -18,11 +18,17 @@ namespace pboman3::model::task::test { document.root()->createHierarchy(PboPath({"f1.txt"})); PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$prefix$"})); + prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* version = document.root()->createHierarchy(PboPath({"$version$"})); + version->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* product = document.root()->createHierarchy(PboPath({"$product$"})); + product->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); const PackConfiguration packConfiguration(&document); packConfiguration.apply(); - ASSERT_EQ(document.root()->count(), 1); //pbo.json removed + ASSERT_EQ(document.root()->count(), 1); //config files removed ASSERT_TRUE(document.root()->get(PboPath({"f1.txt"}))); //but others are in places } @@ -34,8 +40,16 @@ namespace pboman3::model::task::test { json.close(); PboDocument document("file.pbo"); + //this must be applied PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + //these must not be applied + PboNode* prefix = document.root()->createHierarchy(PboPath({"$prefix$"})); + prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* version = document.root()->createHierarchy(PboPath({"$version$"})); + version->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* product = document.root()->createHierarchy(PboPath({"$product$"})); + product->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); const PackConfiguration packConfiguration(&document); packConfiguration.apply(); @@ -47,6 +61,42 @@ namespace pboman3::model::task::test { ASSERT_EQ(document.headers()->at(1)->value(), "v2"); } + TEST(PackConfigurationTest, Apply_Sets_Headers_From_Prefix_Files) { + QTemporaryFile pref; + pref.open(); + pref.write(QByteArray("pref1")); + pref.close(); + + QTemporaryFile prod; + prod.open(); + prod.write(QByteArray("prod1")); + prod.close(); + + QTemporaryFile ver; + ver.open(); + ver.write(QByteArray("ver1")); + ver.close(); + + PboDocument document("file.pbo"); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$prEfix$"})); + prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(pref.fileName())); + PboNode* product = document.root()->createHierarchy(PboPath({"$prOduct$"})); + product->binarySource = QSharedPointer(new io::FsRawBinarySource(prod.fileName())); + PboNode* version = document.root()->createHierarchy(PboPath({"$veRsion$"})); + version->binarySource = QSharedPointer(new io::FsRawBinarySource(ver.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_EQ(document.headers()->count(), 3); + ASSERT_EQ(document.headers()->at(0)->name(), "prefix"); + ASSERT_EQ(document.headers()->at(0)->value(), "pref1"); + ASSERT_EQ(document.headers()->at(1)->name(), "product"); + ASSERT_EQ(document.headers()->at(1)->value(), "prod1"); + ASSERT_EQ(document.headers()->at(2)->name(), "version"); + ASSERT_EQ(document.headers()->at(2)->value(), "ver1"); + } + TEST(PackConfigurationTest, Apply_Compresses_Only_Included_Files) { QTemporaryFile json; json.open(); @@ -153,4 +203,24 @@ namespace pboman3::model::task::test { ASSERT_EQ(document.headers()->at(1)->name(), "p2"); ASSERT_EQ(document.headers()->at(1)->value(), "v2"); } + + TEST(PackConfigurationTest, Apply_Throws_If_Prefix_Files_Non_Text) { + constexpr char data[]{0x01, 0x02, 0x03, 0x04, 0x00, 0x01}; + QTemporaryFile pref; + pref.open(); + pref.write(data, sizeof data); + pref.close(); + + PboDocument document("file.pbo"); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$prefix$"})); + prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(pref.fileName())); + + try { + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + FAIL() << "Should have not reached this line"; + } catch (const PrefixEncodingException& ex) { + ASSERT_EQ(ex.message(), "$prefix$"); + } + } } diff --git a/pbom/model/task/__test__/packoptions_test.cpp b/pbom/model/task/__test__/packoptions_test.cpp index fc0a627..28a3b92 100644 --- a/pbom/model/task/__test__/packoptions_test.cpp +++ b/pbom/model/task/__test__/packoptions_test.cpp @@ -45,4 +45,16 @@ namespace pboman3::model::task::test { ASSERT_EQ(config.compress().include().count(), 0); ASSERT_EQ(config.compress().exclude().count(), 0); } + + TEST(PackOptionsTest, Settle_Throws_If_Header_Name_Is_Empty) { + const QJsonDocument json = QJsonDocument::fromJson("{\"headers\":[{\"name\":\"\"}]}"); + + try { + PackOptions config; + config.settle(json.object(), ""); + FAIL() << "Should have not reached this line"; + } catch (const JsonStructureException& ex) { + ASSERT_EQ(".headers[0].name must not be an empty string", ex.message()); + } + } } diff --git a/pbom/model/task/packconfiguration.cpp b/pbom/model/task/packconfiguration.cpp index 0d6c071..4cc199f 100644 --- a/pbom/model/task/packconfiguration.cpp +++ b/pbom/model/task/packconfiguration.cpp @@ -6,35 +6,69 @@ #include "domain/func.h" #include "io/diskaccessexception.h" #include "model/binarysourceutils.h" +#include "util/log.h" + +#define LOG(...) LOGGER("model/task/PackConfiguration", __VA_ARGS__) namespace pboman3::model::task { + PrefixEncodingException::PrefixEncodingException(QString prefixFileName) + : AppException(std::move(prefixFileName)) { + } + + void PrefixEncodingException::raise() const { + throw *this; + } + + QException* PrefixEncodingException::clone() const { + return new PrefixEncodingException(*this); + } + + QDebug& operator<<(QDebug& debug, const PrefixEncodingException& ex) { + return debug << "PrefixEncodingException(Message=" << ex.message_ << ")"; + } + + PackConfiguration::PackConfiguration(PboDocument* document) : document_(document) { } +#define APPLY_NODE_CONTENT_AS_HEADER(V,P) if (!P && V) { applyNodeContentAsHeader(V, #V); } +#define CLEANUP_NODE(N) if (N) { (N)->removeFromHierarchy(); } + void PackConfiguration::apply() const { PboNode* pboJson = FindDirectChild(document_->root(), "pbo.json"); if (pboJson) { + LOG(info, "Apply configuration from pbo.json") const PackOptions packOptions = readPackOptions(pboJson); applyDocumentHeaders(packOptions); const CompressionRules compressionRules = buildCompressionRules(packOptions); applyDocumentCompressionRules(document_->root(), compressionRules); } - if (!pboJson) { - applyFileContentAsHeader("prefix"); - applyFileContentAsHeader("product"); - applyFileContentAsHeader("version"); - } - if (pboJson) - pboJson->removeFromHierarchy(); + + PboNode* prefix = FindDirectChild(document_->root(), "$prefix$"); + APPLY_NODE_CONTENT_AS_HEADER(prefix, pboJson) + PboNode* product = FindDirectChild(document_->root(), "$product$"); + APPLY_NODE_CONTENT_AS_HEADER(product, pboJson) + PboNode* version = FindDirectChild(document_->root(), "$version$"); + APPLY_NODE_CONTENT_AS_HEADER(version, pboJson) + + CLEANUP_NODE(pboJson) + CLEANUP_NODE(prefix) + CLEANUP_NODE(product) + CLEANUP_NODE(version) } void PackConfiguration::applyDocumentHeaders(const PackOptions& options) const { - if (options.headers().isEmpty()) + if (options.headers().isEmpty()) { + LOG(info, "No headers defined in the config") return; + } + + LOG(info, options.headers().count(), "headers defined in the config") const QSharedPointer tran = document_->headers()->beginTransaction(); for (const PackHeader& header : options.headers()) { + LOG(debug, "Header: ", header.name(), "|", header.value()) tran->add(header.name(), header.value()); } @@ -76,7 +110,9 @@ namespace pboman3::model::task { PackConfiguration::CompressionRules PackConfiguration::buildCompressionRules(const PackOptions& options) { CompressionRules rules; + LOG(info, "Building include rules") convertToCompressionRules(options.compress().include(), &rules.include); + LOG(info, "Building exclude rules") convertToCompressionRules(options.compress().exclude(), &rules.exclude); return rules; } @@ -88,6 +124,7 @@ namespace pboman3::model::task { QRegularExpression reg( rule, QRegularExpression::CaseInsensitiveOption | QRegularExpression::DontCaptureOption); if (!reg.isValid()) { + LOG(warning, "Compression rule is invalid - throwing:", rule) throw JsonStructureException( "The regular expression \"" + reg.pattern() + "\" is invalid: " + reg.errorString()); } @@ -100,9 +137,11 @@ namespace pboman3::model::task { QJsonParseError err; const QJsonDocument json = QJsonDocument::fromJson(data, &err); if (json.isNull()) { + LOG(warning, "Json was null when read - throwing") throw JsonStructureException(err.errorString() + " at offset " + QString::number(err.offset)); } if (!json.isObject()) { + LOG(warning, "Json root was not an object - throwing:", json) throw JsonStructureException("The json must contain an object"); } @@ -114,6 +153,7 @@ namespace pboman3::model::task { QByteArray PackConfiguration::readNodeContent(const PboNode* node) { QFile f(node->binarySource->path()); if (!f.open(QIODeviceBase::ReadOnly)) { + LOG(warning, "Could not open the file - throwing:", node->binarySource->path()) throw io::DiskAccessException("Could not read the file", f.fileName()); } QByteArray data = f.readAll(); @@ -121,14 +161,21 @@ namespace pboman3::model::task { return data; } - void PackConfiguration::applyFileContentAsHeader(const QString& prefix) const { - PboNode* node = FindDirectChild(document_->root(), "$" + prefix + "$"); - if (node) { - const QByteArray data = readNodeContent(node); - const QSharedPointer tran = document_->headers()->beginTransaction(); - tran->add(prefix, data); - tran->commit(); - node->removeFromHierarchy(); + void PackConfiguration::applyNodeContentAsHeader(const PboNode* node, const QString& prefix) const { + LOG(info, "Apply prefix:", prefix) + const QByteArray data = readNodeContent(node); + throwIfBreaksPbo(node, data); + const QSharedPointer tran = document_->headers()->beginTransaction(); + tran->add(prefix, data); + tran->commit(); + } + + void PackConfiguration::throwIfBreaksPbo(const PboNode* node, const QByteArray& data) { + for (const char& byte : data) { + if (byte == 0) { + LOG(warning, "The prefix file is invalid - throwing:", node->title()) + throw PrefixEncodingException(node->title()); + } } } } diff --git a/pbom/model/task/packconfiguration.h b/pbom/model/task/packconfiguration.h index cf58510..71890a8 100644 --- a/pbom/model/task/packconfiguration.h +++ b/pbom/model/task/packconfiguration.h @@ -6,6 +6,17 @@ namespace pboman3::model::task { using namespace domain; + class PrefixEncodingException : public AppException { + public: + PrefixEncodingException(QString prefixFileName); + + void raise() const override; + + QException* clone() const override; + + friend QDebug& operator<<(QDebug& debug, const PrefixEncodingException& ex); + }; + class PackConfiguration { public: explicit PackConfiguration(PboDocument* document); @@ -36,6 +47,8 @@ namespace pboman3::model::task { QList exclude; }; - void applyFileContentAsHeader(const QString& prefix) const; + void applyNodeContentAsHeader(const PboNode* node, const QString& prefix) const; + + static void throwIfBreaksPbo(const PboNode* node, const QByteArray& data); }; } diff --git a/pbom/model/task/packoptions.cpp b/pbom/model/task/packoptions.cpp index a8d827b..8432666 100644 --- a/pbom/model/task/packoptions.cpp +++ b/pbom/model/task/packoptions.cpp @@ -28,6 +28,9 @@ namespace pboman3::model::task { void PackHeader::inflate(const QString& path, const QJsonObject& json) { name_ = JsonValue().settle(json, path, "name").value(); + if (name_.isEmpty()) { + throw JsonStructureException(path + ".name must not be an empty string"); + } value_ = JsonValue().settle(json, path, "value").value(); } diff --git a/pbom/model/task/packtask.cpp b/pbom/model/task/packtask.cpp index 570284c..3b82fd3 100644 --- a/pbom/model/task/packtask.cpp +++ b/pbom/model/task/packtask.cpp @@ -1,4 +1,6 @@ #include "packtask.h" + +#include "packconfiguration.h" #include "io/diskaccessexception.h" #include "io/documentwriter.h" #include "util/log.h" @@ -40,6 +42,17 @@ namespace pboman3::model { return; } + try { + const task::PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + } catch (const JsonStructureException& ex) { + emit taskMessage("Failure | pbo.json malformed | " + ex.message()); + return; + } catch (const task::PrefixEncodingException& ex) { + emit taskMessage("Failure | " + ex.message() + " | The file has unsupported encoding"); + return; + } + DocumentWriter writer(pboFile); //it is tricky to display real PBO pack progress as the process consists of four independent steps. From fbdbcffdf27a1d5c5b8d6dd821a28b022f73e907 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Wed, 8 Dec 2021 18:48:11 +0300 Subject: [PATCH 03/11] Fixed missing namespaces --- pbom/model/task/packtask.cpp | 2 +- pbom/model/task/packtask.h | 2 +- pbom/model/task/packwindowmodel.cpp | 2 +- pbom/model/task/packwindowmodel.h | 2 +- pbom/model/task/task.h | 2 +- pbom/model/task/taskwindowmodel.cpp | 2 +- pbom/model/task/taskwindowmodel.h | 2 +- pbom/model/task/unpacktask.cpp | 2 +- pbom/model/task/unpacktask.h | 2 +- pbom/model/task/unpackwindowmodel.cpp | 2 +- pbom/model/task/unpackwindowmodel.h | 2 +- pbom/ui/taskwindow.h | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pbom/model/task/packtask.cpp b/pbom/model/task/packtask.cpp index 3b82fd3..b1391f1 100644 --- a/pbom/model/task/packtask.cpp +++ b/pbom/model/task/packtask.cpp @@ -7,7 +7,7 @@ #define LOG(...) LOGGER("model/task/PackTask", __VA_ARGS__) -namespace pboman3::model { +namespace pboman3::model::task { using namespace io; PackTask::PackTask(QString folder, QString outputDir) diff --git a/pbom/model/task/packtask.h b/pbom/model/task/packtask.h index 6e99063..0dc3497 100644 --- a/pbom/model/task/packtask.h +++ b/pbom/model/task/packtask.h @@ -4,7 +4,7 @@ #include "task.h" #include "model/interactionparcel.h" -namespace pboman3::model { +namespace pboman3::model::task { using namespace domain; class PackTask : public Task { diff --git a/pbom/model/task/packwindowmodel.cpp b/pbom/model/task/packwindowmodel.cpp index 3ebd040..e455d95 100644 --- a/pbom/model/task/packwindowmodel.cpp +++ b/pbom/model/task/packwindowmodel.cpp @@ -1,7 +1,7 @@ #include "packwindowmodel.h" #include "packtask.h" -namespace pboman3::model { +namespace pboman3::model::task { PackWindowModel::PackWindowModel(const QStringList& folders, const QString& outputDir) { for (const QString& folder : folders) { QSharedPointer task(new PackTask(folder, outputDir)); diff --git a/pbom/model/task/packwindowmodel.h b/pbom/model/task/packwindowmodel.h index f9d1dbf..48e2757 100644 --- a/pbom/model/task/packwindowmodel.h +++ b/pbom/model/task/packwindowmodel.h @@ -2,7 +2,7 @@ #include "taskwindowmodel.h" -namespace pboman3::model { +namespace pboman3::model::task { class PackWindowModel: public TaskWindowModel{ public: PackWindowModel(const QStringList& folders, const QString& outputDir); diff --git a/pbom/model/task/task.h b/pbom/model/task/task.h index 91f03d8..ef10541 100644 --- a/pbom/model/task/task.h +++ b/pbom/model/task/task.h @@ -3,7 +3,7 @@ #include #include "util/util.h" -namespace pboman3::model { +namespace pboman3::model::task { using namespace util; class Task : public QObject { diff --git a/pbom/model/task/taskwindowmodel.cpp b/pbom/model/task/taskwindowmodel.cpp index be680af..148dd8c 100644 --- a/pbom/model/task/taskwindowmodel.cpp +++ b/pbom/model/task/taskwindowmodel.cpp @@ -7,7 +7,7 @@ #define LOG(...) LOGGER("model/task/TaskWindowModel", __VA_ARGS__) -namespace pboman3::model { +namespace pboman3::model::task { void TaskWindowModel::start() { const int numThreads = std::min(QThread::idealThreadCount(), static_cast(tasks_.count())); for (int i = 0; i < numThreads; i++) { diff --git a/pbom/model/task/taskwindowmodel.h b/pbom/model/task/taskwindowmodel.h index bf4f3f2..2bf92ef 100644 --- a/pbom/model/task/taskwindowmodel.h +++ b/pbom/model/task/taskwindowmodel.h @@ -6,7 +6,7 @@ #include #include "task.h" -namespace pboman3::model { +namespace pboman3::model::task { typedef qint32 ThreadId; class TaskWindowModel : public QObject { diff --git a/pbom/model/task/unpacktask.cpp b/pbom/model/task/unpacktask.cpp index b90fbe2..dab4932 100644 --- a/pbom/model/task/unpacktask.cpp +++ b/pbom/model/task/unpacktask.cpp @@ -13,7 +13,7 @@ #define LOG(...) LOGGER("model/task/UnpackTask", __VA_ARGS__) -namespace pboman3::model { +namespace pboman3::model::task { using namespace io; UnpackTask::UnpackTask(QString pboPath, const QString& outputDir) diff --git a/pbom/model/task/unpacktask.h b/pbom/model/task/unpacktask.h index 544e48d..76120df 100644 --- a/pbom/model/task/unpacktask.h +++ b/pbom/model/task/unpacktask.h @@ -4,7 +4,7 @@ #include "task.h" #include "domain/pbodocument.h" -namespace pboman3::model { +namespace pboman3::model::task { using namespace domain; class UnpackTask : public Task { diff --git a/pbom/model/task/unpackwindowmodel.cpp b/pbom/model/task/unpackwindowmodel.cpp index b0e37b6..fbd6efb 100644 --- a/pbom/model/task/unpackwindowmodel.cpp +++ b/pbom/model/task/unpackwindowmodel.cpp @@ -1,7 +1,7 @@ #include "unpackwindowmodel.h" #include "unpacktask.h" -namespace pboman3::model { +namespace pboman3::model::task { UnpackWindowModel::UnpackWindowModel(const QStringList& pboFiles, const QString& outputDir) { for (const QString& pboFile : pboFiles) { QSharedPointer task(new UnpackTask(pboFile, outputDir)); diff --git a/pbom/model/task/unpackwindowmodel.h b/pbom/model/task/unpackwindowmodel.h index 4f98477..bfbc252 100644 --- a/pbom/model/task/unpackwindowmodel.h +++ b/pbom/model/task/unpackwindowmodel.h @@ -2,7 +2,7 @@ #include "taskwindowmodel.h" -namespace pboman3::model { +namespace pboman3::model::task { class UnpackWindowModel: public TaskWindowModel{ public: UnpackWindowModel(const QStringList& pboFiles, const QString& outputDir); diff --git a/pbom/ui/taskwindow.h b/pbom/ui/taskwindow.h index a417902..c31154c 100644 --- a/pbom/ui/taskwindow.h +++ b/pbom/ui/taskwindow.h @@ -12,7 +12,7 @@ namespace Ui { } namespace pboman3::ui { - using namespace model; + using namespace model::task; class TaskWindow : public QMainWindow { Q_OBJECT From 8d2fac957060e4534eef14bedf00bc157f714974 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Wed, 8 Dec 2021 23:10:01 +0300 Subject: [PATCH 04/11] Working on rebuilding headers/compression by existing PBO --- pbom/model/CMakeLists.txt | 2 + pbom/model/binarysourceutils.h | 7 ++ .../__test__/extractconfiguration_test.cpp | 112 ++++++++++++++++++ pbom/model/task/__test__/packoptions_test.cpp | 36 +++--- pbom/model/task/extractconfiguration.cpp | 65 ++++++++++ pbom/model/task/extractconfiguration.h | 24 ++++ pbom/model/task/packconfiguration.cpp | 16 +-- pbom/model/task/packoptions.cpp | 41 ++----- pbom/model/task/packoptions.h | 29 ++--- 9 files changed, 256 insertions(+), 76 deletions(-) create mode 100644 pbom/model/task/__test__/extractconfiguration_test.cpp create mode 100644 pbom/model/task/extractconfiguration.cpp create mode 100644 pbom/model/task/extractconfiguration.h diff --git a/pbom/model/CMakeLists.txt b/pbom/model/CMakeLists.txt index 99aeba7..eb3e341 100644 --- a/pbom/model/CMakeLists.txt +++ b/pbom/model/CMakeLists.txt @@ -1,4 +1,5 @@ list(APPEND PROJECT_SOURCES + "model/task/extractconfiguration.cpp" "model/task/packconfiguration.cpp" "model/task/packoptions.cpp" "model/task/packtask.cpp" @@ -14,6 +15,7 @@ list(APPEND PROJECT_SOURCES set(PROJECT_SOURCES ${PROJECT_SOURCES} PARENT_SCOPE) list(APPEND TEST_SOURCES + "model/task/__test__/extractconfiguration_test.cpp" "model/task/__test__/packconfiguration_test.cpp" "model/task/__test__/packoptions_test.cpp" "model/__test__/conflictsparcel_test.cpp" diff --git a/pbom/model/binarysourceutils.h b/pbom/model/binarysourceutils.h index faae641..ccac28c 100644 --- a/pbom/model/binarysourceutils.h +++ b/pbom/model/binarysourceutils.h @@ -27,4 +27,11 @@ namespace pboman3::model { } } } + + inline bool IsCompressed(const QSharedPointer& bs) { + if (const auto pboBs = dynamic_cast(bs.get())) { + return pboBs->isCompressed(); + } + throw InvalidOperationException("Can't query compression status"); + } } diff --git a/pbom/model/task/__test__/extractconfiguration_test.cpp b/pbom/model/task/__test__/extractconfiguration_test.cpp new file mode 100644 index 0000000..d6b498f --- /dev/null +++ b/pbom/model/task/__test__/extractconfiguration_test.cpp @@ -0,0 +1,112 @@ +#include "model/task/extractconfiguration.h" + +#include +#include +#include "domain/pbodocument.h" +#include "io/bs/pbobinarysource.h" + +namespace pboman3::model::task { + using namespace domain; + + TEST(ExtractConfigurationTest, Extract_Takes_Headers) { + const QList headers{ + QSharedPointer(new DocumentHeader("n1", "v1")), + QSharedPointer(new DocumentHeader("n2", "v2")), + QSharedPointer(new DocumentHeader("n3", "")), + }; + const PboDocument document("file.pbo", headers, QByteArray(1, 20)); + + const PackOptions options = ExtractConfiguration::extractFrom(document); + + ASSERT_EQ(options.headers.count(), 3); + ASSERT_EQ(options.headers.at(0).name, "n1"); + ASSERT_EQ(options.headers.at(0).value, "v1"); + ASSERT_EQ(options.headers.at(1).name, "n2"); + ASSERT_EQ(options.headers.at(1).value, "v2"); + ASSERT_EQ(options.headers.at(2).name, "n3"); + ASSERT_EQ(options.headers.at(2).value, ""); + } + + struct ExtractConfigurationExtParam { + QString fileWithExt; + }; + + class ExtractConfigurationExtensionTest : public testing::TestWithParam { + }; + + TEST_P(ExtractConfigurationExtensionTest, Extract_Picks_Extension_Compression) { + const PboDocument document("file.pbo"); + document.root()->createHierarchy(PboPath("f1.p3d")); + document.root()->createHierarchy(PboPath("f2.paa")); + document.root()->createHierarchy(PboPath("snd/f3.ogg")); + document.root()->createHierarchy(PboPath(GetParam().fileWithExt)); + + const PackOptions options = ExtractConfiguration::extractFrom(document); + + ASSERT_EQ(options.compress.include.count(), 1); + ASSERT_EQ(options.compress.include.at(0), "\\." + GetFileExtension(GetParam().fileWithExt).toLower() + "$"); + } + + INSTANTIATE_TEST_SUITE_P(ExtractConfigurationTest, ExtractConfigurationExtensionTest, testing::Values( + ExtractConfigurationExtParam{"file1.sqf"}, + ExtractConfigurationExtParam{"folder1/file1.sqf"}, + ExtractConfigurationExtParam{"file1.sqs"}, + ExtractConfigurationExtParam{"file1.txt"}, + ExtractConfigurationExtParam{"file1.Xml"}, + ExtractConfigurationExtParam{"file1.cSv"} + )); + + struct ExtractConfigurationFileParam { + QString fileName; + bool compressed; + }; + + class ExtractConfigurationFileTest : public testing::TestWithParam { + }; + + TEST_P(ExtractConfigurationFileTest, Extract_Picks_File_Compression) { + QTemporaryFile file; + file.open(); + file.close(); + + const PboDocument document("file.pbo"); + PboNode* node = document.root()->createHierarchy(PboPath(GetParam().fileName)); + node->binarySource = QSharedPointer( + new io::PboBinarySource(file.fileName(), io::PboDataInfo{10, 10, 0, 0, GetParam().compressed})); + + const PackOptions options = ExtractConfiguration::extractFrom(document); + + if (GetParam().compressed) { + ASSERT_EQ(options.compress.include.count(), 1); + ASSERT_EQ(options.compress.include.at(0), "^" + GetParam().fileName.toLower() + "$"); + } else { + ASSERT_EQ(options.compress.include.count(), 0); + } + } + + INSTANTIATE_TEST_SUITE_P(ExtractConfigurationTest, ExtractConfigurationFileTest, testing::Values( + ExtractConfigurationFileParam{"mission.sQm", true}, + ExtractConfigurationFileParam{"mission.sqm", false}, + ExtractConfigurationFileParam{"descriptIon.ext", true}, + ExtractConfigurationFileParam{"description.ext", false} + )); + + TEST(ExtractConfigurationTest, Extract_Picks_Multiple_Compression_Rules) { + QTemporaryFile file; + file.open(); + file.close(); + + const PboDocument document("file.pbo"); + PboNode* node = document.root()->createHierarchy(PboPath("mission.sqm")); + node->binarySource = QSharedPointer( + new io::PboBinarySource(file.fileName(), io::PboDataInfo{10, 10, 0, 0, true})); + + document.root()->createHierarchy(PboPath("script.sqf")); + + const PackOptions options = ExtractConfiguration::extractFrom(document); + + ASSERT_EQ(options.compress.include.count(), 2); + ASSERT_EQ(options.compress.include.at(0), "\\.sqf$"); + ASSERT_EQ(options.compress.include.at(1), "^mission.sqm$"); + } +} diff --git a/pbom/model/task/__test__/packoptions_test.cpp b/pbom/model/task/__test__/packoptions_test.cpp index 28a3b92..15521c8 100644 --- a/pbom/model/task/__test__/packoptions_test.cpp +++ b/pbom/model/task/__test__/packoptions_test.cpp @@ -10,18 +10,18 @@ namespace pboman3::model::task::test { PackOptions config; config.settle(json.object(), ""); - ASSERT_EQ(config.headers().count(), 2); - ASSERT_EQ(config.headers().at(0).name(), "n1"); - ASSERT_EQ(config.headers().at(0).value(), "v1"); - ASSERT_EQ(config.headers().at(1).name(), "n2"); - ASSERT_EQ(config.headers().at(1).value(), "v2"); - - ASSERT_EQ(config.compress().include().count(), 2); - ASSERT_EQ(config.compress().include().at(0), "i1"); - ASSERT_EQ(config.compress().include().at(1), "i2"); - ASSERT_EQ(config.compress().exclude().count(), 2); - ASSERT_EQ(config.compress().exclude().at(0), "e1"); - ASSERT_EQ(config.compress().exclude().at(1), "e2"); + ASSERT_EQ(config.headers.count(), 2); + ASSERT_EQ(config.headers.at(0).name, "n1"); + ASSERT_EQ(config.headers.at(0).value, "v1"); + ASSERT_EQ(config.headers.at(1).name, "n2"); + ASSERT_EQ(config.headers.at(1).value, "v2"); + + ASSERT_EQ(config.compress.include.count(), 2); + ASSERT_EQ(config.compress.include.at(0), "i1"); + ASSERT_EQ(config.compress.include.at(1), "i2"); + ASSERT_EQ(config.compress.exclude.count(), 2); + ASSERT_EQ(config.compress.exclude.at(0), "e1"); + ASSERT_EQ(config.compress.exclude.at(1), "e2"); } TEST(PackOptionsTest, Settle_Reads_Empty_Configuration) { @@ -30,9 +30,9 @@ namespace pboman3::model::task::test { PackOptions config; config.settle(json.object(), ""); - ASSERT_EQ(config.headers().count(), 0); - ASSERT_EQ(config.compress().include().count(), 0); - ASSERT_EQ(config.compress().exclude().count(), 0); + ASSERT_EQ(config.headers.count(), 0); + ASSERT_EQ(config.compress.include.count(), 0); + ASSERT_EQ(config.compress.exclude.count(), 0); } TEST(PackOptionsTest, Settle_Reads_Empty_Compression) { @@ -41,9 +41,9 @@ namespace pboman3::model::task::test { PackOptions config; config.settle(json.object(), ""); - ASSERT_EQ(config.headers().count(), 0); - ASSERT_EQ(config.compress().include().count(), 0); - ASSERT_EQ(config.compress().exclude().count(), 0); + ASSERT_EQ(config.headers.count(), 0); + ASSERT_EQ(config.compress.include.count(), 0); + ASSERT_EQ(config.compress.exclude.count(), 0); } TEST(PackOptionsTest, Settle_Throws_If_Header_Name_Is_Empty) { diff --git a/pbom/model/task/extractconfiguration.cpp b/pbom/model/task/extractconfiguration.cpp new file mode 100644 index 0000000..1a78a15 --- /dev/null +++ b/pbom/model/task/extractconfiguration.cpp @@ -0,0 +1,65 @@ +#include "extractconfiguration.h" +#include "domain/func.h" +#include "model/binarysourceutils.h" + +namespace pboman3::model::task { + PackOptions ExtractConfiguration::extractFrom(const PboDocument& document) { + PackOptions options; + + extractHeaders(document, options); + extractCompressionRules(document, options); + + return options; + } + + void ExtractConfiguration::extractHeaders(const PboDocument& document, PackOptions& options) { + for (const DocumentHeader* header : *document.headers()) { + options.headers.append(PackHeader(header->name(), header->value())); + } + } + + constexpr const char* extensions[] = {"sqf", "sqs", "txt", "xml", "csv"}; + constexpr const char* files[] = {"mission.sqm", "description.ext"}; + + void ExtractConfiguration::extractCompressionRules(const PboDocument& document, PackOptions& options) { + QSet artifacts; + artifacts.reserve(10); + collectValuableArtifacts(document.root(), artifacts); + + for (const QString ext : extensions) { + if (artifacts.contains(ext)) { + const QString rule = makeExtensionCompressionRule(ext); + options.compress.include.append(rule); + } + } + for (const QString file : files) { + if (const PboNode* node = FindDirectChild(document.root(), file); node) { + if (IsCompressed(node->binarySource)) { + const QString rule = makeFileCompressionRule(file); + options.compress.include.append(rule); + } + } + } + } + + void ExtractConfiguration::collectValuableArtifacts(const PboNode* node, QSet& results) { + const QString title = node->title().toLower(); + if (node->nodeType() == PboNodeType::File) { + const QString ext = GetFileExtension(title); + results.insert(ext); + } else { + for (const PboNode* child : *node) { + collectValuableArtifacts(child, results); + } + } + } + + QString ExtractConfiguration::makeExtensionCompressionRule(const QString& ext) { + return "\\." + ext + "$"; + } + + QString ExtractConfiguration::makeFileCompressionRule(const QString& fileName) { + return "^" + fileName + "$"; + } + +} diff --git a/pbom/model/task/extractconfiguration.h b/pbom/model/task/extractconfiguration.h new file mode 100644 index 0000000..1581641 --- /dev/null +++ b/pbom/model/task/extractconfiguration.h @@ -0,0 +1,24 @@ +#pragma once + +#include "packoptions.h" +#include "domain/pbodocument.h" + +namespace pboman3::model::task { + using namespace domain; + + class ExtractConfiguration { + public: + static PackOptions extractFrom(const PboDocument& document); + + private: + static void extractHeaders(const PboDocument& document, PackOptions& options); + + static void extractCompressionRules(const PboDocument& document, PackOptions& options); + + static void collectValuableArtifacts(const PboNode* node, QSet& results); + + static QString makeExtensionCompressionRule(const QString& ext); + + static QString makeFileCompressionRule(const QString& fileName); + }; +} diff --git a/pbom/model/task/packconfiguration.cpp b/pbom/model/task/packconfiguration.cpp index 4cc199f..71a959a 100644 --- a/pbom/model/task/packconfiguration.cpp +++ b/pbom/model/task/packconfiguration.cpp @@ -32,7 +32,7 @@ namespace pboman3::model::task { : document_(document) { } -#define APPLY_NODE_CONTENT_AS_HEADER(V,P) if (!P && V) { applyNodeContentAsHeader(V, #V); } +#define APPLY_NODE_CONTENT_AS_HEADER(V,P) if (!(P) && (V)) { applyNodeContentAsHeader(V, #V); } #define CLEANUP_NODE(N) if (N) { (N)->removeFromHierarchy(); } void PackConfiguration::apply() const { @@ -59,17 +59,17 @@ namespace pboman3::model::task { } void PackConfiguration::applyDocumentHeaders(const PackOptions& options) const { - if (options.headers().isEmpty()) { + if (options.headers.isEmpty()) { LOG(info, "No headers defined in the config") return; } - LOG(info, options.headers().count(), "headers defined in the config") + LOG(info, options.headers.count(), "headers defined in the config") const QSharedPointer tran = document_->headers()->beginTransaction(); - for (const PackHeader& header : options.headers()) { - LOG(debug, "Header: ", header.name(), "|", header.value()) - tran->add(header.name(), header.value()); + for (const PackHeader& header : options.headers) { + LOG(debug, "Header: ", header.name, "|", header.value) + tran->add(header.name, header.value); } tran->commit(); @@ -111,9 +111,9 @@ namespace pboman3::model::task { PackConfiguration::CompressionRules PackConfiguration::buildCompressionRules(const PackOptions& options) { CompressionRules rules; LOG(info, "Building include rules") - convertToCompressionRules(options.compress().include(), &rules.include); + convertToCompressionRules(options.compress.include, &rules.include); LOG(info, "Building exclude rules") - convertToCompressionRules(options.compress().exclude(), &rules.exclude); + convertToCompressionRules(options.compress.exclude, &rules.exclude); return rules; } diff --git a/pbom/model/task/packoptions.cpp b/pbom/model/task/packoptions.cpp index 8432666..d4989f7 100644 --- a/pbom/model/task/packoptions.cpp +++ b/pbom/model/task/packoptions.cpp @@ -4,47 +4,28 @@ #include namespace pboman3::model::task { - const QList& CompressOptions::include() const { - return include_; - } - - const QList& CompressOptions::exclude() const { - return exclude_; - } - void CompressOptions::inflate(const QString& path, const QJsonObject& json) { - include_ = JsonArray>().settle(json, path, "include", JsonMandatory::No).data(); - exclude_ = JsonArray>().settle(json, path, "exclude", JsonMandatory::No).data(); + include = JsonArray>().settle(json, path, "include", JsonMandatory::No).data(); + exclude = JsonArray>().settle(json, path, "exclude", JsonMandatory::No).data(); } + PackHeader::PackHeader() = default; - const QString& PackHeader::name() const { - return name_; - } - - const QString& PackHeader::value() const { - return value_; + PackHeader::PackHeader(QString name, QString value) + : name(std::move(name)), + value(std::move(value)) { } void PackHeader::inflate(const QString& path, const QJsonObject& json) { - name_ = JsonValue().settle(json, path, "name").value(); - if (name_.isEmpty()) { + name = JsonValue().settle(json, path, "name").value(); + if (name.isEmpty()) { throw JsonStructureException(path + ".name must not be an empty string"); } - value_ = JsonValue().settle(json, path, "value").value(); - } - - - const QList& PackOptions::headers() const { - return headers_; - } - - const CompressOptions& PackOptions::compress() const { - return compress_; + value = JsonValue().settle(json, path, "value").value(); } void PackOptions::inflate(const QString& path, const QJsonObject& json) { - headers_ = JsonArray().settle(json, path, "headers", JsonMandatory::No).data(); - compress_.settle(json, path, "compress", JsonMandatory::No); + headers = JsonArray().settle(json, path, "headers", JsonMandatory::No).data(); + compress.settle(json, path, "compress", JsonMandatory::No); } } diff --git a/pbom/model/task/packoptions.h b/pbom/model/task/packoptions.h index ed77f05..f9600aa 100644 --- a/pbom/model/task/packoptions.h +++ b/pbom/model/task/packoptions.h @@ -7,43 +7,32 @@ namespace pboman3::model::task { class CompressOptions : public JsonObject { public: - const QList& include() const; - - const QList& exclude() const; + QList include; + QList exclude; protected: void inflate(const QString& path, const QJsonObject& json) override; - - private: - QList include_; - QList exclude_; }; class PackHeader : public JsonObject { public: - const QString& name() const; + PackHeader(); + + PackHeader(QString name, QString value); - const QString& value() const; + QString name; + QString value; protected: void inflate(const QString& path, const QJsonObject& json) override; - - private: - QString name_; - QString value_; }; class PackOptions : public JsonObject { public: - const QList& headers() const; - - const CompressOptions& compress() const; + QList headers; + CompressOptions compress; protected: void inflate(const QString& path, const QJsonObject& json) override; - - private: - QList headers_; - CompressOptions compress_; }; } From 6d5786818fdb3325b086fb5c56d21480bb4b1f54 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Mon, 13 Dec 2021 22:20:16 +0300 Subject: [PATCH 05/11] Set up json serialization --- pbom/util/__test__/json_test.cpp | 28 ++++++++++++++++++++++++++++ pbom/util/json.cpp | 9 ++++++++- pbom/util/json.h | 20 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/pbom/util/__test__/json_test.cpp b/pbom/util/__test__/json_test.cpp index adc28a7..65fe2e7 100644 --- a/pbom/util/__test__/json_test.cpp +++ b/pbom/util/__test__/json_test.cpp @@ -58,12 +58,21 @@ namespace pboman3::util::test { class MockJsonObject : public JsonObject { public: + MockJsonObject() = default; + + MockJsonObject(QString prop) : prop(std::move(prop)) { + }; + QString prop; protected: void inflate(const QString& path, const QJsonObject& json) override { prop = JsonValue().settle(json, path, "prop").value(); } + + void serialize(QJsonObject& target) const override { + target["prop"] = QJsonValue(prop); + } }; TEST(JsonObjectTest, Settle_Reads_QJsonValue) { @@ -199,4 +208,23 @@ namespace pboman3::util::test { ASSERT_EQ(ex.message(), ".obj[1] must be a {String}"); } } + + TEST(JsonArrayTest, MakeJson_Writes_Array_Of_Strings) { + const QList data{"s1", "s2", "s3"}; + const QJsonArray json = JsonArray::makeJson(data); + + ASSERT_EQ(json.count(), 3); + ASSERT_EQ(json.at(0).toString(), "s1"); + ASSERT_EQ(json.at(1).toString(), "s2"); + ASSERT_EQ(json.at(2).toString(), "s3"); + } + + TEST(JsonArrayTest, MakeJson_Writes_Array_Of_Objects) { + const QList data{MockJsonObject("p1"), MockJsonObject("p2")}; + const QJsonArray json = JsonArray::makeJson(data); + + ASSERT_EQ(json.count(), 2); + ASSERT_EQ(json.at(0).toObject()["prop"], "p1"); + ASSERT_EQ(json.at(1).toObject()["prop"], "p2"); + } } diff --git a/pbom/util/json.cpp b/pbom/util/json.cpp index 3c226a8..e4a726c 100644 --- a/pbom/util/json.cpp +++ b/pbom/util/json.cpp @@ -39,10 +39,17 @@ namespace pboman3::util { settle(parent[name], parentPath + "." + name); } - void JsonObject::settle(const QJsonValue& parent, const QString& parentPath, const QString& name, JsonMandatory mandatory) { + void JsonObject::settle(const QJsonValue& parent, const QString& parentPath, const QString& name, + JsonMandatory mandatory) { if (!parent.isObject()) { throw JsonStructureException(parentPath + " must be an {Object}"); } settle(parent.toObject(), parentPath, name, mandatory); } + + QJsonObject JsonObject::makeJson() const { + QJsonObject self; + serialize(self); + return self; + } } diff --git a/pbom/util/json.h b/pbom/util/json.h index e6ce67b..967bad6 100644 --- a/pbom/util/json.h +++ b/pbom/util/json.h @@ -89,8 +89,12 @@ namespace pboman3::util { void settle(const QJsonValue& parent, const QString& parentPath, const QString& name, JsonMandatory mandatory = JsonMandatory::Yes); + QJsonObject makeJson() const; + protected: virtual void inflate(const QString& path, const QJsonObject& json) = 0; + + virtual void serialize(QJsonObject& target) const = 0; }; namespace json { @@ -99,6 +103,13 @@ namespace pboman3::util { using TypeOuter = T; using TypeInner = T; static TypeOuter& getValue(TypeInner& t) { return t; } + static QJsonValue makeJson(const TypeOuter& t) { + if constexpr (std::is_base_of_v) { + return QJsonValue(t.makeJson()); + } else { + return QJsonValue(t); + } + } }; template @@ -106,6 +117,7 @@ namespace pboman3::util { using TypeOuter = T; using TypeInner = JsonValue; static TypeOuter& getValue(TypeInner& t) { return t.value(); } + static QJsonValue makeJson(const TypeOuter& t); //not implemented }; } @@ -147,6 +159,14 @@ namespace pboman3::util { return settle(parent.toObject(), parentPath, name, mandatory); } + static QJsonArray makeJson(const QList::TypeOuter>& data) { + QJsonArray result; + for (const typename json::JsonArrayTraits::TypeOuter& d : data) { + result.append(json::JsonArrayTraits::makeJson(d)); + } + return result; + } + private: QList::TypeOuter> data_; From f358d5eb43691c6301c2ace057626687ec99c263 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Mon, 13 Dec 2021 22:21:07 +0300 Subject: [PATCH 06/11] Made the PackOptions serialize to json --- pbom/model/task/__test__/packoptions_test.cpp | 23 +++++++++++++++ pbom/model/task/packoptions.cpp | 28 +++++++++++++++++++ pbom/model/task/packoptions.h | 12 ++++++++ 3 files changed, 63 insertions(+) diff --git a/pbom/model/task/__test__/packoptions_test.cpp b/pbom/model/task/__test__/packoptions_test.cpp index 15521c8..29c52f0 100644 --- a/pbom/model/task/__test__/packoptions_test.cpp +++ b/pbom/model/task/__test__/packoptions_test.cpp @@ -57,4 +57,27 @@ namespace pboman3::model::task::test { ASSERT_EQ(".headers[0].name must not be an empty string", ex.message()); } } + + TEST(PackOptionsTest, MakeJson_Builds_Document) { + PackOptions options; + options.headers = QList{PackHeader("h1", "v1"), PackHeader("h2", "v2")}; + options.compress.include = QList{"i1", "i2"}; + options.compress.exclude = QList{"e1", "e2"}; + + const QJsonObject json = options.makeJson(); + + ASSERT_EQ(json["headers"].toArray().count(), 2); + ASSERT_EQ(json["headers"].toArray().at(0).toObject()["name"], "h1"); + ASSERT_EQ(json["headers"].toArray().at(0).toObject()["value"], "v1"); + ASSERT_EQ(json["headers"].toArray().at(1).toObject()["name"], "h2"); + ASSERT_EQ(json["headers"].toArray().at(1).toObject()["value"], "v2"); + + ASSERT_EQ(json["compress"].toObject()["include"].toArray().count(), 2); + ASSERT_EQ(json["compress"].toObject()["include"].toArray().at(0).toString(), "i1"); + ASSERT_EQ(json["compress"].toObject()["include"].toArray().at(1).toString(), "i2"); + + ASSERT_EQ(json["compress"].toObject()["exclude"].toArray().count(), 2); + ASSERT_EQ(json["compress"].toObject()["exclude"].toArray().at(0).toString(), "e1"); + ASSERT_EQ(json["compress"].toObject()["exclude"].toArray().at(1).toString(), "e2"); + } } diff --git a/pbom/model/task/packoptions.cpp b/pbom/model/task/packoptions.cpp index d4989f7..ad2eb0f 100644 --- a/pbom/model/task/packoptions.cpp +++ b/pbom/model/task/packoptions.cpp @@ -4,11 +4,21 @@ #include namespace pboman3::model::task { + void CompressOptions::inflate(const QString& path, const QJsonObject& json) { include = JsonArray>().settle(json, path, "include", JsonMandatory::No).data(); exclude = JsonArray>().settle(json, path, "exclude", JsonMandatory::No).data(); } + void CompressOptions::serialize(QJsonObject& target) const { + target["include"] = JsonArray::makeJson(include); + target["exclude"] = JsonArray::makeJson(exclude); + } + + QDebug operator<<(QDebug debug, const CompressOptions& options) { + return debug << "(Include=" << options.include << ", Exclude=" << options.exclude << ")"; + } + PackHeader::PackHeader() = default; PackHeader::PackHeader(QString name, QString value) @@ -24,8 +34,26 @@ namespace pboman3::model::task { value = JsonValue().settle(json, path, "value").value(); } + void PackHeader::serialize(QJsonObject& target) const { + target["name"] = QJsonValue(name); + target["value"] = QJsonValue(value); + } + + QDebug operator<<(QDebug debug, const PackHeader& header) { + return debug << "(Name=" << header.name << ", Value=" << header.value << ")"; + } + void PackOptions::inflate(const QString& path, const QJsonObject& json) { headers = JsonArray().settle(json, path, "headers", JsonMandatory::No).data(); compress.settle(json, path, "compress", JsonMandatory::No); } + + void PackOptions::serialize(QJsonObject& target) const { + target["headers"] = JsonArray::makeJson(headers); + target["compress"] = QJsonValue(compress.makeJson()); + } + + QDebug operator<<(QDebug debug, const PackOptions& options) { + return debug << "(Headers=" << options.headers << ", Compress=" << options.compress << ")"; + } } diff --git a/pbom/model/task/packoptions.h b/pbom/model/task/packoptions.h index f9600aa..00cdd92 100644 --- a/pbom/model/task/packoptions.h +++ b/pbom/model/task/packoptions.h @@ -10,8 +10,12 @@ namespace pboman3::model::task { QList include; QList exclude; + friend QDebug operator<<(QDebug debug, const CompressOptions& options); + protected: void inflate(const QString& path, const QJsonObject& json) override; + + void serialize(QJsonObject& target) const override; }; class PackHeader : public JsonObject { @@ -23,8 +27,12 @@ namespace pboman3::model::task { QString name; QString value; + friend QDebug operator<<(QDebug debug, const PackHeader& header); + protected: void inflate(const QString& path, const QJsonObject& json) override; + + void serialize(QJsonObject& target) const override; }; class PackOptions : public JsonObject { @@ -32,7 +40,11 @@ namespace pboman3::model::task { QList headers; CompressOptions compress; + friend QDebug operator<<(QDebug debug, const PackOptions& options); + protected: void inflate(const QString& path, const QJsonObject& json) override; + + void serialize(QJsonObject& target) const override; }; } From 52d58bb8aebe1c1c5e986e3330299292c5baae89 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Tue, 14 Dec 2021 19:17:02 +0300 Subject: [PATCH 07/11] Implemented writing the pbo.json file when unpacking a file --- .../__test__/extractconfiguration_test.cpp | 46 +++++++++++++++++++ pbom/model/task/extractconfiguration.cpp | 36 +++++++++++++++ pbom/model/task/extractconfiguration.h | 6 +++ pbom/model/task/unpacktask.cpp | 11 +++++ pbom/model/task/unpacktask.h | 4 +- 5 files changed, 102 insertions(+), 1 deletion(-) diff --git a/pbom/model/task/__test__/extractconfiguration_test.cpp b/pbom/model/task/__test__/extractconfiguration_test.cpp index d6b498f..77963f4 100644 --- a/pbom/model/task/__test__/extractconfiguration_test.cpp +++ b/pbom/model/task/__test__/extractconfiguration_test.cpp @@ -1,5 +1,6 @@ #include "model/task/extractconfiguration.h" +#include #include #include #include "domain/pbodocument.h" @@ -109,4 +110,49 @@ namespace pboman3::model::task { ASSERT_EQ(options.compress.include.at(0), "\\.sqf$"); ASSERT_EQ(options.compress.include.at(1), "^mission.sqm$"); } + + TEST(ExtractConfigurationTest, SaveTo_Writes_To_Directory) { + QTemporaryDir t; + const QDir target(t.path()); + + PackOptions options; + options.headers = QList{PackHeader("h1", "v1"), PackHeader("h2", "v2")}; + options.compress.include = QList{"i1", "i2"}; + options.compress.exclude = QList{"e1", "e2"}; + + ExtractConfiguration::saveTo(options, target); + + QFile config(target.filePath("pbo.json")); + ASSERT_TRUE(config.exists()); + + ASSERT_TRUE(config.open(QIODeviceBase::ReadOnly)); + + const QByteArray bytes = config.readAll(); + ASSERT_EQ(QString(bytes), QString("{\n \"compress\": {\n \"exclude\": [\n \"e1\",\n \"e2\"\n ],\n \"include\": [\n \"i1\",\n \"i2\"\n ]\n },\n \"headers\": [\n {\n \"name\": \"h1\",\n \"value\": \"v1\"\n },\n {\n \"name\": \"h2\",\n \"value\": \"v2\"\n }\n ]\n}\n")); + } + + TEST(ExtractConfigurationTest, SaveTo_Picks_Non_Conflict_Name) { + QTemporaryDir t; + const QDir target(t.path()); + + //two placeholder files + QFile f1(target.filePath("pbo.json")); + f1.open(QIODeviceBase::NewOnly); + f1.close(); + QFile f2(target.filePath("pbo-1.json")); + f2.open(QIODeviceBase::NewOnly); + f2.close(); + + const PackOptions options; + ExtractConfiguration::saveTo(options, target); + + //the resulting file prevented conflicts + QFile config(target.filePath("pbo-2.json")); + ASSERT_TRUE(config.exists()); + + ASSERT_TRUE(config.open(QIODeviceBase::ReadOnly)); + + const QByteArray bytes = config.readAll(); + ASSERT_EQ(QString(bytes), QString("{\n \"compress\": {\n \"exclude\": [\n ],\n \"include\": [\n ]\n },\n \"headers\": [\n ]\n}\n")); + } } diff --git a/pbom/model/task/extractconfiguration.cpp b/pbom/model/task/extractconfiguration.cpp index 1a78a15..1f1963e 100644 --- a/pbom/model/task/extractconfiguration.cpp +++ b/pbom/model/task/extractconfiguration.cpp @@ -1,5 +1,9 @@ #include "extractconfiguration.h" + +#include + #include "domain/func.h" +#include "io/diskaccessexception.h" #include "model/binarysourceutils.h" namespace pboman3::model::task { @@ -12,6 +16,19 @@ namespace pboman3::model::task { return options; } + void ExtractConfiguration::saveTo(const PackOptions& options, const QDir& dest) { + const QJsonObject json = options.makeJson(); + const QByteArray bytes = QJsonDocument(json).toJson(QJsonDocument::Indented); + + const QString fileName = getConfigFileName(dest); + QFile file(fileName); + if (!file.open(QIODeviceBase::WriteOnly | QIODeviceBase::NewOnly)) { + throw DiskAccessException("Can not access the file. Check if it is used by other processes.", fileName); + } + file.write(bytes); + file.close(); + } + void ExtractConfiguration::extractHeaders(const PboDocument& document, PackOptions& options) { for (const DocumentHeader* header : *document.headers()) { options.headers.append(PackHeader(header->name(), header->value())); @@ -62,4 +79,23 @@ namespace pboman3::model::task { return "^" + fileName + "$"; } + QString ExtractConfiguration::getConfigFileName(const QDir& dir) { + const QString configName("pbo"); + const QString configExt(".json"); + + QString configFile = configName + configExt; + if (!dir.exists(configFile)) { + return dir.absoluteFilePath(configFile); + } + + for (int i = 1; i < std::numeric_limits::max(); i++) { + configFile = configName + "-" + QString::number(i) + configExt; + if (!dir.exists(configFile)) { + return dir.absoluteFilePath(configFile); + } + } + + throw AppException("The code must never reach this line"); + } + } diff --git a/pbom/model/task/extractconfiguration.h b/pbom/model/task/extractconfiguration.h index 1581641..9065b35 100644 --- a/pbom/model/task/extractconfiguration.h +++ b/pbom/model/task/extractconfiguration.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "packoptions.h" #include "domain/pbodocument.h" @@ -10,6 +12,8 @@ namespace pboman3::model::task { public: static PackOptions extractFrom(const PboDocument& document); + static void saveTo(const PackOptions& options, const QDir& dest); + private: static void extractHeaders(const PboDocument& document, PackOptions& options); @@ -20,5 +24,7 @@ namespace pboman3::model::task { static QString makeExtensionCompressionRule(const QString& ext); static QString makeFileCompressionRule(const QString& fileName); + + static QString getConfigFileName(const QDir& dir); }; } diff --git a/pbom/model/task/unpacktask.cpp b/pbom/model/task/unpacktask.cpp index dab4932..3ef3d4e 100644 --- a/pbom/model/task/unpacktask.cpp +++ b/pbom/model/task/unpacktask.cpp @@ -1,6 +1,9 @@ #include "unpacktask.h" #include #include + +#include "extractconfiguration.h" +#include "packoptions.h" #include "io/bb/unpacktaskbackend.h" #include "io/bs/pbobinarysource.h" #include "io/pbonodeentity.h" @@ -57,6 +60,8 @@ namespace pboman3::model::task { childNodes.append(node); be.unpackSync(document->root(), childNodes, cancel); + extractPboConfig(*document, pboDir); + LOG(info, "Unpack complete") } @@ -111,4 +116,10 @@ namespace pboman3::model::task { return true; } + + void UnpackTask::extractPboConfig(const PboDocument& document, const QDir& dir) { + const PackOptions options = ExtractConfiguration::extractFrom(document); + LOG(info, "Extracted the PBO pack config, Options=", options) + ExtractConfiguration::saveTo(options, dir); + } } diff --git a/pbom/model/task/unpacktask.h b/pbom/model/task/unpacktask.h index 76120df..320bbb3 100644 --- a/pbom/model/task/unpacktask.h +++ b/pbom/model/task/unpacktask.h @@ -19,10 +19,12 @@ namespace pboman3::model::task { const QString pboPath_; const QDir outputDir_; - bool tryReadPboHeader(QSharedPointer* document); + bool tryReadPboHeader(QSharedPointer* document); bool tryCreatePboDir(QDir* dir); bool tryCreateEntryDir(const QDir& pboDir, const QSharedPointer& entry); + + void extractPboConfig(const PboDocument& document, const QDir& dir); }; } From bd3ec37e86d69ee84042e5f770932143dfad1b04 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Tue, 14 Dec 2021 19:24:40 +0300 Subject: [PATCH 08/11] The prefix files names now start from PBO --- .../task/__test__/packconfiguration_test.cpp | 22 +++++++++---------- pbom/model/task/packconfiguration.cpp | 6 ++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pbom/model/task/__test__/packconfiguration_test.cpp b/pbom/model/task/__test__/packconfiguration_test.cpp index 32dfbec..185c225 100644 --- a/pbom/model/task/__test__/packconfiguration_test.cpp +++ b/pbom/model/task/__test__/packconfiguration_test.cpp @@ -18,11 +18,11 @@ namespace pboman3::model::task::test { document.root()->createHierarchy(PboPath({"f1.txt"})); PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); - PboNode* prefix = document.root()->createHierarchy(PboPath({"$prefix$"})); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$pboprefix$"})); prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); - PboNode* version = document.root()->createHierarchy(PboPath({"$version$"})); + PboNode* version = document.root()->createHierarchy(PboPath({"$pboversion$"})); version->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); - PboNode* product = document.root()->createHierarchy(PboPath({"$product$"})); + PboNode* product = document.root()->createHierarchy(PboPath({"$pboproduct$"})); product->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); const PackConfiguration packConfiguration(&document); @@ -44,11 +44,11 @@ namespace pboman3::model::task::test { PboNode* pboJson = document.root()->createHierarchy(PboPath({"pbo.json"})); pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); //these must not be applied - PboNode* prefix = document.root()->createHierarchy(PboPath({"$prefix$"})); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$pboprefix$"})); prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); - PboNode* version = document.root()->createHierarchy(PboPath({"$version$"})); + PboNode* version = document.root()->createHierarchy(PboPath({"$pboversion$"})); version->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); - PboNode* product = document.root()->createHierarchy(PboPath({"$product$"})); + PboNode* product = document.root()->createHierarchy(PboPath({"$pboproduct$"})); product->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); const PackConfiguration packConfiguration(&document); @@ -78,11 +78,11 @@ namespace pboman3::model::task::test { ver.close(); PboDocument document("file.pbo"); - PboNode* prefix = document.root()->createHierarchy(PboPath({"$prEfix$"})); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$pbOprEfix$"})); prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(pref.fileName())); - PboNode* product = document.root()->createHierarchy(PboPath({"$prOduct$"})); + PboNode* product = document.root()->createHierarchy(PboPath({"$PboprOduct$"})); product->binarySource = QSharedPointer(new io::FsRawBinarySource(prod.fileName())); - PboNode* version = document.root()->createHierarchy(PboPath({"$veRsion$"})); + PboNode* version = document.root()->createHierarchy(PboPath({"$pBoveRsion$"})); version->binarySource = QSharedPointer(new io::FsRawBinarySource(ver.fileName())); const PackConfiguration packConfiguration(&document); @@ -212,7 +212,7 @@ namespace pboman3::model::task::test { pref.close(); PboDocument document("file.pbo"); - PboNode* prefix = document.root()->createHierarchy(PboPath({"$prefix$"})); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$pboprefix$"})); prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(pref.fileName())); try { @@ -220,7 +220,7 @@ namespace pboman3::model::task::test { packConfiguration.apply(); FAIL() << "Should have not reached this line"; } catch (const PrefixEncodingException& ex) { - ASSERT_EQ(ex.message(), "$prefix$"); + ASSERT_EQ(ex.message(), "$pboprefix$"); } } } diff --git a/pbom/model/task/packconfiguration.cpp b/pbom/model/task/packconfiguration.cpp index 71a959a..cbf622e 100644 --- a/pbom/model/task/packconfiguration.cpp +++ b/pbom/model/task/packconfiguration.cpp @@ -45,11 +45,11 @@ namespace pboman3::model::task { applyDocumentCompressionRules(document_->root(), compressionRules); } - PboNode* prefix = FindDirectChild(document_->root(), "$prefix$"); + PboNode* prefix = FindDirectChild(document_->root(), "$pboprefix$"); APPLY_NODE_CONTENT_AS_HEADER(prefix, pboJson) - PboNode* product = FindDirectChild(document_->root(), "$product$"); + PboNode* product = FindDirectChild(document_->root(), "$pboproduct$"); APPLY_NODE_CONTENT_AS_HEADER(product, pboJson) - PboNode* version = FindDirectChild(document_->root(), "$version$"); + PboNode* version = FindDirectChild(document_->root(), "$pboversion$"); APPLY_NODE_CONTENT_AS_HEADER(version, pboJson) CLEANUP_NODE(pboJson) From c0a5c76f69082e8b73f4a2ed281d929e9c95d8cb Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Tue, 21 Dec 2021 20:30:34 +0300 Subject: [PATCH 09/11] It is possible to use alternative names for the prefix files --- .../task/__test__/packconfiguration_test.cpp | 94 ++++++++++++++++++- pbom/model/task/packconfiguration.cpp | 45 +++++---- pbom/model/task/packconfiguration.h | 4 +- 3 files changed, 124 insertions(+), 19 deletions(-) diff --git a/pbom/model/task/__test__/packconfiguration_test.cpp b/pbom/model/task/__test__/packconfiguration_test.cpp index 185c225..00bf8d0 100644 --- a/pbom/model/task/__test__/packconfiguration_test.cpp +++ b/pbom/model/task/__test__/packconfiguration_test.cpp @@ -8,7 +8,7 @@ namespace pboman3::model::task::test { using namespace domain; - TEST(PackConfigurationTest, Apply_Removes_All_Config_Nodes) { + TEST(PackConfigurationTest, Apply_Removes_All_Config_Nodes_If_PboJson) { QTemporaryFile json; json.open(); json.write(QByteArray("{}")); @@ -20,10 +20,44 @@ namespace pboman3::model::task::test { pboJson->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); PboNode* prefix = document.root()->createHierarchy(PboPath({"$pboprefix$"})); prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* prefixTxt = document.root()->createHierarchy(PboPath({"pboprefix.txt"})); + prefixTxt->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); PboNode* version = document.root()->createHierarchy(PboPath({"$pboversion$"})); version->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* versionTxt = document.root()->createHierarchy(PboPath({"pboversion.txt"})); + versionTxt->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); PboNode* product = document.root()->createHierarchy(PboPath({"$pboproduct$"})); product->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* productTxt = document.root()->createHierarchy(PboPath({"pboproduct.txt"})); + productTxt->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_EQ(document.root()->count(), 1); //config files removed + ASSERT_TRUE(document.root()->get(PboPath({"f1.txt"}))); //but others are in places + } + + TEST(PackConfigurationTest, Apply_Removes_All_Config_Nodes_If_No_PboJson) { + QTemporaryFile json; + json.open(); + json.write(QByteArray("{}")); + json.close(); + + PboDocument document("file.pbo"); + document.root()->createHierarchy(PboPath({"f1.txt"})); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$pboprefix$"})); + prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* prefixTxt = document.root()->createHierarchy(PboPath({"pboprefix.txt"})); + prefixTxt->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* version = document.root()->createHierarchy(PboPath({"$pboversion$"})); + version->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* versionTxt = document.root()->createHierarchy(PboPath({"pboversion.txt"})); + versionTxt->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* product = document.root()->createHierarchy(PboPath({"$pboproduct$"})); + product->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); + PboNode* productTxt = document.root()->createHierarchy(PboPath({"pboproduct.txt"})); + productTxt->binarySource = QSharedPointer(new io::FsRawBinarySource(json.fileName())); const PackConfiguration packConfiguration(&document); packConfiguration.apply(); @@ -97,6 +131,64 @@ namespace pboman3::model::task::test { ASSERT_EQ(document.headers()->at(2)->value(), "ver1"); } + TEST(PackConfigurationTest, Apply_Sets_Headers_From_Alternative_Prefix_Files) { + QTemporaryFile pref; + pref.open(); + pref.write(QByteArray("pref1")); + pref.close(); + + QTemporaryFile prefAlt; + prefAlt.open(); + prefAlt.write(QByteArray("pref1Alt")); + prefAlt.close(); + + QTemporaryFile prod; + prod.open(); + prod.write(QByteArray("prod1")); + prod.close(); + + QTemporaryFile prodAlt; + prodAlt.open(); + prodAlt.write(QByteArray("prod1Alt")); + prodAlt.close(); + + QTemporaryFile ver; + ver.open(); + ver.write(QByteArray("ver1")); + ver.close(); + + QTemporaryFile verAlt; + verAlt.open(); + verAlt.write(QByteArray("ver1Alt")); + verAlt.close(); + + + PboDocument document("file.pbo"); + PboNode* prefix = document.root()->createHierarchy(PboPath({"$pboprefix$"})); + prefix->binarySource = QSharedPointer(new io::FsRawBinarySource(pref.fileName())); + PboNode* prefixAlt = document.root()->createHierarchy(PboPath({"pboprefix.txt"})); + prefixAlt->binarySource = QSharedPointer(new io::FsRawBinarySource(prefAlt.fileName())); + PboNode* product = document.root()->createHierarchy(PboPath({"$pboproduct$"})); + product->binarySource = QSharedPointer(new io::FsRawBinarySource(prod.fileName())); + PboNode* productAlt = document.root()->createHierarchy(PboPath({"pboproduct.txt"})); + productAlt->binarySource = QSharedPointer(new io::FsRawBinarySource(prodAlt.fileName())); + PboNode* version = document.root()->createHierarchy(PboPath({"$pboversion$"})); + version->binarySource = QSharedPointer(new io::FsRawBinarySource(ver.fileName())); + PboNode* versionAlt = document.root()->createHierarchy(PboPath({"pboversion.txt"})); + versionAlt->binarySource = QSharedPointer(new io::FsRawBinarySource(verAlt.fileName())); + + const PackConfiguration packConfiguration(&document); + packConfiguration.apply(); + + ASSERT_EQ(document.headers()->count(), 3); + ASSERT_EQ(document.headers()->at(0)->name(), "prefix"); + ASSERT_EQ(document.headers()->at(0)->value(), "pref1Alt"); + ASSERT_EQ(document.headers()->at(1)->name(), "product"); + ASSERT_EQ(document.headers()->at(1)->value(), "prod1Alt"); + ASSERT_EQ(document.headers()->at(2)->name(), "version"); + ASSERT_EQ(document.headers()->at(2)->value(), "ver1Alt"); + } + TEST(PackConfigurationTest, Apply_Compresses_Only_Included_Files) { QTemporaryFile json; json.open(); diff --git a/pbom/model/task/packconfiguration.cpp b/pbom/model/task/packconfiguration.cpp index cbf622e..de8bcac 100644 --- a/pbom/model/task/packconfiguration.cpp +++ b/pbom/model/task/packconfiguration.cpp @@ -32,10 +32,8 @@ namespace pboman3::model::task { : document_(document) { } -#define APPLY_NODE_CONTENT_AS_HEADER(V,P) if (!(P) && (V)) { applyNodeContentAsHeader(V, #V); } -#define CLEANUP_NODE(N) if (N) { (N)->removeFromHierarchy(); } - void PackConfiguration::apply() const { + bool usedPboJson = false; PboNode* pboJson = FindDirectChild(document_->root(), "pbo.json"); if (pboJson) { LOG(info, "Apply configuration from pbo.json") @@ -43,19 +41,13 @@ namespace pboman3::model::task { applyDocumentHeaders(packOptions); const CompressionRules compressionRules = buildCompressionRules(packOptions); applyDocumentCompressionRules(document_->root(), compressionRules); + pboJson->removeFromHierarchy(); + usedPboJson = true; } - PboNode* prefix = FindDirectChild(document_->root(), "$pboprefix$"); - APPLY_NODE_CONTENT_AS_HEADER(prefix, pboJson) - PboNode* product = FindDirectChild(document_->root(), "$pboproduct$"); - APPLY_NODE_CONTENT_AS_HEADER(product, pboJson) - PboNode* version = FindDirectChild(document_->root(), "$pboversion$"); - APPLY_NODE_CONTENT_AS_HEADER(version, pboJson) - - CLEANUP_NODE(pboJson) - CLEANUP_NODE(prefix) - CLEANUP_NODE(product) - CLEANUP_NODE(version) + processPrefixFile("prefix", "$pboprefix$", "pboprefix.txt", usedPboJson); + processPrefixFile("product", "$pboproduct$", "pboproduct.txt", usedPboJson); + processPrefixFile("version", "$pboversion$", "pboversion.txt", usedPboJson); } void PackConfiguration::applyDocumentHeaders(const PackOptions& options) const { @@ -161,12 +153,31 @@ namespace pboman3::model::task { return data; } - void PackConfiguration::applyNodeContentAsHeader(const PboNode* node, const QString& prefix) const { - LOG(info, "Apply prefix:", prefix) + void PackConfiguration::processPrefixFile(const QString& header, const QString& fileName, + const QString& altFileName, bool cleanupOnly) const { + PboNode* file1 = FindDirectChild(document_->root(), fileName); + PboNode* file2 = FindDirectChild(document_->root(), altFileName); + + if (!cleanupOnly) { + if (file2) { + applyNodeContentAsHeader(file2, header); + } else if (file1) { + applyNodeContentAsHeader(file1, header); + } + } + + if (file1) + file1->removeFromHierarchy(); + if (file2) + file2->removeFromHierarchy(); + } + + void PackConfiguration::applyNodeContentAsHeader(const PboNode* node, const QString& header) const { + LOG(info, "Apply prefix:", header) const QByteArray data = readNodeContent(node); throwIfBreaksPbo(node, data); const QSharedPointer tran = document_->headers()->beginTransaction(); - tran->add(prefix, data); + tran->add(header, data); tran->commit(); } diff --git a/pbom/model/task/packconfiguration.h b/pbom/model/task/packconfiguration.h index 71890a8..8024b8d 100644 --- a/pbom/model/task/packconfiguration.h +++ b/pbom/model/task/packconfiguration.h @@ -47,7 +47,9 @@ namespace pboman3::model::task { QList exclude; }; - void applyNodeContentAsHeader(const PboNode* node, const QString& prefix) const; + void processPrefixFile(const QString& header, const QString& fileName, const QString& altFileName, bool cleanupOnly) const; + + void applyNodeContentAsHeader(const PboNode* node, const QString& header) const; static void throwIfBreaksPbo(const PboNode* node, const QByteArray& data); }; From 404b4aa50e7552625e8291856a10eaded6977f33 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Tue, 21 Dec 2021 21:22:10 +0300 Subject: [PATCH 10/11] Fixed the taskbar was not flashing on operation complete if the window was expanded but moved to background --- pbom/ui/win32/win32taskbarindicator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pbom/ui/win32/win32taskbarindicator.cpp b/pbom/ui/win32/win32taskbarindicator.cpp index 1a3df27..07064cb 100644 --- a/pbom/ui/win32/win32taskbarindicator.cpp +++ b/pbom/ui/win32/win32taskbarindicator.cpp @@ -43,7 +43,7 @@ namespace pboman3::ui { } bool Win32TaskbarIndicator::windowHasFocus() const { - return GetActiveWindow() == window_; + return GetForegroundWindow() == window_; } void Win32TaskbarIndicator::flashWindow() const { From 3dd40c339e4e25a3a1a631eb054228761ae64ad9 Mon Sep 17 00:00:00 2001 From: Nikita Kobzev Date: Thu, 23 Dec 2021 19:17:13 +0300 Subject: [PATCH 11/11] Added the docs regarding pbo.json and prefix file usage --- README.md | 1 + doc/pbo_json.md | 88 +++++++++++++++++++++++++++++++++++++++++++++ doc/prefix_files.md | 15 ++++++++ 3 files changed, 104 insertions(+) create mode 100644 doc/pbo_json.md create mode 100644 doc/prefix_files.md diff --git a/README.md b/README.md index 49b1116..da9613d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ A tool to open, pack and unpack ArmA PBO files. - Can preview files inside a PBO. - Can open mangled (by Mikero's tools) PBO. - Integrates with Windows Explorer. + - Supports PBO metadata provisioning through [pbo.json](doc/pbo_json.md) or [\$PBOPREFIX\$](doc/prefix_files.md) files. ## Screenshots diff --git a/doc/pbo_json.md b/doc/pbo_json.md new file mode 100644 index 0000000..f949eed --- /dev/null +++ b/doc/pbo_json.md @@ -0,0 +1,88 @@ +# Pbo.json + +You can put the `pbo.json` file into the PBO content root directory when you pack a PBO via the PBO Manager. The PBO Manager reads the PBO headers and the compression rules from the `pbo.json`. + +An example `pbo.json`: + +```json +{ + "headers": [{ + "name": "prefix", + "value": "my-addon" + }, { + "name": "version", + "value": "1.0.0" + }, { + "name": "product", + "value": "my-mod" + }], + "compress": { + "include": ["\.txt$", "\.sqf$", "\.ext"], + "exclude": ["^description.ext$"] + } +} +``` + +## Pbo.json fields + +### headers + +The collection of the headers the PBO file should have. + +Type: `array` + +Required: `no` + +Default: `[]` + +#### headers[].name + +The name of the header. + +Type: `string` + +Required: `yes` + +#### headers[].value + +The value of the header. + +Type: `string` + +Required: `no` + +Default: `""` + +### compress + +The rules of compressing the contents of the PBO file. + +#### compress.include + +Which files to compress. + +Type: `array` of [regular expressions](https://en.wikipedia.org/wiki/Regular_expression). (The regular expressions will run against the **relative** file names with **forward slashes** as separators, e.g. `scripts/script1.sqf`) + +Required: `no` + +#### compress.exclude + +Which of the **included** files to exclude from compression. Use this to _"include all then exclude a couple"_ scenario. + +Type: `array` of [regular expressions](https://en.wikipedia.org/wiki/Regular_expression) + +Required: `no` + + +## Regular expression debugging + +There are lots of the services for regular expression debugging, such as https://regex101.com + +## Regular expression examples + +| What it will match | Example files | RegEx | +| ------------------------------------------------- | ------------------------------------------------------------------ | ---------------------- | +| All the `.sqf` files. | `my-script.sqf` or `Scripts/script.sqf` | `\.sqf$` | +| All the `.sqf` files in the `Scripts` root foler. | `Scripts/Client/Alpha/fn_run.sqf` | `^scripts\/.+\.sqf$` | +| All the `.sqf` files in the `Alpha` subfolders. | `Scripts/Client/Alpha/initClient.sqf` or `Scripts/Server/Alpha/initServer.sqf` | `\/.+\/Alpha\/.+\.sqf$` | +| The `decription.ext` file in the root. | `description.ext` | `^description.ext$` | diff --git a/doc/prefix_files.md b/doc/prefix_files.md new file mode 100644 index 0000000..d6b798c --- /dev/null +++ b/doc/prefix_files.md @@ -0,0 +1,15 @@ +# Prefix files + +Since the times of Operation Flashpoint, they used so-called [prefix files](https://community.bistudio.com/wiki/PBOPREFIX) to provide the packaging tools the information regarding the headers a packed PBO should have. + +To use a prefix file, you create a text file with the name i.e. `$pboprefix$` (or `pboprefix.txt`) and some text. And when the PBO Manager packs the folder, the produced `PBO` will have the header with the name `prefix` and the text from the file. + +The PBO Manager supports the following prefix files (names are case-insensitive): + +| File name | Alternative file name | Resulting PBO header | +| -------------- | --------------------- | -------------------- | +| \$pboprefix\$ | pboprefix.txt | prefix | +| \$pboproduct\$ | pboproduct.txt | product | +| \$pboversion\$ | pboversion.txt | version | + +However, there is a more efficient way to control file headers: [pbo.json](pbo_json.md).