From 1041e8ec2aac6562e5084f9adb006946802a3aa0 Mon Sep 17 00:00:00 2001 From: Paul Walker Date: Fri, 23 Aug 2024 08:49:14 -0400 Subject: [PATCH] An experiment with porting an importer from ConvertWithMOSS This is an experiment which we may not finish for 1.0 with porting an importer from ConvertWithMoss, in this case the EXS importer. It kinda shows the approach but it doesn't work very well right now since 1. I don't pick apart the structure carefully and 2. Logic has expanded the format ahead of CWM some and 3. It's just a lot of fiddly work So I'll commit this and perhaps nuke it from the code base later ,but I didn't want to abandon the couple of hours of work so merge it for now. --- src/CMakeLists.txt | 1 + src/browser/browser.cpp | 1 + src/engine/engine.cpp | 14 + src/sample/exs_support/exs_import.cpp | 614 ++++++++++++++++++++++++++ src/sample/exs_support/exs_import.h | 52 +++ 5 files changed, 682 insertions(+) create mode 100644 src/sample/exs_support/exs_import.cpp create mode 100644 src/sample/exs_support/exs_import.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81bcd741..b7c7be12 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,6 +28,7 @@ add_library(${PROJECT_NAME} STATIC sample/loaders/load_flac.cpp sample/loaders/load_mp3.cpp + sample/exs_support/exs_import.cpp sample/multisample_support/multisample_import.cpp sample/sfz_support/sfz_parse.cpp sample/sfz_support/sfz_import.cpp diff --git a/src/browser/browser.cpp b/src/browser/browser.cpp index 28719008..ccab89ab 100644 --- a/src/browser/browser.cpp +++ b/src/browser/browser.cpp @@ -68,6 +68,7 @@ const std::vector Browser::LoadableFile::singleSample{".wav", ".fla ".aiff"}; const std::vector Browser::LoadableFile::multiSample{".sf2", ".sfz", ".multisample"}; +// ".exs"}; const std::vector Browser::LoadableFile::shortcircuitFormats{".scm", ".scp"}; bool Browser::isLoadableFile(const fs::path &p) diff --git a/src/engine/engine.cpp b/src/engine/engine.cpp index bd7a94fc..31936318 100644 --- a/src/engine/engine.cpp +++ b/src/engine/engine.cpp @@ -37,6 +37,7 @@ #include "messaging/audio/audio_messages.h" #include "selection/selection_manager.h" #include "sample/sfz_support/sfz_import.h" +#include "sample/exs_support/exs_import.h" #include "sample/multisample_support/multisample_import.h" #include "infrastructure/user_defaults.h" #include "infrastructure/md5support.h" @@ -548,6 +549,19 @@ void Engine::loadSampleIntoSelectedPartAndGroup(const fs::path &p, int16_t rootK }); return; } + else if (extensionMatches(p, ".exs")) + { + // TODO ok this refresh and restart is a bit unsatisfactory + messageController->stopAudioThreadThenRunOnSerial([this, p](const auto &) { + auto res = exs_support::importEXS(p, *this); + if (!res) + messageController->reportErrorToClient("EXS Import Failed", "Dunno why"); + messageController->restartAudioThreadFromSerial(); + serializationSendToClient(messaging::client::s2c_send_pgz_structure, + getPartGroupZoneStructure(), *messageController); + }); + return; + } else if (extensionMatches(p, ".multisample")) { messageController->stopAudioThreadThenRunOnSerial([this, p](const auto &) { diff --git a/src/sample/exs_support/exs_import.cpp b/src/sample/exs_support/exs_import.cpp new file mode 100644 index 00000000..4a029a01 --- /dev/null +++ b/src/sample/exs_support/exs_import.cpp @@ -0,0 +1,614 @@ +/* + * Shortcircuit XT - a Surge Synth Team product + * + * A fully featured creative sampler, available as a standalone + * and plugin for multiple platforms. + * + * Copyright 2019 - 2024, Various authors, as described in the github + * transaction log. + * + * ShortcircuitXT is released under the Gnu General Public Licence + * V3 or later (GPL-3.0-or-later). The license is found in the file + * "LICENSE" in the root of this repository or at + * https://www.gnu.org/licenses/gpl-3.0.en.html + * + * Individual sections of code which comprises ShortcircuitXT in this + * repository may also be used under an MIT license. Please see the + * section "Licensing" in "README.md" for details. + * + * ShortcircuitXT is inspired by, and shares code with, the + * commercial product Shortcircuit 1 and 2, released by VemberTech + * in the mid 2000s. The code for Shortcircuit 2 was opensourced in + * 2020 at the outset of this project. + * + * All source for ShortcircuitXT is available at + * https://github.com/surge-synthesizer/shortcircuit-xt + */ + +#include +#include +#include +#include +#include "utils.h" +#include "exs_import.h" + +namespace scxt::exs_support +{ +using byteData_t = std::vector; + +uint32_t readU32(std::ifstream &f, bool bigE) +{ + char c4[4]; + f.read(c4, 4); + uint32_t res{0}; + if (bigE) + { + for (int i = 0; i < 4; ++i) + { + res = res << 8; + res += (uint8_t)c4[i]; + } + } + else + { + for (int i = 3; i >= 0; --i) + { + res = res << 8; + res += (uint8_t)c4[i]; + } + } + return res; +} + +struct EXSBlock +{ + /** An instrument block. */ + static constexpr uint32_t TYPE_INSTRUMENT = 0x00; + /** A zone block. */ + static constexpr uint32_t TYPE_ZONE = 0x01; + /** A group block. */ + static constexpr uint32_t TYPE_GROUP = 0x02; + /** A sample block. */ + static constexpr uint32_t TYPE_SAMPLE = 0x03; + /** A parameters block. */ + static constexpr uint32_t TYPE_PARAMS = 0x04; + /** An unknown block. */ + static constexpr uint32_t TYPE_UNKNOWN = 0x08; + + bool isBigEndian{true}; + uint32_t type{0}; + uint32_t size{0}; + uint32_t index{0}; + uint32_t flags{0}; + std::string name; + byteData_t content; + + bool read(std::ifstream &f) + { +#define CHECK(...) \ + __VA_ARGS__; \ + if (f.eof()) \ + return false; + + uint8_t c; + CHECK(f.read((char *)&c, 1)); + + isBigEndian = (c == 0); + + uint8_t v1, v2, tp; + CHECK(f.read((char *)&v1, 1)); + CHECK(f.read((char *)&v2, 1)); + CHECK(f.read((char *)&tp, 1)); + + type = tp & 0x0F; + + if (v1 != 1 && v2 != 0) + { + SCLOG("Unknown EXS Version: " << v1 << " " << v2); + return false; + } + + CHECK(size = readU32(f, isBigEndian)); + CHECK(index = readU32(f, isBigEndian)); + CHECK(flags = readU32(f, isBigEndian)); + + char magic[4]; + CHECK(f.read(magic, 4)); + // TODO: Check + + char namec[64]; + CHECK(f.read(namec, 64)); + name = std::string(namec); // assume null termination + + content.resize(size); + f.read((char *)(content.data()), size); +#undef CHECK + return true; + } +}; + +struct EXSObject +{ + EXSBlock within; + EXSObject(EXSBlock in) : within(in) {} + + size_t position{0}; + void resetPosition() { position = 0; } + void skip(int S) + { + position += S; + assert(position < within.size); + } + + int32_t readByteToInt() + { + char c = within.content[position]; + position++; + assert(position < within.size); + return (int32_t)c; + } + + uint8_t readByte() + { + char c = within.content[position]; + position++; + assert(position < within.size); + return (uint8_t)c; + } + + uint16_t readU16() + { + uint32_t res = 0; + if (within.isBigEndian) + { + for (auto i = position; i < position + 2; ++i) + { + res = res << 8; + res += (uint8_t)within.content[i]; + } + } + else + { + for (auto i = (int)position + 1; i >= (int)position; --i) + { + res = res << 8; + res += (uint8_t)within.content[i]; + } + } + position += 2; + assert(position < within.size); + return res; + } + uint32_t readU32() + { + uint32_t res = 0; + if (within.isBigEndian) + { + for (auto i = position; i < position + 4; ++i) + { + res = res << 8; + res += (uint8_t)within.content[i]; + } + } + else + { + // without this int cast at position = 0 this loop never terminates + for (auto i = (int)position + 3; i >= (int)position; --i) + { + res = res << 8; + res += (uint8_t)within.content[i]; + } + } + position += 4; + assert(position < within.size); + return res; + } + std::string readStringNoTerm(size_t chars) + { + auto res = std::string(within.content[position], chars); + position += chars; + return res; + } + std::string readStringNullTerm(size_t chars) + { + // Sloppy + char buf[512]; + assert(chars < 512); + memset(buf, 0, sizeof(buf)); + memcpy(buf, within.content.data() + position, chars); + position += chars; + return std::string(buf); + } +}; +struct EXSInstrument : EXSObject +{ + EXSInstrument(EXSBlock in) : EXSObject(in) { assert(in.type == EXSBlock::TYPE_INSTRUMENT); } +}; + +struct EXSZone : EXSObject +{ + bool pitch; + bool oneshot; + bool reverse; + int32_t key; + int32_t fineTuning; + int32_t pan; + int32_t volumeAdjust; + int32_t volumeScale; + int32_t coarseTuning; + int32_t keyLow; + int32_t keyHigh; + bool velocityRangeOn; + int32_t velocityLow; + int32_t velocityHigh; + int32_t sampleStart; + int32_t sampleEnd; + int32_t loopStart; + int32_t loopEnd; + int32_t loopCrossfade; + int32_t loopTune; + bool loopOn; + bool loopEqualPower; + bool loopPlayToEndOnRelease; + int32_t loopDirection; + int32_t flexOptions; + int32_t flexSpeed; + int32_t tailTune; + int32_t output; + int32_t groupIndex; + int32_t sampleIndex; + int32_t sampleFadeOut = 0; + int32_t offset = 0; + + EXSZone(EXSBlock in) : EXSObject(in) + { + assert(within.type == EXSBlock::TYPE_ZONE); + parse(); + } + void parse() + { + resetPosition(); + int zoneOpts = readByteToInt(); + pitch = (zoneOpts & 1 << 1) == 0; + oneshot = (zoneOpts & 1 << 0) != 0; + reverse = (zoneOpts & 1 << 2) != 0; + velocityRangeOn = (zoneOpts & 1 << 3) != 0; + + key = readByteToInt(); + fineTuning = decodeTwosComplement(readByte()); + pan = decodeTwosComplement(readByte()); + volumeAdjust = decodeTwosComplement(readByte()); + volumeScale = readByteToInt(); + keyLow = readByteToInt(); + keyHigh = readByteToInt(); + skip(1); + velocityLow = readByteToInt(); + velocityHigh = readByteToInt(); + skip(1); + sampleStart = (int)readU32(); + sampleEnd = (int)readU32(); + loopStart = (int)readU32(); + loopEnd = (int)readU32(); + loopCrossfade = (int)readU32(); + loopTune = readByteToInt(); + + int loopOptions = readByteToInt(); + loopOn = (loopOptions & 1) != 0; + loopEqualPower = (loopOptions & 2) != 0; + loopPlayToEndOnRelease = (loopOptions & 4) != 0; + + loopDirection = readByteToInt(); + skip(42); + flexOptions = readByteToInt(); + flexSpeed = readByteToInt(); + tailTune = readByteToInt(); + coarseTuning = decodeTwosComplement(readByte()); + + skip(1); + + output = readByteToInt(); + if ((zoneOpts & 1 << 6) == 0) + output = -1; + + skip(5); + + groupIndex = (int)readU32(); + sampleIndex = (int)readU32(); + } + + static int32_t decodeTwosComplement(uint8_t value) + { + return (value & 0x80) != 0 ? value - 256 : value; + } +}; + +struct EXSGroup : EXSObject +{ + int32_t volume = 0; + int32_t pan = 0; + int32_t polyphony = 0; // = Max + int32_t exclusive = 0; + int32_t minVelocity = 0; + int32_t maxVelocity = 127; + int32_t sampleSelectRandomOffset = 0; + int32_t releaseTriggerTime = 0; + int32_t velocityRangExFade = 0; + int32_t velocityRangExFadeType = 0; + int32_t keyrangExFadeType = 0; + int32_t keyrangExFade = 0; + int32_t enableByTempoLow = 80; + int32_t enableByTempoHigh = 140; + int32_t cutoffOffset = 0; + int32_t resoOffset = 0; + int32_t env1AttackOffset = 0; + int32_t env1DecayOffset = 0; + int32_t env1SustainOffset = 0; + int32_t env1ReleaseOffset = 0; + bool releaseTrigger = false; + int32_t output = 0; + int32_t enableByNoteValue = 0; + int32_t roundRobinGroupPos = -1; + int32_t enableByType = 0; + int32_t enableByControlValue = 0; + int32_t enableByControlLow = 0; + int32_t enableByControlHigh = 0; + int32_t startNote = 0; + int32_t endNote = 127; + int32_t enableByMidiChannel = 0; + int32_t enableByArticulationValue = 0; + int32_t enableByBenderLow = 0; + int32_t enableByBenderHigh = 0; + int32_t env1HoldOffset = 0; + int32_t env2AttackOffset = 0; + int32_t env2DecayOffset = 0; + int32_t env2SustainOffset = 0; + int32_t env2ReleaseOffset = 0; + int32_t env2HoldOffset = 0; + int32_t env1DelayOffset = 0; + int32_t env2DelayOffset = 0; + + bool mute = false; + bool releaseTriggerDecay = false; + bool fixedSampleSelect = false; + + bool enableByNote = false; + bool enableByRoundRobin = false; + bool enableByControl = false; + bool enableByBend = false; + bool enableByChannel = false; + bool enableByArticulation = false; + bool enablebyTempo = false; + + EXSGroup(EXSBlock in) : EXSObject(in) + { + assert(within.type == EXSBlock::TYPE_GROUP); + parse(); + } + void parse() + { + resetPosition(); + volume = readByteToInt(); + pan = readByteToInt(); + polyphony = readByteToInt(); + int options = readByteToInt(); + mute = (options & 16) > 0; // 0 = OFF, 1 = ON + releaseTriggerDecay = (options & 64) > 0; // 0 = OFF, 1 = ON + fixedSampleSelect = (options & 128) > 0; // 0 = OFF, 1 = ON + + exclusive = readByteToInt(); + minVelocity = readByteToInt(); + maxVelocity = readByteToInt(); + sampleSelectRandomOffset = readByteToInt(); + + skip(8); + + releaseTriggerTime = readU16(); + + skip(14); + + velocityRangExFade = readByteToInt() - 128; + velocityRangExFadeType = readByteToInt(); + keyrangExFadeType = readByteToInt(); + keyrangExFade = readByteToInt() - 128; + + skip(2); + + enableByTempoLow = readByteToInt(); + enableByTempoHigh = readByteToInt(); + + skip(1); + + cutoffOffset = readByteToInt(); + skip(1); + + resoOffset = readByteToInt(); + skip(12); + + env1AttackOffset = (int)readU32(); + env1DecayOffset = (int)readU32(); + env1SustainOffset = (int)readU32(); + env1ReleaseOffset = (int)readU32(); + skip(1); + + releaseTrigger = readByteToInt() > 0; + output = readByteToInt(); + enableByNoteValue = readByteToInt(); + + if (position < within.size) + { + skip(4); + + roundRobinGroupPos = (int)readU32(); + enableByType = readByteToInt(); + enableByNote = enableByType == 1; + enableByRoundRobin = enableByType == 2; + enableByControl = enableByType == 3; + enableByBend = enableByType == 4; + enableByChannel = enableByType == 5; + enableByArticulation = enableByType == 6; + enablebyTempo = enableByType == 7; + + enableByControlValue = readByteToInt(); + enableByControlLow = readByteToInt(); + enableByControlHigh = readByteToInt(); + startNote = readByteToInt(); + endNote = readByteToInt(); + enableByMidiChannel = readByteToInt(); + enableByArticulationValue = readByteToInt(); + } + } +}; + +struct EXSSample : EXSObject +{ + int waveDataStart; + int length; + int sampleRate; + int bitDepth; + int channels; + int channels2; + std::string type; + int size; + bool isCompressed = false; + std::string filePath; + std::string fileName; + + EXSSample(EXSBlock in) : EXSObject(in) + { + assert(in.type == EXSBlock::TYPE_SAMPLE); + assert(in.size); + parse(); + } + void parse() + { + resetPosition(); + waveDataStart = readU32(); + length = readU32(); + sampleRate = readU32(); + bitDepth = readU32(); + + channels = readU32(); + channels2 = readU32(); + + skip(4); + + type = readStringNoTerm(4); + size = readU32(); + isCompressed = readU32() > 0; + + skip(40); + + filePath = readStringNullTerm(256); + + // If not present the name from the header is used! + fileName = (position < within.size) ? readStringNullTerm(256) : within.name; + } +}; +bool importEXS(const fs::path &p, engine::Engine &e) +{ + SCLOG("Importing EXS from " << p.u8string()); + std::ifstream inputFile(p, std::ios_base::binary); + + auto block = EXSBlock(); + std::vector zones; + std::vector groups; + std::vector samples; + + while (block.read(inputFile)) + { + switch (block.type) + { + case EXSBlock::TYPE_INSTRUMENT: + { + auto inst = EXSInstrument(block); + } + break; + case EXSBlock::TYPE_ZONE: + { + zones.emplace_back(block); + } + break; + case EXSBlock::TYPE_GROUP: + { + groups.emplace_back(block); + } + break; + case EXSBlock::TYPE_SAMPLE: + { + samples.emplace_back(block); + } + break; + case EXSBlock::TYPE_PARAMS: + { + SCLOG("EXS Parameter Parsing not yet implemented"); + } + break; + default: + { + SCLOG("Un-parsed EXS block type " << block.type); + break; + } + } + } + + auto pt = std::clamp(e.getSelectionManager()->selectedPart, (int16_t)0, (int16_t)numParts); + auto &part = e.getPatch()->getPart(pt); + + std::unordered_map exsIndexToGroupIndex; + std::vector groupIndexByOrder; + std::unordered_map exsIndexToSampleId; + std::vector sampleIDByOrder; + + for (auto &s : samples) + { + auto p = fs::path{s.filePath} / s.fileName; + auto lsid = e.getSampleManager()->loadSampleByPath(p); + if (lsid.has_value()) + { + sampleIDByOrder.push_back(*lsid); + exsIndexToSampleId[s.within.index] = *lsid; + } + } + + for (auto &g : groups) + { + auto groupId = part->addGroup() - 1; + auto &group = part->getGroup(groupId); + group->name = g.within.name; + exsIndexToGroupIndex[g.within.index] = groupId; + groupIndexByOrder.push_back(groupId); + } + + for (auto &z : zones) + { + auto exsgi = z.groupIndex; + auto exssi = z.sampleIndex; + + if (exsIndexToGroupIndex.find(exsgi) == exsIndexToGroupIndex.end()) + { + SCLOG("Out-of-bounds group index : " << SCD(exsgi)); + continue; + } + if (exssi < 0 || exssi >= sampleIDByOrder.size()) + { + SCLOG("Out-of-bounds sample index " << SCD(exsgi) << SCD(exssi)); + continue; + } + auto gi = exsIndexToGroupIndex[exsgi]; + auto si = sampleIDByOrder[exssi]; + auto &group = part->getGroup(gi); + auto zone = std::make_unique(si); + + zone->mapping.rootKey = z.key; + zone->mapping.keyboardRange.keyStart = z.keyLow; + zone->mapping.keyboardRange.keyEnd = z.keyHigh; + zone->mapping.velocityRange.velStart = z.velocityLow; + zone->mapping.velocityRange.velEnd = z.velocityHigh; + + zone->attachToSample(*(e.getSampleManager())); + group->addZone(std::move(zone)); + } + + return true; +} +} // namespace scxt::exs_support \ No newline at end of file diff --git a/src/sample/exs_support/exs_import.h b/src/sample/exs_support/exs_import.h new file mode 100644 index 00000000..280a202e --- /dev/null +++ b/src/sample/exs_support/exs_import.h @@ -0,0 +1,52 @@ +/* + * Shortcircuit XT - a Surge Synth Team product + * + * A fully featured creative sampler, available as a standalone + * and plugin for multiple platforms. + * + * Copyright 2019 - 2024, Various authors, as described in the github + * transaction log. + * + * ShortcircuitXT is released under the Gnu General Public Licence + * V3 or later (GPL-3.0-or-later). The license is found in the file + * "LICENSE" in the root of this repository or at + * https://www.gnu.org/licenses/gpl-3.0.en.html + * + * Individual sections of code which comprises ShortcircuitXT in this + * repository may also be used under an MIT license. Please see the + * section "Licensing" in "README.md" for details. + * + * ShortcircuitXT is inspired by, and shares code with, the + * commercial product Shortcircuit 1 and 2, released by VemberTech + * in the mid 2000s. The code for Shortcircuit 2 was opensourced in + * 2020 at the outset of this project. + * + * All source for ShortcircuitXT is available at + * https://github.com/surge-synthesizer/shortcircuit-xt + */ + +#ifndef SCXT_SRC_SAMPLE_EXS_SUPPORT_EXS_IMPORT_H +#define SCXT_SRC_SAMPLE_EXS_SUPPORT_EXS_IMPORT_H + +#include "filesystem/import.h" +#include + +namespace scxt::exs_support +{ +/* + * The EXS import ability is based almost entirely on the java + * implementation from mossgraber's ConvertWithMoss available + * + * https://github.com/git-moss/ConvertWithMoss/ + * + * which is released under Gnu GPL3. None of the code is copied + * here but without the reference to that code, this implementation + * would have been impossible. + * + * The implementation here is very incomplete, and may be removed + * before our 1.0 release if it turns out to not be tenable. + */ +bool importEXS(const fs::path &, engine::Engine &); +} // namespace scxt::exs_support + +#endif // SHORTCIRCUITXT_EXS_IMPORT_H