diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f7186be --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.syx diff=syx + +* text=auto +*.jucer text eol=crlf +**/AppConfig.h text eol=crlf +3rd_party/** -text \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..99145f5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,64 @@ +# Windows Temp Cache Files +[Tt]humbs.db + +#Visual Studio files +*.[Oo]bj +*.aps +*.pch +*.vspscc +*.vssscc +*.ncb +*.suo +*.tlb +*.tlh +*.bak +*.[Cc]ache +*.ilk +*.log +*.sbr +*.sdf +*.opensdf +*.xccheckout +*.VC.opendb +*.VC.db +ipch/ +obj/ + +# Build products +build/ +*.o + +# Other source repository archive directories (protects when importing) +.hg +.svn +CVS + +#Other files +._* +*.mode1v3 +*.pbxuser +*.perspectivev3 +*.user +*.suo +*.obj +*.ilk +*.pch +*.pdb +*.dep +*.idb +*.manifest +*.manifest.res +*.o +*.d +*.sdf +*.xcscmblueprint +xcuserdata +contents.xcworkspacedata +.DS_Store +.svn +enc_temp_folder/ +**/Builds/ +**/JuceLibraryCode/ +!**/JuceLibraryCode/AppConfig.h +install/testresults.txt +**/bin diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e8e6ae7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "modules/juce"] + path = modules/juce + url = git@github.com:WeAreROLI/JUCE.git diff --git a/README.md b/README.md new file mode 100644 index 0000000..a58b7dd --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# pluginval + +pluginval is a cross-platform plugin validator and tester application. It is designed to be used by both plugin and host developers to ensure stability and compatibility between plugins and hosts. + +###### Highlights: + - Test VST/AU/VST3 plugins + - Compatible with macOS/Windows/Linux + - Run in GUI or headless mode + - Validation is performed in a separate process to avoid crashing + + +### Installation + +Either grab one of the pre-compiled binaries from the Releases page or clone the repo and build from sources. +The projects are generated by the Projucer so you may have to build that first to generate the various project files. The easiest way to do this is to run one of the build scripts in the `install` directory. +```sh +$ git clone git@github.com:Tracktion/pluginval.git +$ cd pluginval/ +$ git submodule init +$ git submodule update +$ cd install +$ ./mac_build +``` + +### Running in GUI Mode +Once the app has built for your platform it will be found in `/bin`. Simply open the app to start it in GUI mode. Once open, you'll be presented with an empty plugin list. Click "Options" to scan for plugins, making sure to add any directories required. + +Once the list has populated, simply select a plugin and press the "Test Selected" button to validate it. The plugin will be loaded and each of the tests run in turn. Any output from the tests will be shown on the "Console" tab. +If you find problems with a plugin, this can be useful to send to the plugin developers. + +### Running in Headless Mode +As well as being a GUI app, `pluginval` can be run from the command line in a headless mode. +This is great if you want to add validation as part of your CI process and be notified immidiately if tests start failing. + +###### Basic usage is as follows: +``` +./pluginval --strictnessLevel 5 --validate +``` +This will run all the tests up to level 5 on the plugin at the specified path. +Output will be fed to the console. +If all the tests pass cleanly, `pluginval` will return with an exit code of `0`. If any tests fail, the exit code will be `1`. +This means you can check the exit code on your various CI and mark builds a failing if all tests don't pass. + +`strictnessLevel` is optional but can be between 1 & 10 with 5 being generally recognised as the lowest level for host compatibility. Lower levels are generally quick tests, mainly checking call coverage for crashes. Higher levels contain tests which take longer to run such as parameter fuzz tests and multiple state restoration. + +###### You can also list all the options with: +``` +./pluginval -h +``` + +### Contributing +If you would like to contribute to the project please do! It's very simple to add tests, simply: +1) Subclass `PluginTest` + ``` + struct FuzzParametersTest : public PluginTest + ``` +2) Override `runTest` to perform any tests + ``` + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + ``` +3) Log passes or failures with the `UnitTest` parameter + ``` + ut.expect (editor != nullptr, "Unable to create editor"); + ``` +4) Register your test with a static instance of it in a cpp file: + ``` + static FuzzParametersTest fuzzParametersTest; + ``` + +If you have a case you would like tests, please simply write the test in a fork and create a pull request. The more tests the better! + +### Todos + - Create a better logo! + - Write more Tests + - Possibly add more command line options + -- Run only specif tests, either named or from a certain strictness level + +License +---- + +Licencing is under the `GPLv3` as we want `pluginval` to be as transparent as possible. If this conflicts with your requirements though please let us know and we can accomodate these. \ No newline at end of file diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp new file mode 100644 index 0000000..347fda2 --- /dev/null +++ b/Source/CommandLine.cpp @@ -0,0 +1,288 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "CommandLine.h" +#include "Validator.h" + +//============================================================================== +struct CommandLineError +{ + CommandLineError (const String& s) : message (s) {} + + const String message; +}; + +struct TestError +{ + TestError (const String& s) : message (s) {} + TestError (const String& s, int code) : message (s), exitCode (code) {} + + const String message; + const int exitCode = 1; +}; + + +//============================================================================== +static bool matchArgument (const String& arg, const String& possible) +{ + return arg == possible + || arg == "-" + possible + || arg == "--" + possible; +} + + +static int indexOfArgument (const StringArray& args, const String& possible) +{ + for (auto& a : args) + { + if (a == possible + || a == "-" + possible + || a == "--" + possible) + return args.indexOf (a); + } + + return -1; +} + +static bool containsArgument (const StringArray& args, const String& possible) +{ + return indexOfArgument (args, possible) != -1; +} + +static void checkArgumentCount (const StringArray& args, int minNumArgs) +{ + if (args.size() < minNumArgs) + throw CommandLineError ("Not enough arguments!"); +} + +static void hideDockIcon() +{ + #if JUCE_MAC + Process::setDockIconVisible (false); + #endif +} + + +//============================================================================== +struct CommandLineValidator : private ChangeListener, + private Validator::Listener +{ + CommandLineValidator() + { + validator.addChangeListener (this); + validator.addListener (this); + } + + ~CommandLineValidator() + { + validator.removeChangeListener (this); + validator.removeListener (this); + } + + void validate (const StringArray& fileOrIDs, int strictnessLevel) + { + inProgress = true; + validator.validate (fileOrIDs, strictnessLevel); + + while (inProgress && validator.isConnected()) + Thread::sleep (100); + + if (exitError) + throw (*exitError); + + if (numFailures > 0) + throw (TestError ("*** FAILED: " + String (numFailures) + " TESTS")); + } + +private: + Validator validator; + String currentID; + std::atomic inProgress { false }; + std::atomic numFailures { 0 }; + std::unique_ptr exitError; + + void changeListenerCallback (ChangeBroadcaster*) override + { + if (! validator.isConnected() && currentID.isNotEmpty()) + { + logMessage ("\n*** FAILED: VALIDATION CRASHED"); + currentID = String(); + } + + if (! validator.isConnected()) + inProgress = false; + } + + void validationStarted (const String& id) override + { + currentID = id; + logMessage ("Started validating: " + id); + } + + void logMessage (const String& m) override + { + std::cout << m << "\n"; + } + + void itemComplete (const String& id, int numItemFailures) override + { + logMessage ("\nFinished validating: " + id); + + if (numItemFailures == 0) + logMessage ("ALL TESTS PASSED"); + else + logMessage ("*** FAILED: " + String (numItemFailures) + " TESTS"); + + numFailures += numItemFailures; + currentID = String(); + inProgress = false; + } + + void allItemsComplete() override + { + } + + void connectionLost() override + { + if (currentID.isNotEmpty()) + { + logMessage ("\n*** FAILED: VALIDATION CRASHED"); + exitError = std::make_unique ("\n*** FAILED: VALIDATION CRASHED WHILST VALIDATING " + currentID); + currentID = String(); + } + else + { + exitError = std::make_unique ("\n*** FAILED: VALIDATION CRASHED"); + } + + inProgress = false; + } +}; + +static void validate (const StringArray& args, int strictnessLevel) +{ + hideDockIcon(); + checkArgumentCount (args, 2); + const int startIndex = indexOfArgument (args, "--validate"); + + if (startIndex != -1) + { + StringArray fileOrIDs; + fileOrIDs.addArray (args, startIndex + 1); + + for (int i = fileOrIDs.size(); --i >= 0;) + if (fileOrIDs.getReference (i).startsWith ("--")) + fileOrIDs.remove (i); + + if (! fileOrIDs.isEmpty()) + CommandLineValidator().validate (fileOrIDs, strictnessLevel); + } +} + +static int getStrictnessLevel (const StringArray& args) +{ + const int strictnessIndex = indexOfArgument (args, "strictnessLevel"); + + if (strictnessIndex != -1) + { + if (args.size() > strictnessIndex) + { + const int strictness = args[strictnessIndex + 1].getIntValue(); + + if (strictness > 1 && strictness <= 10) + return strictness; + } + + throw CommandLineError ("Missing strictness level argument! (Must be between 1 - 10)"); + } + + return 5; +} + +//============================================================================== +static void showHelp() +{ + hideDockIcon(); + + const String appName (JUCEApplication::getInstance()->getApplicationName()); + + std::cout << "//==============================================================================" + << appName << std::endl + << SystemStats::getJUCEVersion() << std::endl + << std::endl + << "Description: " << std::endl + << " Validate plugins to test compatibility with hosts and verify plugin API conformance" << std::endl << std::endl + << "Usage: " + << std::endl + << " --validate [list]" << std::endl + << " Validates the files (or IDs for AUs)." << std::endl + << " --strictnessLevel [1-10]" << std::endl + << " Sets the strictness level to use. A minimum level of 5 (also the default) is recomended for compatibility. Higher levels include longer, more thorough tests such as fuzzing." << std::endl + << std::endl + << "Exit code: " + << std::endl + << " 0 if all tests complete successfully" << std::endl + << " 1 if there are any errors" << std::endl; +} + + +//============================================================================== +int performCommandLine (const String& commandLine) +{ + StringArray args; + args.addTokens (commandLine, true); + args.trim(); + + for (auto& s : args) + s = s.unquoted(); + + String command (args[0]); + + try + { + if (matchArgument (command, "help")) { showHelp(); return 0; } + if (matchArgument (command, "h")) { showHelp(); return 0; } + if (containsArgument (args, "validate")) { validate (args, getStrictnessLevel (args)); return 0; } + } + catch (const TestError& error) + { + std::cout << error.message << std::endl << std::endl; + JUCEApplication::getInstance()->setApplicationReturnValue (error.exitCode); + + return error.exitCode; + } + catch (const CommandLineError& error) + { + std::cout << error.message << std::endl << std::endl; + return 1; + } + + return commandLineNotPerformed; +} + +bool shouldPerformCommandLine (const String& commandLine) +{ + StringArray args; + args.addTokens (commandLine, true); + args.trim(); + + String command (args[0]); + + if (matchArgument (command, "help")) return true; + if (matchArgument (command, "h")) return true; + if (containsArgument (args, "validate")) return true; + + return false; +} + diff --git a/Source/CommandLine.h b/Source/CommandLine.h new file mode 100644 index 0000000..55df6e7 --- /dev/null +++ b/Source/CommandLine.h @@ -0,0 +1,23 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +int performCommandLine (const String& commandLine); +bool shouldPerformCommandLine (const String& commandLine); + +enum { commandLineNotPerformed = 0x72346231 }; + diff --git a/Source/Main.cpp b/Source/Main.cpp new file mode 100644 index 0000000..3e0afa7 --- /dev/null +++ b/Source/Main.cpp @@ -0,0 +1,152 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "../JuceLibraryCode/JuceHeader.h" +#include "MainComponent.h" +#include "Validator.h" +#include "CommandLine.h" + +//============================================================================== +class PluginValidatorApplication : public JUCEApplication, + private AsyncUpdater +{ +public: + //============================================================================== + PluginValidatorApplication() = default; + + PropertiesFile& getAppPreferences() + { + jassert (propertiesFile); // Calling this from the child process? + return *propertiesFile; + } + + //============================================================================== + const String getApplicationName() override { return ProjectInfo::projectName; } + const String getApplicationVersion() override { return ProjectInfo::versionString; } + bool moreThanOneInstanceAllowed() override { return true; } + + //============================================================================== + void initialise (const String& commandLine) override + { + if (shouldPerformCommandLine (commandLine)) + { + triggerAsyncUpdate(); + return; + } + + if (invokeSlaveProcessValidator (commandLine)) + return; + + validator = std::make_unique(); + propertiesFile.reset (getPropertiesFile()); + mainWindow = std::make_unique (*validator, getApplicationName()); + } + + void shutdown() override + { + mainWindow.reset(); + validator.reset(); + } + + //============================================================================== + void systemRequestedQuit() override + { + // This is called when the app is being asked to quit: you can ignore this + // request and let the app carry on running, or call quit() to allow the app to close. + quit(); + } + + void anotherInstanceStarted (const String&) override + { + // When another instance of the app is launched while this one is running, + // this method is invoked, and the commandLine parameter tells you what + // the other instance's command-line arguments were. + } + + //============================================================================== + /* + This class implements the desktop window that contains an instance of + our MainComponent class. + */ + class MainWindow : public DocumentWindow + { + public: + MainWindow (Validator& v, String name) + : DocumentWindow (name, + Desktop::getInstance().getDefaultLookAndFeel() + .findColour (ResizableWindow::backgroundColourId), + DocumentWindow::allButtons) + { + setUsingNativeTitleBar (true); + setContentOwned (new MainComponent (v), true); + + setResizable (true, false); + centreWithSize (getWidth(), getHeight()); + setVisible (true); + } + + void closeButtonPressed() override + { + // This is called when the user tries to close this window. Here, we'll just + // ask the app to quit when this happens, but you can change this to do + // whatever you need. + JUCEApplication::getInstance()->systemRequestedQuit(); + } + + /* Note: Be careful if you override any DocumentWindow methods - the base + class uses a lot of them, so by overriding you might break its functionality. + It's best to do all your work in your content component instead, but if + you really have to override any DocumentWindow methods, make sure your + subclass also calls the superclass's method. + */ + + private: + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (MainWindow) + }; + +private: + std::unique_ptr validator; + std::unique_ptr propertiesFile; + std::unique_ptr mainWindow; + + static PropertiesFile* getPropertiesFile() + { + PropertiesFile::Options opts; + opts.millisecondsBeforeSaving = 2000; + opts.storageFormat = PropertiesFile::storeAsXML; + + opts.applicationName = "PluginValidator"; + opts.filenameSuffix = ".xml"; + opts.folderName = "PluginValidator"; + opts.osxLibrarySubFolder = "Application Support"; + + return new PropertiesFile (opts.getDefaultFile(), opts); + } + + void handleAsyncUpdate() override + { + if (performCommandLine (JUCEApplication::getCommandLineParameters()) != commandLineNotPerformed) + quit(); + } +}; + +//============================================================================== +// This macro generates the main() routine that launches the app. +START_JUCE_APPLICATION (PluginValidatorApplication) + +PropertiesFile& getAppPreferences() +{ + auto app = dynamic_cast (PluginValidatorApplication::getInstance()); + return app->getAppPreferences(); +} diff --git a/Source/MainComponent.cpp b/Source/MainComponent.cpp new file mode 100644 index 0000000..575e30a --- /dev/null +++ b/Source/MainComponent.cpp @@ -0,0 +1,129 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "MainComponent.h" +#include "PluginTests.h" + +//============================================================================== +MainComponent::MainComponent (Validator& v) + : validator (v) +{ + formatManager.addDefaultFormats(); + + const auto tabCol = getLookAndFeel().findColour (ResizableWindow::backgroundColourId); + addAndMakeVisible (tabbedComponent); + tabbedComponent.addTab ("Plugin List", tabCol, &pluginListComponent, false); + tabbedComponent.addTab ("Console", tabCol, &console, false); + + addAndMakeVisible (connectionStatus); + addAndMakeVisible (clearButton); + addAndMakeVisible (saveButton); + addAndMakeVisible (testSelectedButton); + addAndMakeVisible (testAllButton); + + testSelectedButton.onClick = [this] + { + auto rows = pluginListComponent.getTableListBox().getSelectedRows(); + Array plugins; + + for (int i = 0; i < rows.size(); ++i) + if (auto pd = knownPluginList.getType (rows[i])) + plugins.add (pd); + + validator.validate (plugins, 10); + }; + + testAllButton.onClick = [this] + { + Array plugins; + + for (int i = 0; i < knownPluginList.getNumTypes(); ++i) + if (auto pd = knownPluginList.getType (i)) + plugins.add (pd); + + validator.validate (plugins, 10); + }; + + clearButton.onClick = [this] + { + console.clearLog(); + }; + + saveButton.onClick = [this] + { + FileChooser fc (TRANS("Save Log File"), + getAppPreferences().getValue ("lastSaveLocation", File::getSpecialLocation (File::userDesktopDirectory).getFullPathName()), + "*.txt"); + + if (fc.browseForFileToSave (true)) + { + const auto f = fc.getResult(); + + if (f.replaceWithText (console.getLog())) + { + getAppPreferences().setValue ("lastSaveLocation", f.getFullPathName()); + } + else + { + AlertWindow::showMessageBoxAsync (AlertWindow::WarningIcon, TRANS("Unable to Save"), + TRANS("Unable to save to the file at location: XYYX").replace ("XYYX", f.getFullPathName())); + } + } + }; + + if (auto xml = std::unique_ptr (getAppPreferences().getXmlValue ("scannedPlugins"))) + knownPluginList.recreateFromXml (*xml); + + knownPluginList.addChangeListener (this); + + setSize (800, 600); +} + +MainComponent::~MainComponent() +{ + savePluginList(); +} + +//============================================================================== +void MainComponent::paint (Graphics& g) +{ + g.fillAll (getLookAndFeel().findColour (ResizableWindow::backgroundColourId)); +} + +void MainComponent::resized() +{ + auto r = getLocalBounds(); + + auto bottomR = r.removeFromBottom (28); + saveButton.setBounds (bottomR.removeFromRight (100).reduced (2)); + clearButton.setBounds (bottomR.removeFromRight (100).reduced (2)); + + connectionStatus.setBounds (bottomR.removeFromLeft (bottomR.getHeight()).reduced (2)); + testSelectedButton.setBounds (bottomR.removeFromLeft (130).reduced (2)); + testAllButton.setBounds (bottomR.removeFromLeft (130).reduced (2)); + + tabbedComponent.setBounds (r); +} + +//============================================================================== +void MainComponent::savePluginList() +{ + if (auto xml = std::unique_ptr (knownPluginList.createXml())) + getAppPreferences().setValue ("scannedPlugins", xml.get()); +} + +void MainComponent::changeListenerCallback (ChangeBroadcaster*) +{ + savePluginList(); +} diff --git a/Source/MainComponent.h b/Source/MainComponent.h new file mode 100644 index 0000000..e7a2b8c --- /dev/null +++ b/Source/MainComponent.h @@ -0,0 +1,232 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "../JuceLibraryCode/JuceHeader.h" +#include "Validator.h" + +PropertiesFile& getAppPreferences(); + +//============================================================================== +struct ConnectionStatus : public Component, + private ChangeListener, + private Validator::Listener +{ + ConnectionStatus (Validator& v) + : validator (v) + { + validator.addListener (this); + validator.addChangeListener (this); + } + + ~ConnectionStatus() + { + validator.removeListener (this); + validator.removeChangeListener (this); + } + + void paint (Graphics& g) override + { + auto r = getLocalBounds().toFloat(); + + g.setColour ([this] { + switch (status) + { + case Status::disconnected: return Colours::darkred; + case Status::validating: return Colours::orange; + case Status::connected: + case Status::complete: return Colours::lightgreen; + } + + return Colours::darkred; + }()); + g.fillEllipse (r); + + g.setColour (Colours::darkgrey); + g.drawEllipse (r.reduced (1.0f), 2.0f); + } + +private: + enum class Status + { + disconnected, + connected, + validating, + complete + }; + + Validator& validator; + std::atomic status { Status::disconnected }; + + void setStatus (Status newStatus) + { + status = newStatus; + MessageManager::getInstance()->callAsync ([sp = SafePointer (this)] () mutable { if (sp != nullptr) sp->repaint(); }); + } + + void changeListenerCallback (ChangeBroadcaster*) override + { + setStatus (validator.isConnected() ? Status::connected : Status::disconnected); + } + + void validationStarted (const String&) override + { + setStatus (Status::validating); + } + + void logMessage (const String&) override + { + } + + void itemComplete (const String&, int) override + { + } + + void allItemsComplete() override + { + setStatus (Status::complete); + } +}; + +//============================================================================== +struct ConsoleComponent : public Component, + private ChangeListener, + private Validator::Listener +{ + ConsoleComponent (Validator& v) + : validator (v) + { + validator.addChangeListener (this); + validator.addListener (this); + + addAndMakeVisible (editor); + editor.setReadOnly (true); + editor.setLineNumbersShown (false); + editor.setScrollbarThickness (8); + } + + ~ConsoleComponent() + { + validator.removeChangeListener (this); + validator.removeListener (this); + } + + String getLog() const + { + return codeDocument.getAllContent(); + } + + void clearLog() + { + codeDocument.replaceAllContent (String()); + } + + void resized() override + { + auto r = getLocalBounds(); + editor.setBounds (r); + } + +private: + Validator& validator; + + CodeDocument codeDocument; + CodeEditorComponent editor { codeDocument, nullptr }; + String currentID; + + void changeListenerCallback (ChangeBroadcaster*) override + { + if (! validator.isConnected() && currentID.isNotEmpty()) + { + logMessage ("\n*** FAILED: VALIDATION CRASHED"); + currentID = String(); + } + } + + void validationStarted (const String& id) override + { + currentID = id; + logMessage ("Started validating: " + id); + } + + void logMessage (const String& m) override + { + MessageManager::getInstance()->callAsync ([sp = SafePointer (this), m] () mutable + { + if (sp != nullptr) + { + sp->codeDocument.insertText (sp->editor.getCaretPos(), m + "\n"); + sp->editor.scrollToKeepCaretOnScreen(); + } + }); + + std::cout << m << "\n"; + } + + void itemComplete (const String& id, int numFailures) override + { + logMessage ("\nFinished validating: " + id); + + if (numFailures == 0) + logMessage ("ALL TESTS PASSED"); + else + logMessage ("*** FAILED: " + String (numFailures) + " TESTS"); + + currentID = String(); + } + + void allItemsComplete() override + { + } +}; + +//============================================================================== +/* + This component lives inside our window, and this is where you should put all + your controls and content. +*/ +class MainComponent : public Component, + private ChangeListener +{ +public: + //============================================================================== + MainComponent (Validator&); + ~MainComponent(); + + //============================================================================== + void paint (Graphics&) override; + void resized() override; + +private: + //============================================================================== + Validator& validator; + + AudioPluginFormatManager formatManager; + KnownPluginList knownPluginList; + + TabbedComponent tabbedComponent { TabbedButtonBar::TabsAtTop }; + PluginListComponent pluginListComponent { formatManager, knownPluginList, + getAppPreferences().getFile().getSiblingFile ("PluginsListDeadMansPedal"), + &getAppPreferences() }; + ConsoleComponent console { validator }; + TextButton testSelectedButton { "Test Selected" }, testAllButton { "Test All" }, clearButton { "Clear log" }, saveButton { "Save log" }; + ConnectionStatus connectionStatus { validator }; + + void savePluginList(); + + void changeListenerCallback (ChangeBroadcaster*) override; + + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (MainComponent) +}; diff --git a/Source/PluginTests.cpp b/Source/PluginTests.cpp new file mode 100644 index 0000000..71e2375 --- /dev/null +++ b/Source/PluginTests.cpp @@ -0,0 +1,106 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "PluginTests.h" +#include "TestUtilities.h" + +PluginTests::PluginTests (const String& fileOrIdentifier, int strictnessLevelToTest) + : UnitTest ("PluginValidator"), + fileOrID (fileOrIdentifier), + strictnessLevel (strictnessLevelToTest) +{ + jassert (isPositiveAndNotGreaterThan (strictnessLevelToTest, 10)); + formatManager.addDefaultFormats(); +} + +PluginTests::PluginTests (const PluginDescription& desc, int strictnessLevelToTest) + : PluginTests (String(), strictnessLevelToTest) +{ + typesFound.add (new PluginDescription (desc)); +} + +void PluginTests::runTest() +{ + logMessage ("Validation started: " + Time::getCurrentTime().toString (true, true) + "\n"); + + if (fileOrID.isNotEmpty()) + { + beginTest ("Scan for known types: " + fileOrID); + knownPluginList.scanAndAddDragAndDroppedFiles (formatManager, StringArray (fileOrID), typesFound); + logMessage ("Num types found: " + String (typesFound.size())); + expect (! typesFound.isEmpty(), "No types found"); + } + + for (auto pd : typesFound) + testType (*pd); +} + +std::unique_ptr PluginTests::testOpenPlugin (const PluginDescription& pd) +{ + String errorMessage; + auto instance = std::unique_ptr (formatManager.createPluginInstance (pd, 44100.0, 512, errorMessage)); + expectEquals (errorMessage, String()); + expect (instance != nullptr, "Unable to create AudioPluginInstance"); + + return instance; +} + +void PluginTests::testType (const PluginDescription& pd) +{ + const auto idString = pd.createIdentifierString(); + logMessage ("\nTesting plugin: " + idString); + + { + beginTest ("Open plugin (cold)"); + StopwatchTimer sw; + testOpenPlugin (pd); + logMessage ("\nTime taken to open plugin (cold): " + sw.getDescription()); + } + + { + beginTest ("Open plugin (warm)"); + StopwatchTimer sw; + + if (auto instance = testOpenPlugin (pd)) + { + logMessage ("\nTime taken to open plugin (warm): " + sw.getDescription()); + + for (auto t : PluginTest::getAllTests()) + { + if (strictnessLevel < t->strictnessLevel) + continue; + + StopwatchTimer sw2; + beginTest (t->name); + + if (t->needsToRunOnMessageThread()) + { + WaitableEvent completionEvent; + MessageManager::getInstance()->callAsync ([&, this]() mutable + { + t->runTest (*this, *instance); + completionEvent.signal(); + }); + completionEvent.wait(); + } + else + { + t->runTest (*this, *instance); + } + + logMessage ("\nTime taken to run test: " + sw2.getDescription()); + } + } + } +} diff --git a/Source/PluginTests.h b/Source/PluginTests.h new file mode 100644 index 0000000..23f337e --- /dev/null +++ b/Source/PluginTests.h @@ -0,0 +1,96 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +//============================================================================== +/** + Represents a test to be run on a plugin instance. + Override the runTest test method to perform the tests. + Create a static instance of any subclasses to automatically register tests. +*/ +struct PluginTest +{ + /** + Creates a named PluginTest. + @param testName The name of the test + @param testStrictnessLevel The conformance level of the test + */ + PluginTest (const String& testName, int testStrictnessLevel) + : name (testName), strictnessLevel (testStrictnessLevel) + { + jassert (isPositiveAndNotGreaterThan (strictnessLevel, 10)); + getAllTests().add (this); + } + + /** Destructor. */ + virtual ~PluginTest() + { + getAllTests().removeFirstMatchingValue (this); + } + + /** Returns a static list of all the tests. */ + static Array& getAllTests() + { + static Array tests; + return tests; + } + + //============================================================================== + /** By default tests are run on background threads, you can override this to + return true of you need the runTest method to be called on the message thread. + */ + virtual bool needsToRunOnMessageThread() { return false; } + + /** Override to perform any tests. + Note that because PluginTest doesn't not inherit from UnitTest (due to being passed + in the AudioPluginInstance), you can use the UnitTest parameter to log messages or + call expect etc. + */ + virtual void runTest (UnitTest& runningTest, AudioPluginInstance&) = 0; + + //============================================================================== + const String name; + const int strictnessLevel; +}; + + +//============================================================================== +/** + The UnitTest which will create the plugins and run each of the registered tests on them. +*/ +struct PluginTests : public UnitTest +{ + /** Creates a set of tests for a fileOrIdentifier. */ + PluginTests (const String& fileOrIdentifier, int strictnessLevelToTest); + + /** Creates a set of tests for a PluginDescription. */ + PluginTests (const PluginDescription&, int strictnessLevelToTest); + + /** @internal. */ + void runTest() override; + +private: + const String fileOrID; + const int strictnessLevel; + AudioPluginFormatManager formatManager; + KnownPluginList knownPluginList; + + OwnedArray typesFound; + + std::unique_ptr testOpenPlugin (const PluginDescription&); + void testType (const PluginDescription&); +}; diff --git a/Source/TestUtilities.h b/Source/TestUtilities.h new file mode 100644 index 0000000..f18eaa0 --- /dev/null +++ b/Source/TestUtilities.h @@ -0,0 +1,97 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +//============================================================================== +struct StopwatchTimer +{ + StopwatchTimer() + { + reset(); + } + + void reset() + { + startTime = Time::getMillisecondCounter(); + } + + String getDescription() const + { + const auto relTime = RelativeTime::milliseconds (static_cast (Time::getMillisecondCounter() - startTime)); + return relTime.getDescription(); + } + +private: + uint32 startTime; +}; + +//============================================================================== +template +void iterateAudioBuffer (AudioBuffer& ab, UnaryFunction fn) +{ + float** sampleData = ab.getArrayOfWritePointers(); + + for (int c = ab.getNumChannels(); --c >= 0;) + for (int s = ab.getNumSamples(); --s >= 0;) + fn (sampleData[c][s]); +} + +static inline void fillNoise (AudioBuffer& ab) noexcept +{ + Random r; + ScopedNoDenormals noDenormals; + + float** sampleData = ab.getArrayOfWritePointers(); + + for (int c = ab.getNumChannels(); --c >= 0;) + for (int s = ab.getNumSamples(); --s >= 0;) + sampleData[c][s] = r.nextFloat() * 2.0f - 1.0f; +} + +static inline int countNaNs (AudioBuffer& ab) noexcept +{ + int count = 0; + iterateAudioBuffer (ab, [&count] (float s) + { + if (std::isnan (s)) + ++count; + }); + + return count; +} + +static inline int countInfs (AudioBuffer& ab) noexcept +{ + int count = 0; + iterateAudioBuffer (ab, [&count] (float s) + { + if (std::isinf (s)) + ++count; + }); + + return count; +} + +static inline int countSubnormals (AudioBuffer& ab) noexcept +{ + int count = 0; + iterateAudioBuffer (ab, [&count] (float s) + { + if (s != 0.0f && std::fpclassify (s) == FP_SUBNORMAL) + ++count; + }); + + return count; +} diff --git a/Source/Validator.cpp b/Source/Validator.cpp new file mode 100644 index 0000000..6e02ba6 --- /dev/null +++ b/Source/Validator.cpp @@ -0,0 +1,452 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "Validator.h" +#include "PluginTests.h" +#include + +//============================================================================== +struct ForwardingUnitTestRunner : public UnitTestRunner +{ + ForwardingUnitTestRunner (std::function fn) + : callback (std::move (fn)) + { + jassert (callback); + } + + void logMessage (const String& message) override + { + callback (message); + } + +private: + std::function callback; +}; + + +//============================================================================== +inline Array runTests (PluginTests& test, std::function callback) +{ + Array results; + ForwardingUnitTestRunner testRunner (std::move (callback)); + testRunner.setAssertOnFailure (false); + + Array testsToRun; + testsToRun.add (&test); + testRunner.runTests (testsToRun); + + for (int i = 0; i < testRunner.getNumResults(); ++i) + results.add (*testRunner.getResult (i)); + + return results; +} + +inline Array validate (const PluginDescription& pluginToValidate, int strictnessLevel, std::function callback) +{ + PluginTests test (pluginToValidate, strictnessLevel); + return runTests (test, std::move (callback)); +} + +inline Array validate (const String& fileOrIDToValidate, int strictnessLevel, std::function callback) +{ + PluginTests test (fileOrIDToValidate, strictnessLevel); + return runTests (test, std::move (callback)); +} + +inline int getNumFailures (Array results) +{ + return std::accumulate (results.begin(), results.end(), 0, + [] (int count, const UnitTestRunner::TestResult& r) { return count + r.failures; }); +} + +//============================================================================== +namespace IDs +{ + #define DECLARE_ID(name) const Identifier name (#name); + + DECLARE_ID(PLUGINS) + DECLARE_ID(PLUGIN) + DECLARE_ID(fileOrID) + DECLARE_ID(pluginDescription) + DECLARE_ID(strictnessLevel) + + DECLARE_ID(MESSAGE) + DECLARE_ID(type) + DECLARE_ID(text) + DECLARE_ID(log) + DECLARE_ID(numFailures) + + #undef DECLARE_ID +} + +//============================================================================== +// This is a token that's used at both ends of our parent-child processes, to +// act as a unique token in the command line arguments. +static const char* validatorCommandLineUID = "validatorUID"; + +// A few quick utility functions to convert between raw data and ValueTrees +static ValueTree memoryBlockToValueTree (const MemoryBlock& mb) +{ + return ValueTree::readFromData (mb.getData(), mb.getSize()); +} + +static MemoryBlock valueTreeToMemoryBlock (const ValueTree& v) +{ + MemoryOutputStream mo; + v.writeToStream (mo); + + return mo.getMemoryBlock(); +} + +//============================================================================== +class ValidatorMasterProcess : public ChildProcessMaster +{ +public: + ValidatorMasterProcess() = default; + + // Callback which can be set to log any calls sent to the slave + std::function logCallback; + + // Callback which can be set to be notified of a lost connection + std::function connectionLostCallback; + + //============================================================================== + // Callback which can be set to be informed when validation starts + std::function validationStartedCallback; + + // Callback which can be set to be informed when a log message is posted + std::function logMessageCallback; + + // Callback which can be set to be informed when a validation completes + std::function validationCompleteCallback; + + // Callback which can be set to be informed when all validations have been completed + std::function completeCallback; + + //============================================================================== + bool launch() + { + // Make sure we send 0 as the streamFlags args or the pipe can hang during DBG messages + return launchSlaveProcess (File::getSpecialLocation (File::currentExecutableFile), + validatorCommandLineUID, 2000, 0); + } + + //============================================================================== + void handleMessageFromSlave (const MemoryBlock& mb) override + { + auto v = memoryBlockToValueTree (mb); + + if (v.hasType (IDs::MESSAGE)) + { + const auto type = v[IDs::type].toString(); + + if (logMessageCallback && type == "log") + logMessageCallback (v[IDs::text].toString()); + + if (validationCompleteCallback && type == "result") + validationCompleteCallback (v[IDs::fileOrID].toString(), v[IDs::numFailures]); + + if (validationStartedCallback && type == "started") + validationStartedCallback (v[IDs::fileOrID].toString()); + + if (completeCallback && type == "complete") + completeCallback(); + } + + logMessage ("Received: " + v.toXmlString()); + } + + // This gets called if the slave process dies. + void handleConnectionLost() override + { + logMessage ("Connection lost to child process!"); + + if (connectionLostCallback) + connectionLostCallback(); + } + + //============================================================================== + /** Triggers validation of a set of files or IDs. */ + void validate (const StringArray& fileOrIDsToValidate, int strictnessLevel) + { + auto v = createPluginsTree (strictnessLevel); + + for (auto fileOrID : fileOrIDsToValidate) + { + jassert (fileOrID.isNotEmpty()); + v.appendChild ({ IDs::PLUGIN, {{ IDs::fileOrID, fileOrID }} }, nullptr); + } + + logMessage ("Sending: " + v.toXmlString()); + sendMessageToSlave (valueTreeToMemoryBlock (v)); + } + + /** Triggers validation of a set of PluginDescriptions. */ + void validate (const Array& pluginsToValidate, int strictnessLevel) + { + auto v = createPluginsTree (strictnessLevel); + + for (auto pd : pluginsToValidate) + if (auto xml = std::unique_ptr (pd->createXml())) + v.appendChild ({ IDs::PLUGIN, {{ IDs::pluginDescription, Base64::toBase64 (xml->createDocument ("")) }} }, nullptr); + + logMessage ("Sending: " + v.toXmlString()); + sendMessageToSlave (valueTreeToMemoryBlock (v)); + } + +private: + static ValueTree createPluginsTree (int strictnessLevel) + { + ValueTree v (IDs::PLUGINS); + v.setProperty (IDs::strictnessLevel, strictnessLevel, nullptr); + + return v; + } + + void logMessage (const String& s) + { + if (logCallback) + logCallback (s); + } +}; + +//============================================================================== +Validator::Validator() {} +Validator::~Validator() {} + +bool Validator::isConnected() const +{ + return masterProcess != nullptr; +} + +bool Validator::validate (const StringArray& fileOrIDsToValidate, int strictnessLevel) +{ + if (! ensureConnection()) + return false; + + masterProcess->validate (fileOrIDsToValidate, strictnessLevel); + return true; +} + +bool Validator::validate (const Array& pluginsToValidate, int strictnessLevel) +{ + if (! ensureConnection()) + return false; + + masterProcess->validate (pluginsToValidate, strictnessLevel); + return true; +} + +//============================================================================== +bool Validator::ensureConnection() +{ + if (! masterProcess) + { + sendChangeMessage(); + masterProcess = std::make_unique(); + + #if LOG_PIPE_COMMUNICATION + masterProcess->logCallback = [this] (const String& m) { DBG(m); }; + #endif + masterProcess->connectionLostCallback = [this] + { + listeners.call (&Listener::connectionLost); + triggerAsyncUpdate(); + }; + + masterProcess->validationStartedCallback = [this] (const String& id) { listeners.call (&Listener::validationStarted, id); }; + masterProcess->logMessageCallback = [this] (const String& m) { listeners.call (&Listener::logMessage, m); }; + masterProcess->validationCompleteCallback = [this] (const String& id, int numFailures) { listeners.call (&Listener::itemComplete, id, numFailures); }; + masterProcess->completeCallback = [this] { listeners.call (&Listener::allItemsComplete); triggerAsyncUpdate(); }; + + return masterProcess->launch(); + } + + return true; +} + +void Validator::handleAsyncUpdate() +{ + masterProcess.reset(); + sendChangeMessage(); +} + +//============================================================================== +/* This class gets instantiated in the child process, and receives messages from + the master process. +*/ +class ValidatorSlaveProcess : public ChildProcessSlave, + private Thread, + private DeletedAtShutdown +{ +public: + ValidatorSlaveProcess() + : Thread ("ValidatorSlaveProcess") + { + startThread (4); + } + + ~ValidatorSlaveProcess() + { + stopThread (5000); + } + + void handleMessageFromMaster (const MemoryBlock& mb) override + { + addRequest (mb); + } + + void handleConnectionLost() override + { + JUCEApplication::quit(); + } + +private: + CriticalSection requestsLock; + std::vector requestsToProcess; + + void logMessage (const String& m) + { + sendMessageToMaster (valueTreeToMemoryBlock ({ IDs::MESSAGE, {{ IDs::type, "log" }, { IDs::text, m }} })); + } + + void run() override + { + while (! threadShouldExit()) + { + processRequests(); + + const ScopedLock sl (requestsLock); + + if (requestsToProcess.empty()) + Thread::sleep (500); + } + } + + void addRequest (const MemoryBlock& mb) + { + { + const ScopedLock sl (requestsLock); + requestsToProcess.push_back (mb); + } + + notify(); + } + + void processRequests() + { + std::vector requests; + + { + const ScopedLock sl (requestsLock); + requests.swap (requestsToProcess); + } + + for (const auto& r : requests) + processRequest (r); + } + + void processRequest (MemoryBlock mb) + { + const ValueTree v (memoryBlockToValueTree (mb)); + + if (v.hasType (IDs::PLUGINS)) + { + const int strictnessLevel = v.getProperty (IDs::strictnessLevel, 5); + + for (auto c : v) + { + String fileOrID; + Array results; + + if (c.hasProperty (IDs::fileOrID)) + { + fileOrID = c[IDs::fileOrID].toString(); + sendMessageToMaster (valueTreeToMemoryBlock ({ + IDs::MESSAGE, {{ IDs::type, "started" }, { IDs::fileOrID, fileOrID }} + })); + + results = validate (c[IDs::fileOrID].toString(), strictnessLevel, [this] (const String& m) { logMessage (m); }); + } + else if (c.hasProperty (IDs::pluginDescription)) + { + MemoryOutputStream ms; + + if (Base64::convertFromBase64 (ms, c[IDs::pluginDescription].toString())) + { + if (auto xml = std::unique_ptr (XmlDocument::parse (ms.toString()))) + { + PluginDescription pd; + + if (pd.loadFromXml (*xml)) + { + fileOrID = pd.createIdentifierString(); + sendMessageToMaster (valueTreeToMemoryBlock ({ + IDs::MESSAGE, {{ IDs::type, "started" }, { IDs::fileOrID, fileOrID }} + })); + + results = validate (pd, strictnessLevel, [this] (const String& m) { logMessage (m); }); + } + } + } + } + + jassert (fileOrID.isNotEmpty()); + sendMessageToMaster (valueTreeToMemoryBlock ({ + IDs::MESSAGE, {{ IDs::type, "result" }, { IDs::fileOrID, fileOrID }, { IDs::numFailures, getNumFailures (results) }} + })); + } + } + + sendMessageToMaster (valueTreeToMemoryBlock ({ + IDs::MESSAGE, {{ IDs::type, "complete" }} + })); + } +}; + +#if JUCE_MAC +static void killWithoutMercy (int) +{ + kill (getpid(), SIGKILL); +} + +static void setupSignalHandling() +{ + const int signals[] = { SIGFPE, SIGILL, SIGSEGV, SIGBUS, SIGABRT }; + + for (int i = 0; i < numElementsInArray (signals); ++i) + { + ::signal (signals[i], killWithoutMercy); + ::siginterrupt (signals[i], 1); + } +} +#endif + +//============================================================================== +bool invokeSlaveProcessValidator (const String& commandLine) +{ + #if JUCE_MAC + setupSignalHandling(); + #endif + + ScopedPointer slave (new ValidatorSlaveProcess()); + + if (slave->initialiseFromCommandLine (commandLine, validatorCommandLineUID)) + { + slave.release(); // allow the slave object to stay alive - it'll handle its own deletion. + return true; + } + + return false; +} diff --git a/Source/Validator.h b/Source/Validator.h new file mode 100644 index 0000000..b333c46 --- /dev/null +++ b/Source/Validator.h @@ -0,0 +1,81 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include +#include "PluginTests.h" + +#ifndef LOG_PIPE_COMMUNICATION + #define LOG_PIPE_COMMUNICATION 0 +#endif + +class ValidatorMasterProcess; + +//============================================================================== +/** + Manages validation calls via a separate process and provides a listener + interface to find out the results of the validation. +*/ +class Validator : public ChangeBroadcaster, + private AsyncUpdater +{ +public: + //============================================================================== + /** Constructor. */ + Validator(); + + /** Destructor. */ + ~Validator(); + + /** Returns true if there is currently an open connection to a validator process. */ + bool isConnected() const; + + /** Validates an array of fileOrIDs. */ + bool validate (const StringArray& fileOrIDsToValidate, int strictnessLevel); + + /** Validates an array of PluginDescriptions. */ + bool validate (const Array& pluginsToValidate, int strictnessLevel); + + //============================================================================== + struct Listener + { + virtual ~Listener() = default; + + virtual void validationStarted (const String& idString) = 0; + virtual void logMessage (const String&) = 0; + virtual void itemComplete (const String& idString, int numFailures) = 0; + virtual void allItemsComplete() = 0; + virtual void connectionLost() {} + }; + + void addListener (Listener* l) { listeners.add (l); } + void removeListener (Listener* l) { listeners.remove (l); } + +private: + //============================================================================== + std::unique_ptr masterProcess; + ListenerList listeners; + + bool ensureConnection(); + + void handleAsyncUpdate() override; +}; + +//============================================================================== +/* The JUCEApplication::initialise method calls this function to allow the + child process to launch when the command line parameters indicate that we're + being asked to run as a child process. +*/ +bool invokeSlaveProcessValidator (const String& commandLine); diff --git a/Source/binarydata/icon.png b/Source/binarydata/icon.png new file mode 100644 index 0000000..69bf74a Binary files /dev/null and b/Source/binarydata/icon.png differ diff --git a/Source/tests/BasicTests.cpp b/Source/tests/BasicTests.cpp new file mode 100644 index 0000000..9041776 --- /dev/null +++ b/Source/tests/BasicTests.cpp @@ -0,0 +1,353 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "../PluginTests.h" +#include "../TestUtilities.h" + +//============================================================================== +struct PluginInfoTest : public PluginTest +{ + PluginInfoTest() + : PluginTest ("Plugin info", 1) + { + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + ut.logMessage ("\nPlugin name: " + instance.getName()); + ut.logMessage ("Alternative names: " + instance.getAlternateDisplayNames().joinIntoString ("|")); + ut.logMessage ("SupportsDoublePrecision: " + String (instance.supportsDoublePrecisionProcessing() ? "yes" : "no")); + ut.logMessage ("Reported latency: " + String (instance.getLatencySamples())); + ut.logMessage ("Reported taillength: " + String (instance.getTailLengthSeconds())); + } +}; + +static PluginInfoTest pluginInfoTest; + + +//============================================================================== +struct EditorTest : public PluginTest +{ + EditorTest() + : PluginTest ("Editor", 2) + { + } + + bool needsToRunOnMessageThread() override + { + return true; + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + if (instance.hasEditor()) + { + StopwatchTimer timer; + + { + std::unique_ptr editor (instance.createEditor()); + ut.expect (editor != nullptr, "Unable to create editor"); + + if (editor) + { + editor->addToDesktop (0); + editor->setVisible (true); + } + + ut.logMessage ("\nTime taken to open editor (cold): " + timer.getDescription()); + } + + { + timer.reset(); + std::unique_ptr editor (instance.createEditor()); + ut.expect (editor != nullptr, "Unable to create editor on second attempt"); + + if (editor) + { + editor->addToDesktop (0); + editor->setVisible (true); + } + + ut.logMessage ("Time taken to open editor (warm): " + timer.getDescription()); + } + } + } +}; + +static EditorTest editorTest; + + +//============================================================================== +struct AudioProcessingTest : public PluginTest +{ + AudioProcessingTest() + : PluginTest ("Audio processing", 3) + { + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + const double sampleRates[] = { 44100.0, 48000.0, 96000.0 }; + const int blockSizes[] = { 64, 128, 256, 512, 1024 }; + + for (auto sr : sampleRates) + { + for (auto bs : blockSizes) + { + ut.logMessage (String ("Testing with sample rate [SR] and block size [BS]") + .replace ("SR", String (sr, 0), false) + .replace ("BS", String (bs), false)); + instance.releaseResources(); + instance.prepareToPlay (sr, bs); + + const int numChannelsRequired = jmax (instance.getTotalNumInputChannels(), instance.getTotalNumOutputChannels()); + AudioBuffer ab (numChannelsRequired, bs); + MidiBuffer mb; + + for (int i = 0; i < 10; ++i) + { + mb.clear(); + fillNoise (ab); + instance.processBlock (ab, mb); + } + + ut.expectEquals (countNaNs (ab), 0, "NaNs found in buffer"); + ut.expectEquals (countInfs (ab), 0, "Infs found in buffer"); + ut.expectEquals (countSubnormals (ab), 0, "Subnormals found in buffer"); + } + } + } +}; + +static AudioProcessingTest audioProcessingTest; + + +//============================================================================== +struct PluginStateTest : public PluginTest +{ + PluginStateTest() + : PluginTest ("Plugin state", 2) + { + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + auto& parameters = instance.getParameters(); + MemoryBlock originalState; + + // Read state + instance.getStateInformation (originalState); + + // Set random parameter values + for (auto parameter : parameters) + parameter->setValue (ut.getRandom().nextFloat()); + + // Restore original state + instance.setStateInformation (originalState.getData(), (int) originalState.getSize()); + } +}; + +static PluginStateTest pluginStateTest; + + +//============================================================================== +struct AutomationTest : public PluginTest +{ + AutomationTest() + : PluginTest ("Automation", 3) + { + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + const double sampleRates[] = { 44100.0, 48000.0, 96000.0 }; + const int blockSizes[] = { 64, 128, 256, 512, 1024 }; + + for (auto sr : sampleRates) + { + for (auto bs : blockSizes) + { + const int subBlockSize = 32; + ut.logMessage (String ("Testing with sample rate [SR] and block size [BS] and sub-block size [SB]") + .replace ("SR", String (sr, 0), false) + .replace ("BS", String (bs), false) + .replace ("SB", String (subBlockSize), false)); + + instance.releaseResources(); + instance.prepareToPlay (sr, bs); + + int numSamplesDone = 0; + const int numChannelsRequired = jmax (instance.getTotalNumInputChannels(), instance.getTotalNumOutputChannels()); + AudioBuffer ab (numChannelsRequired, bs); + MidiBuffer mb; + + for (;;) + { + // Set random parameter values + { + auto& parameters = instance.getParameters(); + + for (int i = 0; i < jmin (10, parameters.size()); ++i) + { + const int paramIndex = ut.getRandom().nextInt (parameters.size()); + parameters[paramIndex]->setValue (ut.getRandom().nextFloat()); + } + } + + // Create a sub-buffer and process + const int numSamplesThisTime = jmin (subBlockSize, bs - numSamplesDone); + mb.clear(); + + AudioBuffer subBuffer (ab.getArrayOfWritePointers(), + ab.getNumChannels(), + numSamplesDone, + numSamplesThisTime); + fillNoise (subBuffer); + instance.processBlock (subBuffer, mb); + numSamplesDone += numSamplesThisTime; + + if (numSamplesDone >= bs) + break; + } + + ut.expectEquals (countNaNs (ab), 0, "NaNs found in buffer"); + ut.expectEquals (countInfs (ab), 0, "Infs found in buffer"); + ut.expectEquals (countSubnormals (ab), 0, "Subnormals found in buffer"); + } + } + } +}; + +static AutomationTest automationTest; + + +//============================================================================== +struct ParametersTest : public PluginTest +{ + ParametersTest() + : PluginTest ("Parameters", 2) + { + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + for (auto parameter : instance.getParameters()) + { + ut.logMessage (String ("\nTesting parameter: ") + String (parameter->getParameterIndex()) + " - " + parameter->getName (512)); + testParameterInfo (ut, *parameter); + testParameterDefaults (ut, *parameter); + } + } + +private: + void testParameterInfo (UnitTest& ut, AudioProcessorParameter& parameter) + { + const int index = parameter.getParameterIndex(); + const String paramName = parameter.getName (512); + + const float defaultValue = parameter.getDefaultValue(); + const String label = parameter.getLabel(); + const int numSteps = parameter.getNumSteps(); + const bool isDiscrete = parameter.isDiscrete(); + const bool isBoolean = parameter.isBoolean(); + const StringArray allValueStrings = parameter.getAllValueStrings(); + + const bool isOrientationInverted = parameter.isOrientationInverted(); + const bool isAutomatable = parameter.isAutomatable(); + const bool isMetaParameter = parameter.isMetaParameter(); + const auto category = parameter.getCategory(); + + #define LOGP(x) JUCE_STRINGIFY(x) + " - " + String (x) + ", " + #define LOGP_B(x) JUCE_STRINGIFY(x) + " - " + String (static_cast (x)) + ", " + ut.logMessage (String ("Parameter info: ") + + LOGP(index) + + LOGP(paramName) + + LOGP(defaultValue) + + LOGP(label) + + LOGP(numSteps) + + LOGP_B(isDiscrete) + + LOGP_B(isBoolean) + + LOGP_B(isOrientationInverted) + + LOGP_B(isAutomatable) + + LOGP_B(isMetaParameter) + + LOGP_B(category) + + "all value strings - " + allValueStrings.joinIntoString ("|")); + } + + void testParameterDefaults (UnitTest& ut, AudioProcessorParameter& parameter) + { + ut.logMessage ("\nTesting parameter defaults..."); + const float value = parameter.getValue(); + const String text = parameter.getText (value, 1024); + const float valueForText = parameter.getValueForText (text); + const String currentValueAsText = parameter.getCurrentValueAsText(); + ignoreUnused (value, text, valueForText, currentValueAsText); + } +}; + +static ParametersTest parametersTest; + + +//============================================================================== +struct BackgroundThreadStateTest : public PluginTest +{ + BackgroundThreadStateTest() + : PluginTest ("Background thread state", 7) + { + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + WaitableEvent waiter; + std::unique_ptr editor; + MessageManager::getInstance()->callAsync ([&] + { + editor.reset (instance.createEditor()); + ut.expect (editor != nullptr, "Unable to create editor"); + + if (editor) + { + editor->addToDesktop (0); + editor->setVisible (true); + } + + waiter.signal(); + }); + + auto& parameters = instance.getParameters(); + MemoryBlock originalState; + + // Read state + instance.getStateInformation (originalState); + + // Set random parameter values + for (auto parameter : parameters) + parameter->setValue (ut.getRandom().nextFloat()); + + // Restore original state + instance.setStateInformation (originalState.getData(), (int) originalState.getSize()); + + waiter.wait(); + Thread::sleep (2000); + + MessageManager::getInstance()->callAsync ([&] + { + editor.reset(); + waiter.signal(); + }); + waiter.wait(); + } +}; + +static BackgroundThreadStateTest backgroundThreadStateTest; diff --git a/Source/tests/ParameterFuzzTests.cpp b/Source/tests/ParameterFuzzTests.cpp new file mode 100644 index 0000000..2b17b78 --- /dev/null +++ b/Source/tests/ParameterFuzzTests.cpp @@ -0,0 +1,52 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "../PluginTests.h" +#include "../TestUtilities.h" + +//============================================================================== +struct FuzzParametersTest : public PluginTest +{ + FuzzParametersTest() + : PluginTest ("Fuzz parameters", 6) + { + } + + void runTest (UnitTest& ut, AudioPluginInstance& instance) override + { + for (auto parameter : instance.getParameters()) + fuzzTestParameter (ut, *parameter); + } + +private: + void fuzzTestParameter (UnitTest& ut, AudioProcessorParameter& parameter) + { + ut.logMessage (String ("Fuzz testing parameter: ") + String (parameter.getParameterIndex()) + " - " + parameter.getName (512)); + + for (int i = 0; i < 5; ++i) + { + const float value = ut.getRandom().nextFloat(); + + parameter.setValue (value); + const float v = parameter.getValue(); + + const String currentValueAsText = parameter.getCurrentValueAsText(); + const String text = parameter.getText (value, 1024); + const float valueForText = parameter.getValueForText (text); + ignoreUnused (v, text, valueForText, currentValueAsText); + } + } +}; + +static FuzzParametersTest fuzzParametersTest; diff --git a/install/linux_build b/install/linux_build new file mode 100755 index 0000000..928c22f --- /dev/null +++ b/install/linux_build @@ -0,0 +1,51 @@ +#!/bin/sh -e + +ROOT=$(cd "$(dirname "$0")/.."; pwd) + +if [ -z "$ROOT" ]; then + echo "ERROR: Unknown workspase" + exit 1 +fi + +PROJECT_NAME=pluginval +DEPLOYMENT_DIR="$ROOT/bin/linux" + +BINARY_NAME="$PROJECT_NAME" +APP_NAME=$BINARY_NAME +APP_FILE=$ROOT/Builds/LinuxMakefile/build/$APP_NAME + +echo "\n==========================================" +echo "\nRoot dir: $ROOT" + +#============================================================ +# Build Projucer and generate projects +#============================================================ +echo "Building Projucer and creating projects" +PROJUCER_ROOT=$ROOT/modules/juce/extras/Projucer/Builds/LinuxMakefile +PROJUCER_EXE=$PROJUCER_ROOT/build/Projucer +cd "$PROJUCER_ROOT" +#make clean +make CONFIG=Release -j4 + +# Resave project +"$PROJUCER_EXE" --resave "$ROOT/$PROJECT_NAME.jucer" + + +#============================================================ +# Build Xcode projects +#============================================================ +cd "$ROOT/Builds/LinuxMakefile" +rm -rf $ROOT/Builds/LinuxMakefile/build/ +#make clean +make CONFIG=Release -j4 + + +#============================================================ +# Copy to deployment directory +#============================================================ +cd "$ROOT" +rm -rf "$DEPLOYMENT_DIR" +mkdir -p "$DEPLOYMENT_DIR" + +echo "\nDeploying to: " $DEPLOYMENT_DIR +mv -fv "$APP_FILE" "$DEPLOYMENT_DIR/$APP_NAME" diff --git a/install/mac_build b/install/mac_build new file mode 100755 index 0000000..3003984 --- /dev/null +++ b/install/mac_build @@ -0,0 +1,50 @@ +#!/bin/sh -e + +ROOT=$(cd "$(dirname "$0")/.."; pwd) + +if [ -z "$ROOT" ]; then + echo "ERROR: Unknown workspase" + exit 1 +fi + +PROJECT_NAME=pluginval +DEPLOYMENT_DIR="$ROOT/bin/mac" + +BINARY_NAME="$PROJECT_NAME" +APP_NAME=$BINARY_NAME".app" +APP_FILE=$ROOT/Builds/MacOSX/build/Release/$APP_NAME + +echo "\n==========================================" +echo "\nRoot dir: $ROOT" + +#============================================================ +# Build Projucer and generate projects +#============================================================ +echo "Building Projucer and creating projects" +PROJUCER_ROOT=$ROOT/modules/juce/extras/Projucer/Builds/MacOSX +PROJUCER_EXE=$PROJUCER_ROOT/build/Release/Projucer.app/Contents/MacOS/Projucer +cd "$PROJUCER_ROOT" +xcodebuild -configuration Release + +# Resave project +"$PROJUCER_EXE" --resave "$ROOT/$PROJECT_NAME.jucer" + + +#============================================================ +# Build Xcode projects +#============================================================ +cd $ROOT/Builds/MacOSX +rm -rf $ROOT/Builds/MacOSX/build/$PROJECT_NAME.build +xcodebuild -configuration Release clean +xcodebuild -configuration Release GCC_TREAT_WARNINGS_AS_ERRORS=YES + + +#============================================================ +# Copy to deployment directory +#============================================================ +cd $ROOT +rm -rf $DEPLOYMENT_DIR +mkdir -p "$DEPLOYMENT_DIR" + +echo "\nDeploying to: " $DEPLOYMENT_DIR +mv -fv "$APP_FILE" "$DEPLOYMENT_DIR/$APP_NAME" diff --git a/install/windows_build.bat b/install/windows_build.bat new file mode 100755 index 0000000..9e90039 --- /dev/null +++ b/install/windows_build.bat @@ -0,0 +1,55 @@ +@echo OFF + +cd "%~dp0%.." +set ROOT=%cd% + +echo ROOT: "%ROOT%" +if not exist "%ROOT%" exit 1 + +set PROJECT_NAME=pluginval +set DEPLOYMENT_DIR=%ROOT%/bin/windows + +set BINARY_NAME=%PROJECT_NAME%.exe +set APP_NAME=%BINARY_NAME% +set APP_FILE=%ROOT%\Builds\VisualStudio2017\x64\Release\App\%APP_NAME% + + +:: First clear the bin +echo "==========================================================" +echo "Removing old files %ROOT%/bin/windows" +rd /S /Q "%ROOT%\bin\windows" + +::============================================================ +:: Build Projucer and generate projects +::============================================================ +echo "Building Projucer and creating projects" +set PROJUCER_ROOT=%ROOT%/modules/juce/extras/Projucer/Builds/VisualStudio2017 +set PROJUCER_EXE=%PROJUCER_ROOT%/x64/Release/App/Projucer.exe + +cd "%PROJUCER_ROOT%" +"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe" Projucer.sln /p:VisualStudioVersion=15.0 /m /p:Configuration=Release /p:Platform=x64 /p:PreferredToolArchitecture=x64 +if not exist "%PROJUCER_EXE%" exit 1 + +:: Resave Waveform project +"%PROJUCER_EXE%" --resave "%ROOT%/%PROJECT_NAME%.jucer" + + +::============================================================ +:: Build Xcode projects +::============================================================ +echo "==========================================================" +echo "Building products" +cd "%ROOT%/Builds/VisualStudio2017" +rd /S /Q "x64/Release" +"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe" %PROJECT_NAME%.sln /p:VisualStudioVersion=15.0 /m /t:Build /p:Configuration=Release /p:Platform=x64 /p:PreferredToolArchitecture=x64 /p:TreatWarningsAsErrors=true + + +::============================================================ +:: Copy to deployment directory +::============================================================ +cd $ROOT +rd /S /Q "%DEPLOYMENT_DIR%" +mkdir "%DEPLOYMENT_DIR%" + +echo "\nDeploying to: " %DEPLOYMENT_DIR% +move "%APP_FILE%" "%DEPLOYMENT_DIR%" diff --git a/modules/juce b/modules/juce new file mode 160000 index 0000000..d4762f1 --- /dev/null +++ b/modules/juce @@ -0,0 +1 @@ +Subproject commit d4762f1d9a8b4cd39d669997dd09df3f215a7fc6 diff --git a/pluginval.jucer b/pluginval.jucer new file mode 100644 index 0000000..1e1dfb0 --- /dev/null +++ b/pluginval.jucer @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +