From 6536bdda1c4c07347db006e033dd43377ff95996 Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Thu, 14 Nov 2024 14:12:12 +1100 Subject: [PATCH 01/10] issue #909: bump version for CI --- version.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.php b/version.php index bedee946..cbc0819e 100644 --- a/version.php +++ b/version.php @@ -28,7 +28,7 @@ $plugin->version = 2024101000; $plugin->release = 2024101000; $plugin->requires = 2022112800; // Our lowest supported Moodle (3.3.0). -$plugin->supported = [400, 402]; +$plugin->supported = [400, 405]; // TODO $plugin->incompatible = ; // Available as of Moodle 3.9.0 or later. $plugin->component = 'tool_dataflows'; $plugin->maturity = MATURITY_ALPHA; From 99210992d157ed1c619582417e90c65e6e34355b Mon Sep 17 00:00:00 2001 From: Daniel Neis Araujo Date: Mon, 21 Oct 2024 17:26:17 -0300 Subject: [PATCH 02/10] make the eventlist compatible with Moodle 4.3 --- classes/local/step/trigger_event.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/classes/local/step/trigger_event.php b/classes/local/step/trigger_event.php index 6c47161c..a312a48e 100644 --- a/classes/local/step/trigger_event.php +++ b/classes/local/step/trigger_event.php @@ -133,11 +133,13 @@ private function get_events_list() { $eventlist = \tool_monitor\eventlist::get_all_eventlist(); $pluginlist = \tool_monitor\eventlist::get_plugin_list($eventlist); $plugineventlist = []; - foreach ($pluginlist as $plugin => $pluginname) { - foreach ($eventlist[$plugin] as $event => $eventname) { - // Filter out events which cannot be triggered for some reason. - if (!$event::is_deprecated()) { - $plugineventlist[$event] = "{$pluginname}: {$eventname}"; + foreach ($pluginlist as $plugintype => $plugins) { + foreach ($plugins as $plugin => $pluginname) { + foreach ($eventlist[$plugin] as $event => $eventname) { + // Filter out events which cannot be triggered for some reason. + if (!$event::is_deprecated()) { + $plugineventlist[$event] = "{$pluginname}: {$eventname}"; + } } } } From 496d3057169033d9796856e7a4522b49a0b4f74e Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Thu, 14 Nov 2024 15:27:17 +1100 Subject: [PATCH 03/10] issue #909: migrate after_config to hooks --- classes/local/hooks/after_config.php | 44 ++++++++++++++++++++++++++++ db/hooks.php | 33 +++++++++++++++++++++ lib.php | 9 ------ version.php | 2 +- 4 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 classes/local/hooks/after_config.php create mode 100644 db/hooks.php diff --git a/classes/local/hooks/after_config.php b/classes/local/hooks/after_config.php new file mode 100644 index 00000000..c267b0ad --- /dev/null +++ b/classes/local/hooks/after_config.php @@ -0,0 +1,44 @@ +. + +namespace tool_dataflows\local\hooks; + +/** + * After config hook + * + * @package tool_dataflows + * @copyright 2024 Catalyst IT Australia + * @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class after_config { + + /** + * Hook to be run after initial site config. + * + * Triggered as soon as practical on every moodle bootstrap after config has + * been loaded. The $USER object is available at this point too. + * + * This currently ensures all vendor libraries are loaded. + * + * @param \core\hook\after_config $hook + * return void + */ + public static function callback(\core\hook\after_config $hook): void { + global $CFG; + + require_once ($CFG->dirroot. '/admin/tool/dataflows/lib.php'); + } +} diff --git a/db/hooks.php b/db/hooks.php new file mode 100644 index 00000000..772f4c10 --- /dev/null +++ b/db/hooks.php @@ -0,0 +1,33 @@ +. + +/** + * Hooks callbacks. + * + * @package tool_dataflows + * @copyright 2024 Catalyst IT Australia + * @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +$callbacks = [ + [ + 'hook' => \core\hook\after_config::class, + 'callback' => '\tool_dataflows\local\hooks\after_config::callback', + 'priority' => 0, + ], +]; diff --git a/lib.php b/lib.php index f5bf130a..0198f912 100644 --- a/lib.php +++ b/lib.php @@ -30,15 +30,6 @@ use tool_dataflows\local\formats\encoders; use tool_dataflows\local\step; -/** - * Triggered as soon as practical on every moodle bootstrap after config has - * been loaded. The $USER object is available at this point too. - * - * NOTE: DO NOT REMOVE. This currently ensures all vendor libraries are loaded. - */ -function tool_dataflows_after_config() { -} - /** * Returns a list of step types available for this plugin. * diff --git a/version.php b/version.php index cbc0819e..9ed18dc5 100644 --- a/version.php +++ b/version.php @@ -28,7 +28,7 @@ $plugin->version = 2024101000; $plugin->release = 2024101000; $plugin->requires = 2022112800; // Our lowest supported Moodle (3.3.0). -$plugin->supported = [400, 405]; +$plugin->supported = [405, 405]; // TODO $plugin->incompatible = ; // Available as of Moodle 3.9.0 or later. $plugin->component = 'tool_dataflows'; $plugin->maturity = MATURITY_ALPHA; From 11e9fa7275486e5078d260bca8cfb6183812ae30 Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Thu, 14 Nov 2024 15:27:45 +1100 Subject: [PATCH 04/10] issue #909: use assertObjectHasProperty --- tests/tool_dataflows_connector_curl_test.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/tool_dataflows_connector_curl_test.php b/tests/tool_dataflows_connector_curl_test.php index 8d3df5ed..27e5c34a 100644 --- a/tests/tool_dataflows_connector_curl_test.php +++ b/tests/tool_dataflows_connector_curl_test.php @@ -85,9 +85,9 @@ public function test_execute() { $this->assertEquals('3d188fbf-d0b7-4d4e-ae4d-4b5548df824e', $result->uuid); $this->assertEquals(200, $vars->httpcode); - $this->assertObjectHasAttribute('connecttime', $vars); - $this->assertObjectHasAttribute('totaltime', $vars); - $this->assertObjectHasAttribute('sizeupload', $vars); + $this->assertObjectHasProperty('connecttime', $vars); + $this->assertObjectHasProperty('totaltime', $vars); + $this->assertObjectHasProperty('sizeupload', $vars); $testurl = $this->get_mock_url('/test_post.php'); @@ -115,9 +115,9 @@ public function test_execute() { $vars = $engine->get_variables_root()->get('steps.connector.vars'); $this->assertEquals(200, $vars->httpcode); - $this->assertObjectHasAttribute('connecttime', $vars); - $this->assertObjectHasAttribute('totaltime', $vars); - $this->assertObjectHasAttribute('sizeupload', $vars); + $this->assertObjectHasProperty('connecttime', $vars); + $this->assertObjectHasProperty('totaltime', $vars); + $this->assertObjectHasProperty('sizeupload', $vars); // Tests put method. $stepdef->config = Yaml::dump([ @@ -145,9 +145,9 @@ public function test_execute() { // PUT has no response body so it shouldn't be checked. $this->assertEquals(200, $vars->httpcode); - $this->assertObjectHasAttribute('connecttime', $vars); - $this->assertObjectHasAttribute('totaltime', $vars); - $this->assertObjectHasAttribute('sizeupload', $vars); + $this->assertObjectHasProperty('connecttime', $vars); + $this->assertObjectHasProperty('totaltime', $vars); + $this->assertObjectHasProperty('sizeupload', $vars); $expectedbash = "curl -s -X PUT {$testurl} --max-time 30 --data-raw 'data=moodletest'"; $this->assertEquals($expectedbash, $variables->dbgcommand); From 706d8b33c68ea1725fb098550f0e1166eee76be4 Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Thu, 14 Nov 2024 15:35:15 +1100 Subject: [PATCH 05/10] issue #909: fix CI complain --- classes/local/hooks/after_config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/local/hooks/after_config.php b/classes/local/hooks/after_config.php index c267b0ad..7448acbf 100644 --- a/classes/local/hooks/after_config.php +++ b/classes/local/hooks/after_config.php @@ -39,6 +39,6 @@ class after_config { public static function callback(\core\hook\after_config $hook): void { global $CFG; - require_once ($CFG->dirroot. '/admin/tool/dataflows/lib.php'); + require_once($CFG->dirroot. '/admin/tool/dataflows/lib.php'); } } From 261108b770156f472ed40b2d5baaacfd624f6ce4 Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Thu, 14 Nov 2024 16:21:20 +1100 Subject: [PATCH 06/10] issue #909: fix PHPUnit warnings --- tests/application_trait.php | 15 ++------------- tests/tool_dataflows_variables_test.php | 2 +- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/tests/application_trait.php b/tests/application_trait.php index 47f1d6d3..2aae6a83 100644 --- a/tests/application_trait.php +++ b/tests/application_trait.php @@ -104,7 +104,7 @@ public function compatible_assertMatchesRegularExpression(string $pattern, strin if (method_exists($this, 'assertMatchesRegularExpression')) { $this->assertMatchesRegularExpression($pattern, $string, $message); } else { - $this->assertRegExp($pattern, $string, $message); + $this->assertDoesNotMatchRegularExpression($pattern, $string, $message); } } @@ -119,18 +119,7 @@ public function compatible_assertDoesNotMatchRegularExpression(string $pattern, if (method_exists($this, 'assertDoesNotMatchRegularExpression')) { $this->assertDoesNotMatchRegularExpression($pattern, $string, $message); } else { - $this->assertNotRegExp($pattern, $string, $message); - } - } - - /** - * Asserts that an error was expected - */ - public function compatible_expectError(): void { - if (method_exists($this, 'expectError')) { - $this->expectError(); - } else { - $this->expectException(\PHPUnit\Framework\Error\Error::class); + $this->assertDoesNotMatchRegularExpression($pattern, $string, $message); } } // @codingStandardsIgnoreEnd diff --git a/tests/tool_dataflows_variables_test.php b/tests/tool_dataflows_variables_test.php index b630adbb..49d75e5b 100644 --- a/tests/tool_dataflows_variables_test.php +++ b/tests/tool_dataflows_variables_test.php @@ -140,7 +140,7 @@ public function test_variable_parsing_involving_indirect_self_recursion() { // Expecting it to throw an exception during execution, particularly // when preparing the SQL and finding out it contains an unparsed // expression. - $this->compatible_expectError(); + $this->expectException(\TypeError::class); try { $engine->execute(); } finally { From db53bee4f723ddb675d2b3c3ceb465ed45ab2022 Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Fri, 15 Nov 2024 14:58:57 +1100 Subject: [PATCH 07/10] issue #909: fix dynamic propeties --- .../execution/iterators/dataflow_iterator.php | 10 ++++++ classes/local/step/flow_cap.php | 6 ++++ classes/step.php | 18 ++++++---- tests/tool_dataflows_event_processor_test.php | 31 ++++++++++++++++ tests/tool_dataflows_flow_sql_test.php | 35 ++++++++++++++++++ tests/tool_dataflows_json_reader_test.php | 36 +++++++++++++++++++ tests/tool_dataflows_reader_csv_test.php | 11 ++++++ 7 files changed, 141 insertions(+), 6 deletions(-) diff --git a/classes/local/execution/iterators/dataflow_iterator.php b/classes/local/execution/iterators/dataflow_iterator.php index a6d8386b..daded4a2 100644 --- a/classes/local/execution/iterators/dataflow_iterator.php +++ b/classes/local/execution/iterators/dataflow_iterator.php @@ -48,6 +48,16 @@ class dataflow_iterator implements iterator { /** @var int */ protected $iterationcount = 0; + /** + * @var \tool_dataflows\local\variables\var_step|null + */ + protected $stepvars = null; + + /** + * @var null + */ + protected $pulled = null; + /** * Create an instance of this class. * diff --git a/classes/local/step/flow_cap.php b/classes/local/step/flow_cap.php index 479f211f..8e2249f7 100644 --- a/classes/local/step/flow_cap.php +++ b/classes/local/step/flow_cap.php @@ -33,6 +33,12 @@ */ final class flow_cap extends flow_step { + /** + * Upstream interator. + * @var + */ + protected $upstream; + /** * Generates an engine step for this type. * diff --git a/classes/step.php b/classes/step.php index 78b6e4f4..1c8288fd 100644 --- a/classes/step.php +++ b/classes/step.php @@ -57,6 +57,12 @@ class step extends persistent { /** @var array array for lazy loading step dependants */ private $dependents = null; + /** @var dataflow Dataflow */ + protected $dataflow = null; + + /** @var base_step step type */ + protected $steptype; + /** * Return the definition of the properties of this model. * @@ -495,7 +501,7 @@ public function upsert() { // Update the local dependencies to the database. $this->update_depends_on(); - $this->steptype->on_save(); + $this->get_steptype()->on_save(); $this->get_dataflow()->on_steps_save(); return $this; @@ -628,7 +634,7 @@ protected function before_delete() { $DB->delete_records('tool_dataflows_step_depends', ['stepid' => $this->id]); $DB->delete_records('tool_dataflows_step_depends', ['dependson' => $this->id]); - $steptype = $this->steptype; + $steptype = $this->get_steptype(); if (isset($steptype)) { $steptype->on_delete(); } @@ -728,7 +734,7 @@ protected function validate_config() { return new \lang_string('invalidyaml', 'tool_dataflows'); } - $validation = $this->steptype->validate_config($yaml); + $validation = $this->get_steptype()->validate_config($yaml); if ($validation !== true) { // NOTE: This will only return the first error as the persistent // class expects the return value to be an instance of lang_string. @@ -748,7 +754,7 @@ protected function validate_config() { */ protected function validate_link_count(int $count, string $inputoutput, string $flowconnector) { $fn = "get_number_of_{$inputoutput}_{$flowconnector}s"; - $steptype = $this->steptype; + $steptype = $this->get_steptype(); [$min, $max] = $steptype->$fn(); if ($inputoutput === 'output') { $min = max($min, count($steptype->get_output_labels())); @@ -778,7 +784,7 @@ protected function validate_links(array $deps, string $inputoutput) { $count = count($deps); $errors = []; - $steptype = $this->steptype; + $steptype = $this->get_steptype(); if ($count != 0) { $dep = array_shift($deps); @@ -877,7 +883,7 @@ public function has_side_effect(): bool { if ($typevalidation !== true) { return false; } - return $this->steptype->has_side_effect(); + return $this->get_steptype()->has_side_effect(); } /** diff --git a/tests/tool_dataflows_event_processor_test.php b/tests/tool_dataflows_event_processor_test.php index 6633a349..bf0ac016 100644 --- a/tests/tool_dataflows_event_processor_test.php +++ b/tests/tool_dataflows_event_processor_test.php @@ -35,6 +35,37 @@ */ class tool_dataflows_event_processor_test extends \advanced_testcase { + /** + * Test workflow. + * @var \tool_dataflows\dataflow + */ + protected $dataflow; + + /** + * Test writer. + * @var + */ + protected $writer; + + /** + * Test reader. + * @var + */ + protected $reader; + + /** + * Test course. + * + * @var \stdClass + */ + protected $course; + + /** + * Test output path. + * @var + */ + protected $outputpath; + /** * Set up before each test */ diff --git a/tests/tool_dataflows_flow_sql_test.php b/tests/tool_dataflows_flow_sql_test.php index 9a21bee1..7765708e 100644 --- a/tests/tool_dataflows_flow_sql_test.php +++ b/tests/tool_dataflows_flow_sql_test.php @@ -34,6 +34,41 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class tool_dataflows_flow_sql_test extends \advanced_testcase { + /** + * Test workflow. + * @var \tool_dataflows\dataflow + */ + protected $dataflow; + + /** + * Test writer. + * @var + */ + protected $writer; + + /** + * Test reader. + * @var + */ + protected $reader; + + /** + * Test input path. + * @var false|string + */ + protected $inputpath; + + /** + * Test output path. + * @var + */ + protected $outputpath; + + /** + * Test flow step. + * @var \tool_dataflows\step + */ + protected $flow; /** * Set up before each test diff --git a/tests/tool_dataflows_json_reader_test.php b/tests/tool_dataflows_json_reader_test.php index d6ff8f3b..d834cd0b 100644 --- a/tests/tool_dataflows_json_reader_test.php +++ b/tests/tool_dataflows_json_reader_test.php @@ -36,6 +36,42 @@ */ class tool_dataflows_json_reader_test extends \advanced_testcase { + /** + * Test workflow. + * @var \tool_dataflows\dataflow + */ + protected $dataflow; + + /** + * Test writer. + * @var + */ + protected $writer; + + /** + * Test reader. + * @var + */ + protected $reader; + + /** + * Test input path. + * @var false|string + */ + protected $inputpath; + + /** + * Test output path. + * @var + */ + protected $outputpath; + + /** + * Test users. + * @var + */ + protected $users; + /** * Set up before each test */ diff --git a/tests/tool_dataflows_reader_csv_test.php b/tests/tool_dataflows_reader_csv_test.php index 7d2791f9..55a2d8eb 100644 --- a/tests/tool_dataflows_reader_csv_test.php +++ b/tests/tool_dataflows_reader_csv_test.php @@ -35,6 +35,17 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class tool_dataflows_reader_csv_test extends \advanced_testcase { + /** + * Test writer. + * @var + */ + protected $writer; + + /** + * Test reader. + * @var + */ + protected $reader; /** * Set up before each test From f9ea1a1547e1337c86b32b83a32dd50157c18bdc Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Fri, 15 Nov 2024 16:12:28 +1100 Subject: [PATCH 08/10] issue #909: update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a711835d..c09df4f4 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ Dataflows is a generic workflow and processing engine which can be configured to | Moodle version | Branch | PHP | |----------------|------------------|-----------| -| Moodle 4.1+ | MOODLE_41_STABLE | 7.4 | 8.0 | +| Moodle 4.1-4.2 | MOODLE_41_STABLE | 7.4 | +| Moodle 4.5 | MOODLE_45_STABLE | 8.1 - 8.3 | | Totara 10+ | MOODLE_35_STABLE | 7.1 - 7.4 | Note: Moodle 402 is supported with PHP 8.0 maximum at the moment From 646dbf785aed5e445a5a0a4a7aaa77e933d28f83 Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Fri, 15 Nov 2024 16:26:32 +1100 Subject: [PATCH 09/10] issue #909: fix branch names --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c09df4f4..a4c8e3b1 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,11 @@ Dataflows is a generic workflow and processing engine which can be configured to ## Branches -| Moodle version | Branch | PHP | -|----------------|------------------|-----------| -| Moodle 4.1-4.2 | MOODLE_41_STABLE | 7.4 | -| Moodle 4.5 | MOODLE_45_STABLE | 8.1 - 8.3 | -| Totara 10+ | MOODLE_35_STABLE | 7.1 - 7.4 | +| Moodle version | Branch | PHP | +|----------------|-------------------|-----------| +| Moodle 4.1-4.2 | MOODLE_401_STABLE | 7.4 | +| Moodle 4.5 | MOODLE_405_STABLE | 8.1 - 8.3 | +| Totara 10+ | MOODLE_35_STABLE | 7.1 - 7.4 | Note: Moodle 402 is supported with PHP 8.0 maximum at the moment From 91d9fc22ad66ff33e1a0659f3c07f0866ec353ae Mon Sep 17 00:00:00 2001 From: Dmitrii Metelkin Date: Fri, 15 Nov 2024 16:53:59 +1100 Subject: [PATCH 10/10] issue #909: bump version --- version.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/version.php b/version.php index 9ed18dc5..b8c890e1 100644 --- a/version.php +++ b/version.php @@ -25,11 +25,10 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2024101000; -$plugin->release = 2024101000; -$plugin->requires = 2022112800; // Our lowest supported Moodle (3.3.0). +$plugin->version = 2024111500; +$plugin->release = 2024111500; +$plugin->requires = 2024100700; // Our lowest supported Moodle (4.5.0). $plugin->supported = [405, 405]; -// TODO $plugin->incompatible = ; // Available as of Moodle 3.9.0 or later. $plugin->component = 'tool_dataflows'; $plugin->maturity = MATURITY_ALPHA; $plugin->dependencies = [];