#37 - Upgrade to version 3 spec

lib/extract_test_metadata.rb

@@ -19,7 +19,8 @@ def call
         test: test_identifier,
         name: test_name,
         test_code:,
-        index:
+        index:,
+        task_id:
       }
     end
 
@@ -52,9 +53,6 @@ def test_name
     # - Plus an bits we've chosen to add back as the test_code part
     memoize
     def test_code
-      # Get the lines excluding the first (def) and last (end)
-      body_line_numbers = ((test_node.first_line + 1)..(test_node.last_line - 1))
-
       # Map through those lines, skipping any that were
       # part of assertions
       test_code = body_line_numbers.map do |idx|
@@ -63,13 +61,25 @@ def test_code
         c = code_for_line(idx)
 
         # Only return if it's not a skip comment
-        c.start_with?(/\s*#\s*skip/) ? nil : c
+        c.start_with?(/\s*#\s*skip/, /\s*###\s*task_id/) ? nil : c
       end.compact.join("").rstrip
 
       # Align everything to the left as the final step
       clean_leading_whitespace(test_code)
     end
 
+    memoize
+    def task_id
+      body_line_numbers.map do |idx|
+        next if ignore_line_numbers.include?(idx)
+
+        c = code_for_line(idx)
+        # Find a line started with `### task_id` and get the number
+        ch_task_id = /\s*###\s*task_id/
+        c.chomp.strip.delete('### task_id:').to_i if c.start_with?(ch_task_id)
+      end.compact.first
+    end
+
     # Remove the minimum amount of leading whitespace
     # from all lines
     def clean_leading_whitespace(multiline)
@@ -82,5 +92,11 @@ def clean_leading_whitespace(multiline)
     def code_for_line(one_indexed_idx)
       filelines[one_indexed_idx - 1]
     end
+
+    memoize
+    def body_line_numbers
+      # Get the lines excluding the first (def) and last (end)
+      ((test_node.first_line + 1)..(test_node.last_line - 1))
+    end
   end
 end

lib/minitest_ext/exercism_reporter.rb

@@ -95,6 +95,7 @@ def to_h
           hash[:status]    = status
           hash[:output]    = output  if attach_output?
           hash[:message]   = message if attach_message?
+          hash[:task_id]   = metadata[:task_id]
         end
       end
 

lib/write_report.rb

@@ -18,7 +18,7 @@ def call
 
     def json
       {
-        version: 2,
+        version: 3,
         status:,
         message:,
         tests:

test/attacks_test.rb

@@ -4,7 +4,7 @@ class AttacksTest < Minitest::Test
   def test_large_output_is_truncated
     assert_fixture(
       :attack_large_output, {
-        version: 2,
+        version: 3,
         status: :fail,
         message: nil,
         tests: [
@@ -13,7 +13,8 @@ def test_large_output_is_truncated
             test_code: 'assert_equal "One for you, one for me.", TwoFer.two_fer',
             status: :fail,
             output: %(#{Array.new(500) { 'a' }.join}\n\n...Output was truncated. Please limit to 500 chars...),
-            message: "Expected: \"One for you, one for me.\"\n  Actual: false"
+            message: "Expected: \"One for you, one for me.\"\n  Actual: false",
+            task_id: nil
           }
         ]
       }

test/extract_metadata_test.rb

@@ -7,7 +7,8 @@ def test_assert_equal
       test: "test_assert_equal_works_properly",
       name: "Assert equal works properly",
       test_code: %(some_result = TwoFer.two_fer\nassert_equal "One for you, one for me.", some_result),
-      index: 0
+      index: 0,
+      task_id: 123
     }]
 
     actual = TestRunner::ExtractMetadata.(File.expand_path("fixtures/metadata/assert_equal.rb", __dir__))
@@ -19,7 +20,8 @@ def test_no_skips
       test: "test_skip_works_properly",
       name: "Skip works properly",
       test_code: "something = \"Something\"\nassert something.present?",
-      index: 0
+      index: 0,
+      task_id: 456
     }]
 
     actual = TestRunner::ExtractMetadata.(File.expand_path("fixtures/metadata/skip.rb", __dir__))
@@ -31,7 +33,8 @@ def test_no_skip_comments
       test: "test_skip_works_properly",
       name: "Skip works properly",
       test_code: "something = \"Something\"\nassert something.present?",
-      index: 0
+      index: 0,
+      task_id: 789
     }]
 
     actual = TestRunner::ExtractMetadata.(File.expand_path("fixtures/metadata/skip_comment.rb", __dir__))
@@ -44,25 +47,29 @@ def test_extracting_indices
         test: "test_zebra",
         name: "Zebra",
         test_code: %(some_result = TwoFer.two_fer("zebra")\nassert_equal "One for you, one for zebra.", some_result),
-        index: 0
+        index: 0,
+        task_id: 789
       },
       {
         test: "test_anaconda",
         name: "Anaconda",
         test_code: %(some_result = TwoFer.two_fer("anaconda")\nassert_equal "One for you, one for anaconda.", some_result),
-        index: 1
+        index: 1,
+        task_id: nil
       },
       {
         test: "test_gorilla",
         name: "Gorilla",
         test_code: %(some_result = TwoFer.two_fer("gorilla")\nassert_equal "One for you, one for gorilla.", some_result),
-        index: 2
+        index: 2,
+        task_id: nil
       },
       {
         test: "test_boa",
         name: "Boa",
         test_code: %(some_result = TwoFer.two_fer("boa")\nassert_equal "One for you, one for boa.", some_result),
-        index: 3
+        index: 3,
+        task_id: nil
       }
     ]
 

test/fixtures/metadata/assert_equal.rb

@@ -1,5 +1,6 @@
 class SomeTest < Minitest::Test
   def test_assert_equal_works_properly
+    ### task_id: 123
     some_result = TwoFer.two_fer
     assert_equal "One for you, one for me.", some_result
   end

test/fixtures/metadata/indices.rb

@@ -1,5 +1,6 @@
 class SomeTest < Minitest::Test
   def test_zebra
+    ### task_id: 789
     some_result = TwoFer.two_fer("zebra")
     assert_equal "One for you, one for zebra.", some_result
   end

test/fixtures/metadata/skip.rb

@@ -1,5 +1,6 @@
 class SomeTest < Minitest::Test
   def test_skip_works_properly
+    ### task_id: 456
     skip
     something = "Something"
     assert something.present?

test/fixtures/metadata/skip_comment.rb

@@ -1,5 +1,6 @@
 class SomeTest < Minitest::Test
   def test_skip_works_properly
+    ### task_id: 789
     #skip
     # skip
     something = "Something"

test/test_runner_test.rb

@@ -5,24 +5,27 @@ def test_pass
     assert_fixture(
       :pass,
       {
-        version: 2,
+        version: 3,
         status: :pass,
         message: nil,
         tests: [
           {
             name: "No name given",
             status: :pass,
-            test_code: %(assert_equal "One for you, one for me.", TwoFer.two_fer)
+            test_code: %(assert_equal "One for you, one for me.", TwoFer.two_fer),
+            task_id: nil
           },
           {
             name: 'A name given',
             test_code: 'assert_equal "One for Alice, one for me.", TwoFer.two_fer("Alice")',
-            status: :pass
+            status: :pass,
+            task_id: nil
           },
           {
             name: "Another name given",
             status: :pass,
-            test_code: 'assert_equal "One for Bob, one for me.", TwoFer.two_fer("Bob")'
+            test_code: 'assert_equal "One for Bob, one for me.", TwoFer.two_fer("Bob")',
+            task_id: nil
           }
         ]
       }
@@ -33,19 +36,21 @@ def test_pass_ruby_3
     assert_fixture(
       :pass_ruby_3_syntax,
       {
-        version: 2,
+        version: 3,
         status: :pass,
         message: nil,
         tests: [
           {
             name: "Rightward assign",
             status: :pass,
-            test_code: %(assert_equal Ruby3Syntax.rightward_assign, 'is fun')
+            test_code: %(assert_equal Ruby3Syntax.rightward_assign, 'is fun'),
+            task_id: nil
           },
           {
             name: "Endless method def",
             status: :pass,
-            test_code: %(assert_equal Ruby3Syntax.endless_methods, 'are fun')
+            test_code: %(assert_equal Ruby3Syntax.endless_methods, 'are fun'),
+            task_id: nil
           }
         ]
       }
@@ -55,7 +60,7 @@ def test_pass_ruby_3
   def test_fail
     assert_fixture(
       :fail, {
-        version: 2,
+        version: 3,
         status: :fail,
         message: nil,
         tests: [
@@ -64,19 +69,22 @@ def test_fail
             test_code: %(assert_equal "One for you, one for me.", TwoFer.two_fer),
             status: :fail,
             message: %(Expected: \"One for you, one for me.\"\n  Actual: \"One for fred, one for me.\"),
-            output: "The name is fred.\nHere's another line.\n"
+            output: "The name is fred.\nHere's another line.\n",
+            task_id: nil
           },
           {
             name: "A name given",
             test_code: 'assert_equal "One for Alice, one for me.", TwoFer.two_fer("Alice")',
             status: :pass,
-            output: "The name is Alice.\nHere's another line.\n"
+            output: "The name is Alice.\nHere's another line.\n",
+            task_id: nil
           },
           {
             name: "Another name given",
             test_code: 'assert_equal "One for Bob, one for me.", TwoFer.two_fer("Bob")',
             status: :pass,
-            output: "The name is Bob.\nHere's another line.\n"
+            output: "The name is Bob.\nHere's another line.\n",
+            task_id: nil
           }
         ]
       }
@@ -95,15 +103,16 @@ def test_deep_exception
     assert_fixture(
       :deep_exception,
       {
-        version: 2,
+        version: 3,
         status: :fail,
         message: nil,
         tests: [
           {
             name: "No name given",
             test_code: 'assert_equal "One for you, one for me.", TwoFer.two_fer',
             status: :error,
-            message:
+            message:,
+            task_id: nil
           }
         ]
       }

tests/attack_large_output/expected_results.json

@@ -1 +1 @@
-{"version":2,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"fail","output":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\n...Output was truncated. Please limit to 500 chars...","message":"Expected: \"One for you, one for me.\"\n  Actual: false"}]}
+{"version":3,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"fail","output":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\n...Output was truncated. Please limit to 500 chars...","message":"Expected: \"One for you, one for me.\"\n  Actual: false","task_id":null}]}

tests/benchmarks/expected_results.json

@@ -1 +1 @@
-{"version":2,"status":"pass","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"pass"},{"name":"A name given","test_code":"assert_equal \"One for Alice, one for me.\", TwoFer.two_fer(\"Alice\")","status":"pass"},{"name":"Another name given","test_code":"assert_equal \"One for Bob, one for me.\", TwoFer.two_fer(\"Bob\")","status":"pass"}]}
+{"version":3,"status":"pass","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"pass","task_id":null},{"name":"A name given","test_code":"assert_equal \"One for Alice, one for me.\", TwoFer.two_fer(\"Alice\")","status":"pass","task_id":null},{"name":"Another name given","test_code":"assert_equal \"One for Bob, one for me.\", TwoFer.two_fer(\"Bob\")","status":"pass","task_id":null}]}

tests/deep_exception/expected_results.json

@@ -1 +1 @@
-{"version":2,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"error","message":"NoMethodError: undefined method `non_existant_method' for nil\n\nTraceback (most recent call first):\n    Line 8:in `work_out_name'\n    Line 3:in `two_fer'"}]}
+{"version":3,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"error","message":"NoMethodError: undefined method `non_existant_method' for nil\n\nTraceback (most recent call first):\n    Line 8:in `work_out_name'\n    Line 3:in `two_fer'","task_id":null}]}

tests/empty/expected_results.json

@@ -1 +1 @@
-{"version":2,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"error","message":"NameError: uninitialized constant TwoFerTest::TwoFer\n\nTraceback (most recent call first):\n    Line 7:in `test_no_name_given'"}]}
+{"version":3,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"error","message":"NameError: uninitialized constant TwoFerTest::TwoFer\n\nTraceback (most recent call first):\n    Line 7:in `test_no_name_given'","task_id":null}]}

tests/exception/expected_results.json

@@ -1 +1 @@
-{"version":2,"status":"error","message":"Line 3: undefined local variable or method `raise_an_error_because_i_am_a_random_method' for main (NameError)","tests":null}
+{"version":3,"status":"error","message":"Line 3: undefined local variable or method `raise_an_error_because_i_am_a_random_method' for main (NameError)","tests":null}

tests/fail/expected_results.json

@@ -1 +1 @@
-{"version":2,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"fail","output":"The name is fred.\nHere's another line.\n","message":"Expected: \"One for you, one for me.\"\n  Actual: \"One for fred, one for me.\""},{"name":"A name given","test_code":"assert_equal \"One for Alice, one for me.\", TwoFer.two_fer(\"Alice\")","status":"pass","output":"The name is Alice.\nHere's another line.\n"},{"name":"Another name given","test_code":"assert_equal \"One for Bob, one for me.\", TwoFer.two_fer(\"Bob\")","status":"pass","output":"The name is Bob.\nHere's another line.\n"}]}
+{"version":3,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"fail","output":"The name is fred.\nHere's another line.\n","message":"Expected: \"One for you, one for me.\"\n  Actual: \"One for fred, one for me.\"","task_id":null},{"name":"A name given","test_code":"assert_equal \"One for Alice, one for me.\", TwoFer.two_fer(\"Alice\")","status":"pass","output":"The name is Alice.\nHere's another line.\n","task_id":null},{"name":"Another name given","test_code":"assert_equal \"One for Bob, one for me.\", TwoFer.two_fer(\"Bob\")","status":"pass","output":"The name is Bob.\nHere's another line.\n","task_id":null}]}

tests/input/expected_results.json

@@ -0,0 +1 @@
+{"version":3,"status":"fail","message":null,"tests":[{"name":"No name given","test_code":"assert_equal \"One for you, one for me.\", TwoFer.two_fer","status":"fail","output":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\n...Output was truncated. Please limit to 500 chars...","message":"Expected: \"One for you, one for me.\"\n  Actual: false","task_id":null}]}

tests/input/two_fer.rb

@@ -0,0 +1,6 @@
+class TwoFer
+  def self.two_fer(_name = "you")
+    debug Array.new(1000) { "a" }.join
+    false
+  end
+end

tests/input/two_fer_test.rb

@@ -0,0 +1,9 @@
+require 'minitest/autorun'
+require_relative 'two_fer'
+
+# Common test data version: 1.2.0 4fc1acb
+class TwoFerTest < Minitest::Test
+  def test_no_name_given
+    assert_equal "One for you, one for me.", TwoFer.two_fer
+  end
+end

tests/metadata/expected_results.json

@@ -1 +1 @@
-{"version":2,"status":"fail","message":null,"tests":[{"name":"Assert equal works properly","test_code":"some_result = TwoFer.two_fer\nassert_equal \"One for you, one for me.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 3:in `test_assert_equal_works_properly'"},{"name":"Skip works properly","test_code":"something = \"Something\"\nassert something.present?","status":"error","message":"NoMethodError: undefined method `present?' for an instance of String\n\nTraceback (most recent call first):\n    Line 5:in `test_skip_works_properly'"},{"name":"Zebra","test_code":"some_result = TwoFer.two_fer(\"zebra\")\nassert_equal \"One for you, one for zebra.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 3:in `test_zebra'"},{"name":"Anaconda","test_code":"some_result = TwoFer.two_fer(\"anaconda\")\nassert_equal \"One for you, one for anaconda.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 8:in `test_anaconda'"},{"name":"Gorilla","test_code":"some_result = TwoFer.two_fer(\"gorilla\")\nassert_equal \"One for you, one for gorilla.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 13:in `test_gorilla'"},{"name":"Boa","test_code":"some_result = TwoFer.two_fer(\"boa\")\nassert_equal \"One for you, one for boa.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 18:in `test_boa'"}]}
+{"version":3,"status":"fail","message":null,"tests":[{"name":"Assert equal works properly","test_code":"some_result = TwoFer.two_fer\nassert_equal \"One for you, one for me.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 3:in `test_assert_equal_works_properly'","task_id":null},{"name":"Skip works properly","test_code":"something = \"Something\"\nassert something.present?","status":"error","message":"NoMethodError: undefined method `present?' for an instance of String\n\nTraceback (most recent call first):\n    Line 5:in `test_skip_works_properly'","task_id":null},{"name":"Zebra","test_code":"some_result = TwoFer.two_fer(\"zebra\")\nassert_equal \"One for you, one for zebra.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 3:in `test_zebra'","task_id":null},{"name":"Anaconda","test_code":"some_result = TwoFer.two_fer(\"anaconda\")\nassert_equal \"One for you, one for anaconda.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 8:in `test_anaconda'","task_id":null},{"name":"Gorilla","test_code":"some_result = TwoFer.two_fer(\"gorilla\")\nassert_equal \"One for you, one for gorilla.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 13:in `test_gorilla'","task_id":null},{"name":"Boa","test_code":"some_result = TwoFer.two_fer(\"boa\")\nassert_equal \"One for you, one for boa.\", some_result","status":"error","message":"NameError: uninitialized constant SomeTest::TwoFer\n\nTraceback (most recent call first):\n    Line 18:in `test_boa'","task_id":null}]}