feat: handling multilines results with custom output handler

Lazily `require` JSON module in output handler

We `require` the output handler in Neovim in order to get the full path
to its source.  This executes all top-level statements of the handler,
including any `require` calls.  If one of the required modules is not
installed (in this case `dkjson`) an error is raised.

The solution is to move the `require` call into the function call of the
module.  The module will not be called by Neovim, but by busted, and the
dkjson module will be present in that context.

Add commentary (motivation) to output handler

Undo stylistic changes

Shorten name of `output_handler_path` variable

Makes it easier to read.

Search for last result line containing the marker

Rename marker variable

The variable is already scoped to the output handler, so the name can be
something short.

Add tests which write to standard output

These tests are meant to be run from within Neovim.

Add test for when output contains result marker

This is why it is important to search for the output marker backwards.

lua/neotest-busted/_build_spec.lua

@@ -69,7 +69,7 @@ return function(args)
 	local command = vim.tbl_flatten {
 		vim.g.bustedprg or 'busted',
 		'--output',
-		'json'
+		require('neotest-busted._output-handler').source
 	}
 
 	-- The user has selected a specific node inside the file

lua/neotest-busted/_output-handler.lua

@@ -0,0 +1,56 @@
+---This is a modified version of the standard JSON output handler from busted.
+---The main difference is that this handler inserts an explicit separator
+---before the JSON result output.  The reason is that busted writes both the
+---standard output from tests and the result of running tests to standard
+---output, potentially mixing the two on the same line.
+
+local io_write = io.write
+local io_flush = io.flush
+
+local M = {}
+
+---Prefix to mark the line containing test results as opposed to the standard
+---output of the test
+M.marker = '::NEOTEST_LINE::'
+
+---Full path to this module so it can be referenced by Neovim.  We have no
+---control over the working directory of the Neovim process, so the output
+---handler has to know its own absolute file path.  Neovim can then require the
+---handler as a module and get this information.
+M.source = debug.getinfo(1).source:sub(2)
+
+function M:__call(_options)
+	local json = require('dkjson')
+	local busted = require('busted')
+	local handler = require('busted.outputHandlers.base')()
+
+	handler.suiteEnd = function()
+		local error_info = {
+			pendings = handler.pendings,
+			successes = handler.successes,
+			failures = handler.failures,
+			errors = handler.errors,
+			duration = handler.getDuration(),
+		}
+		local ok, result = pcall(json.encode, error_info)
+
+		io_write('\n' .. M.marker)
+
+		if ok then
+			io_write(result)
+		else
+			io_write('Failed to encode test results to json: ' .. result)
+		end
+
+		io_write('\n')
+		io_flush()
+
+		return nil, true
+	end
+
+	busted.subscribe({ 'suite', 'end' }, handler.suiteEnd)
+
+	return handler
+end
+
+return setmetatable(M, M)

lua/neotest-busted/_results.lua

@@ -10,9 +10,17 @@ local writefile = vim.fn.writefile
 ---@param path string  Path to file containing JSON output
 ---@return table output  Arbitrary JSON data from the output
 local function decode_result_output(path)
-	-- Assumption: the output will be all one line.  There might be other junk
-	-- on subsequent lines and we don't want that.
-	local result = vim.json.decode(vim.fn.readfile(path)[1])
+	local marker = require('neotest-busted._output-handler').marker
+	local lines = vim.fn.readfile(path)
+
+	-- NOTE: Searching backwards because there might be a line of regular text
+	-- output which happens to have the same prefix.  We want only the last
+	-- matching line.
+	local line = vim.iter(lines)
+		:rev()
+		:find(function(l) return vim.startswith(l, marker) end)
+		:sub(#marker + 1)
+	local result = vim.json.decode(line)
 
 	-- Write a human-readable representation of the test result to the output
 	-- file. The output file contains JSON which we convert into regular text.

makefile

@@ -33,6 +33,8 @@ unit-test:
 integration-test:
 	@./test/bin/busted --run integration
 
+test: unit-test integration-test
+
 clean:
 	@# Delete everything except for the trust file
 	@for f in test/xdg/local/state/nvim/*; do ([ "$$(basename $$f)" != 'trust' ] && rm -r "$$f") || true; done

test/unit/build_spec_spec.lua

@@ -1,10 +1,11 @@
-local adapter = require 'neotest-busted'
-local conf    = require 'neotest-busted._conf'
-local nio = require 'nio'
-local types = require 'neotest.types'
+local adapter        = require 'neotest-busted'
+local conf           = require 'neotest-busted._conf'
+local nio            = require 'nio'
+local types          = require 'neotest.types'
+local output_handler = require 'neotest-busted._output-handler'.source
 
-local split = vim.fn.split
-local writefile = vim.fn.writefile
+local split          = vim.fn.split
+local writefile      = vim.fn.writefile
 
 
 describe('Building the test run specification', function()
@@ -26,11 +27,11 @@ describe('Building the test run specification', function()
 		return assert(adapter.build_spec(args))
 	end
 
-	before_each(function()  -- Create temporary file
+	before_each(function() -- Create temporary file
 		tempfile = vim.fn.tempname() .. '.lua'
 	end)
 
-	after_each(function()  -- Delete temporary file
+	after_each(function() -- Delete temporary file
 		if vim.fn.filereadable(tempfile) ~= 0 then
 			vim.fn.delete(tempfile)
 		end
@@ -55,7 +56,13 @@ describe('Building the test run specification', function()
 			return add(x, y)
 		]]
 
-		local expected = {'busted', '--output', 'json', '--', tempfile}
+		local expected = {
+			'busted',
+			'--output',
+			output_handler,
+			'--',
+			tempfile,
+		}
 		assert.are.same(expected, spec.command)
 	end)
 
@@ -69,9 +76,13 @@ describe('Building the test run specification', function()
 		local spec = build_spec(content, key)
 
 		local expected = {
-			'busted', '--output', 'json', '--filter',
+			'busted',
+			'--output',
+			output_handler,
+			'--filter',
 			'Fulfills%sa%stautology,%sa%sself%-evident%s100%%%strue%sstatement',
-			'--', tempfile
+			'--',
+			tempfile,
 		}
 		assert.are.same(expected, spec.command)
 	end)
@@ -90,8 +101,13 @@ describe('Building the test run specification', function()
 		local spec = build_spec(content, tempfile .. '::Arithmetic')
 
 		local expected = {
-			'busted', '--output', 'json', '--filter', 'Arithmetic',
-			'--', tempfile
+			'busted',
+			'--output',
+			output_handler,
+			'--filter',
+			'Arithmetic',
+			'--',
+			tempfile,
 		}
 		assert.are.same(expected, spec.command)
 	end)
@@ -110,9 +126,13 @@ describe('Building the test run specification', function()
 		local spec = build_spec(content, tempfile .. '::Arithmetic::Adds two numbers')
 
 		local expected = {
-			'busted', '--output', 'json',
-			'--filter', 'Arithmetic%sAdds%stwo%snumbers',
-			'--', tempfile
+			'busted',
+			'--output',
+			output_handler,
+			'--filter',
+			'Arithmetic%sAdds%stwo%snumbers',
+			'--',
+			tempfile,
 		}
 		assert.are.same(expected, spec.command)
 	end)
@@ -153,14 +173,30 @@ describe('Building the test run specification', function()
 				end)
 			]]
 
-			local expected = {'busted', '--output', 'json', '--run', 'unit', '--', tempfile}
+			local expected = {
+				'busted',
+				'--output',
+				output_handler,
+				'--run',
+				'unit',
+				'--',
+				tempfile,
+			}
 			assert.are.same(expected, spec.command)
 		end)
 
 		it('Specifies the bustedrc file', function()
 			conf.set({_all = {verbose = true}}, 'bustedrc')
 			local spec = build_spec ''
-			local expected = {'busted', '--output', 'json', '--config-file', 'bustedrc', '--', tempfile}
+			local expected = {
+				'busted',
+				'--output',
+				output_handler,
+				'--config-file',
+				'bustedrc',
+				'--',
+				tempfile,
+			}
 			assert.are.same(expected, spec.command)
 		end)
 	end)
@@ -173,14 +209,27 @@ describe('Building the test run specification', function()
 		it('Uses the custom executable', function()
 			vim.g.bustedprg = './test/busted'
 			local spec = build_spec ''
-			local expected = {'./test/busted', '--output', 'json', '--', tempfile}
+			local expected = {
+				'./test/busted',
+				'--output',
+				output_handler,
+				'--',
+				tempfile,
+			}
 			assert.are.same(expected, spec.command)
 		end)
 
 		it('Splices in a custom busted command list', function()
 			vim.g.bustedprg = {'busted', '--verbose'}
 			local spec = build_spec ''
-			local expected = {'busted', '--verbose', '--output', 'json', '--', tempfile}
+			local expected = {
+				'busted',
+				'--verbose',
+				'--output',
+				output_handler,
+				'--',
+				tempfile,
+			}
 			assert.are.same(expected, spec.command)
 		end)
 	end)
@@ -210,8 +259,8 @@ describe('Building the test run specification', function()
 
 		it('Runs all tasks with matching roots', function()
 			local expected = {
-				{command = {'busted', '--output', 'json', '--config-file', 'bustedrc', '--run', 'integration', '--', 'test/integration'}},
-				{command = {'busted', '--output', 'json', '--config-file', 'bustedrc', '--run', 'unit', '--', 'test/unit'}},
+				{command = {'busted', '--output', output_handler, '--config-file', 'bustedrc', '--run', 'integration', '--', 'test/integration'}},
+				{command = {'busted', '--output', output_handler, '--config-file', 'bustedrc', '--run', 'unit', '--', 'test/unit'}},
 			}
 
 			-- A directory tree which contains two more directory trees which
@@ -242,8 +291,8 @@ describe('Building the test run specification', function()
 								path = 'test/integration/foo_spec.lua',
 								range = {0, 0, 4, 0},
 								type = 'test',
-							}
-						}
+							},
+						},
 					},
 				}, {
 					{
@@ -265,20 +314,19 @@ describe('Building the test run specification', function()
 								path = 'test/unit/foo_spec.lua',
 								range = {0, 0, 4, 0},
 								type = 'test',
-							}
+							},
 						}
 					},
-
 				}
 			}
 			local dir_tree = types.Tree.from_list(t, function(pos) return pos.id end)
-			local spec = assert(adapter.build_spec {tree = dir_tree, strategy = 'integrated'})
+			local spec = assert(adapter.build_spec { tree = dir_tree, strategy = 'integrated' })
 
 			-- NOTE: The order of specifications is undefined, so we need to
 			-- explicitly sort the two list.
 			local function comp(t1, t2) return t1.command[7] < t2.command[7] end
 			table.sort(expected, comp)
-			table.sort(spec    , comp)
+			table.sort(spec,     comp)
 			assert.are.same(expected, spec)
 		end)
 	end)

test/unit/results_spec.lua

@@ -21,8 +21,10 @@ describe('Result from running busted', function()
 	local testfile
 
 	---@param sample string  Name of the sample file (without path or extension)
+	---@param pre string|nil Text to prepend results
+	---@param post string|nil text to append to results
 	---@return table<string, neotest.Result>, table<string, neotest.Result>
-	local function compute_test_results(sample)
+	local function compute_test_results(sample, pre, post)
 		local sample_file = string.format('test/unit/samples/%s.lua', sample)
 		local content, output, spec, expected, descend = loadfile(sample_file)()(testfile)
 
@@ -44,7 +46,14 @@ describe('Result from running busted', function()
 
 		-- We need to write the output to an actual file for the `results`
 		-- function
-		vim.fn.writefile({vim.json.encode(output)}, strategy_result.output, 's')
+		if pre then
+			vim.fn.writefile({pre}, strategy_result.output, 's')
+		end
+		local marker = require('neotest-busted._output-handler').marker
+		vim.fn.writefile({marker .. vim.json.encode(output)}, strategy_result.output, 'as')
+		if post then
+			vim.fn.writefile({post}, strategy_result.output, 'as')
+		end
 
 		local results = adapter.results(spec, strategy_result, tree)
 		return expected, results
@@ -123,4 +132,12 @@ describe('Result from running busted', function()
 		local expected, results = compute_test_results('error-msg-with-esc-seq')
 		assert.are.same(expected, results)
 	end)
+
+	it('Handles multiple lines output', function()
+		local expected, results = compute_test_results('single-standalone-success', 'prepend\n', '\nappend')
+		assert.are.same(expected, results)
+
+		expected, results = compute_test_results('single-standalone-error', 'prepend', 'append')
+		assert.are.same(expected, results)
+	end)
 end)

test/unit/samples/empty.lua

@@ -1,5 +1,6 @@
 -- An empty test file.
 
+local output_handler = require 'neotest-busted._output-handler'.source
 
 local content = ''
 
@@ -15,7 +16,7 @@ local output = {
 
 return function(tempfile)
 	local spec = {
-		command = {'busted', '--output', 'json', '--', tempfile}
+		command = {'busted', '--output', output_handler, '--', tempfile}
 	}
 
 	return content, output, spec, expected_results

test/unit/samples/error-msg-with-esc-seq.lua

@@ -1,5 +1,6 @@
 -- Failure where the error message contains escape sequences
 
+local output_handler = require 'neotest-busted._output-handler'.source
 
 local types = require 'neotest.types'
 
@@ -66,7 +67,7 @@ return function(tempfile)
 	}
 
 	local spec = {
-		command = {'busted', '--output', 'json', '--', tempfile}
+		command = {'busted', '--output', output_handler, '--', tempfile}
 	}
 
 	return content, output, spec, expected_results

test/unit/samples/error-parent-after_each.lua

@@ -1,5 +1,6 @@
 -- A parent `after_each` throws an error
 
+local output_handler = require 'neotest-busted._output-handler'.source
 
 local types = require 'neotest.types'
 
@@ -71,7 +72,7 @@ return function(tempfile)
 	}
 
 	local spec = {
-		command = {'busted','--output', 'json',  '--filter', 'Arithmetic%sAdditive%sAdds%stwo%snumbers', '--', tempfile}
+		command = {'busted', '--output', output_handler, '--filter', 'Arithmetic%sAdditive%sAdds%stwo%snumbers', '--', tempfile}
 	}
 
 	return content, output, spec, expected_results, {1, 2}