Merge pull request #27 from education/add-python-to-integration-job

Add Python Runner to Integration Job
github-education-resources · Mar 5, 2024 · a688083 · a688083
2 parents aae65f2 + 0b6cc78
commit a688083
Show file tree

Hide file tree

Showing 4 changed files with 126 additions and 90 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -26,7 +26,7 @@ jobs:
           expected-output: HELLO
           comparison-method: exact
           timeout: 10
-          max-score: 100
+          max-score: 10
       - name: A command test
         id: a-command-test
         uses: education/autograding-command-grader@v1
@@ -35,11 +35,21 @@ jobs:
           setup-command: bundle install
           command: rspec hello_spec.rb
           timeout: 10
-          max-score: 100
+          max-score: 20
+      - name: Python test
+        id: python-test
+        uses: education/autograding-python-grader@add-max-score
+      - name: Python test with score
+        id: python-test-with-score
+        uses: education/autograding-python-grader@add-max-score
+        with:
+          max-score: 30
       - name: Autograding Reporter
         uses: ./
         env:
           SHOUT-TEST_RESULTS: "${{steps.shout-test.outputs.result}}"
           A-COMMAND-TEST_RESULTS: "${{steps.a-command-test.outputs.result}}"
+          PYTHON-TEST_RESULTS: "${{steps.python-test.outputs.result}}"
+          PYTHON-TEST-WITH-SCORE_RESULTS: "${{steps.python-test-with-score.outputs.result}}"
         with:
-          runners: shout-test,a-command-test
+          runners: shout-test,a-command-test,python-test,python-test-with-score
diff --git a/__tests__/main.test.js b/__tests__/main.test.js
@@ -1,83 +1,103 @@
-const process = require("process");
-const cp = require("child_process");
-const path = require("path");
+const process = require('process')
+const cp = require('child_process')
+const path = require('path')
 
-const node = process.execPath;
-const ip = path.join(__dirname, "..", "src", "main.js");
+const node = process.execPath
+const ip = path.join(__dirname, '..', 'src', 'main.js')
 const options = {
   env: process.env,
-  encoding: "utf-8",
-};
-
-const { parseRunnerResults } = require("../src/main");
-
-test("test runs", () => {
-  process.env.RESULT1_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 1", "status": "pass", "message": null }] }').toString("base64");
-  process.env.INPUT_RUNNERS = "result1";
-
-  const child = cp.spawnSync(node, [ip], options);
-  const stdout = child.stdout.toString();
-  expect(stdout).toContain(`✅ Test 1`);
-});
-
-test("test fails", () => {
-  process.env.RESULT1_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 1", "status": "fail", "message": "Test failed with non-zero exit code." }] }').toString("base64");
-  process.env.INPUT_RUNNERS = "result1";
-
-  const child = cp.spawnSync(node, [ip], options);
-  const stdout = child.stdout.toString();
-  expect(stdout).toContain(`❌ Test 1`);
-});
-
-test("test errors out", () => {
-  process.env.RESULT1_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 1", "status": "error", "message": "Test failed to execute." }] }').toString("base64");
-  process.env.INPUT_RUNNERS = "result1";
-
-  const child = cp.spawnSync(node, [ip], options);
-  const stdout = child.stdout.toString();
-  expect(stdout).toContain(`Error: Test failed to execute.`);
-});
-
-test("fails to run if input not in the right format", () => {
-  process.env.RESULT1_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 1", "status": "pass", "message": null }] }').toString("base64");
-  process.env.RESULT2_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 2", "status": "pass", "message": null }] }').toString("base64");
-  process.env.INPUT_RUNNERS = "[result1,result2]";
-
-  const child = cp.spawnSync(node, [ip], options);
-  const stderr = child.stderr.toString();
-  expect(stderr).toContain(`The runners input must be a comma-separated list of strings.`);
-});
-
-// test('test runs with multiple runners', () => {
-//   process.env.RESULT1_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 1", "status": "pass", "message": null }] }').toString('base64')
-//   process.env.RESULT2_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 2", "status": "pass", "message": null }] }').toString('base64')
-//   process.env.INPUT_RUNNERS = 'result1,result2'
-
-//   const child = cp.spawnSync(node, [ip], options)
-//   const stdout = child.stdout.toString()
-//   console.log(stdout)
-//   expect(stdout).toBe('pass')
-// })
-
-// test('test fails with multiple runners', () => {
-//   process.env.RESULT1_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 1", "status": "fail", "message": null }] }').toString('base64')
-//   process.env.RESULT2_RESULTS = Buffer.from('{ "tests": [{ "name": "Test 2", "status": "pass", "message": null }] }').toString('base64')
-//   process.env.INPUT_RUNNERS = 'result1,result2'
-
-//   const child = cp.spawnSync(node, [ip], options)
-//   const stdout = child.stdout.toString()
-//   console.log(stdout)
-//   expect(stdout).toBe('fail')
-// })
-
-test("test autograding-output.parseRunnerResults", function () {
-  process.env.INPUT_RUNNERS = "result1,result2";
-
-  let testResults1 = parseRunnerResults(process.env.INPUT_RUNNERS);
-  expect(testResults1.length).toBe(2);
-  expect(testResults1[0].runner).toBe("result1");
-  expect(testResults1[1].runner).toBe("result2");
-
-  process.env.INPUT_RUNNERS = "[result1,result2]";
-  expect(() => parseRunnerResults(process.env.INPUT_RUNNERS)).toThrow("The runners input must be a comma-separated list of strings.");
-});
+  encoding: 'utf-8',
+}
+
+const {parseRunnerResults} = require('../src/main')
+
+test('test runs', () => {
+  process.env.RESULT1_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 1", "status": "pass", "message": null }] }',
+  ).toString('base64')
+  process.env.INPUT_RUNNERS = 'result1'
+
+  const child = cp.spawnSync(node, [ip], options)
+  const stdout = child.stdout.toString()
+  expect(stdout).toContain(`✅ Test 1`)
+})
+
+test('test fails', () => {
+  process.env.RESULT1_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 1", "status": "fail", "message": "Test failed with non-zero exit code." }] }',
+  ).toString('base64')
+  process.env.INPUT_RUNNERS = 'result1'
+
+  const child = cp.spawnSync(node, [ip], options)
+  const stdout = child.stdout.toString()
+  expect(stdout).toContain(`❌ Test 1`)
+})
+
+test('test errors out', () => {
+  process.env.RESULT1_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 1", "status": "error", "message": "Test failed to execute." }] }',
+  ).toString('base64')
+  process.env.INPUT_RUNNERS = 'result1'
+
+  const child = cp.spawnSync(node, [ip], options)
+  const stdout = child.stdout.toString()
+  expect(stdout).toContain(`Error: Test failed to execute.`)
+})
+
+test('fails to run if input not in the right format', () => {
+  process.env.RESULT1_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 1", "status": "pass", "message": null }] }',
+  ).toString('base64')
+  process.env.RESULT2_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 2", "status": "pass", "message": null }] }',
+  ).toString('base64')
+  process.env.INPUT_RUNNERS = '[result1,result2]'
+
+  const child = cp.spawnSync(node, [ip], options)
+  const stderr = child.stderr.toString()
+  expect(stderr).toContain(`The runners input must be a comma-separated list of strings.`)
+})
+
+test('test runs with multiple runners', () => {
+  process.env.RESULT1_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 1", "status": "pass", "message": null }] }',
+  ).toString('base64')
+  process.env.RESULT2_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 2", "status": "pass", "message": null }] }',
+  ).toString('base64')
+  process.env.INPUT_RUNNERS = 'result1,result2'
+
+  const child = cp.spawnSync(node, [ip], options)
+  const stdout = child.stdout.toString()
+  console.log(stdout)
+  expect(stdout).toContain('✅')
+})
+
+test('test fails with multiple runners', () => {
+  process.env.RESULT1_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 1", "status": "fail", "message": null }] }',
+  ).toString('base64')
+  process.env.RESULT2_RESULTS = Buffer.from(
+    '{ "tests": [{ "name": "Test 2", "status": "pass", "message": null }] }',
+  ).toString('base64')
+  process.env.INPUT_RUNNERS = 'result1,result2'
+
+  const child = cp.spawnSync(node, [ip], options)
+  const stdout = child.stdout.toString()
+  console.log(stdout)
+  expect(stdout).toContain('❌')
+})
+
+test('test autograding-output.parseRunnerResults', function () {
+  process.env.INPUT_RUNNERS = 'result1,result2'
+
+  const testResults1 = parseRunnerResults(process.env.INPUT_RUNNERS)
+  expect(testResults1.length).toBe(2)
+  expect(testResults1[0].runner).toBe('result1')
+  expect(testResults1[1].runner).toBe('result2')
+
+  process.env.INPUT_RUNNERS = '[result1,result2]'
+  expect(() => parseRunnerResults(process.env.INPUT_RUNNERS)).toThrow(
+    'The runners input must be a comma-separated list of strings.',
+  )
+})
diff --git a/src/console-results.js b/src/console-results.js
@@ -22,13 +22,23 @@ exports.ConsoleResults = function ConsoleResults(runnerResults) {
 
       results.tests.forEach((test) => {
         if (test.status === 'pass') {
-          console.log(`${COLORS.green}✅ ${test.name}${COLORS.reset}`)
           passedTests += 1
+          if (test.line_no !== 0) {
+            console.log(`${COLORS.green}✅ ${test.name} - line ${test.line_no}${COLORS.reset}`)
+          } else {
+            console.log(`${COLORS.green}✅ ${test.name}${COLORS.reset}`)
+          }
         } else if (test.status === 'error') {
           console.log(`Error: ${test.message || `Failed to run test '${test.name}'`}\n${COLORS.reset}`)
         } else {
-          console.log(`${COLORS.red}❌ ${test.name}\n`)
-          console.log(`${test.message || ''}\n${COLORS.reset}`)
+          if (test.line_no !== 0) {
+            console.log(`${COLORS.red}❌ ${test.name} - line ${test.line_no}${COLORS.reset}`)
+          } else {
+            console.log(`${COLORS.red}❌ ${test.name}${COLORS.reset}`)
+          }
+        }
+        if (test.test_code) {
+          console.log(`Test code:\n${test.test_code}\n`)
         }
       })
 

diff --git a/test_sample.py b/test_sample.py
@@ -5,7 +5,3 @@ def sample_from_collection(collection):
 def test_sample():
     collection = [1, 2, 3, 4, 5]
     assert sample_from_collection(collection) in collection
-
-def test_sample_poorly():
-    collection = [1, 2, 3, 4, 5]
-    assert sample_from_collection(collection) in [6]