Chore: Add tests for linters

integration-tests/project-folder/sources/meta-fixtures/diagnostics.bb

@@ -1,3 +1,11 @@
-python () {
-    error()
-}
+foo () {
+    echo "${@"" if True else False}"
+    echo "${@d.getVar('PV').split('.')[0]}"
+}
+
+def test ():
+    error('looooooooooooooooooooooooooooooooooooooooooooooooooooooooooong line')
+
+python __anonymous() {
+    import os
+}

integration-tests/src/runTest.ts

@@ -10,7 +10,7 @@ import {
   resolveCliArgsFromVSCodeExecutablePath,
   runTests
 } from '@vscode/test-electron'
-import { pythonVersion, bashVersion } from './utils/version'
+import { pythonVersion, bashVersion, pylintVersion, flake8Version, shellcheckVersion } from './utils/version'
 
 async function main (): Promise<void> {
   try {
@@ -23,7 +23,10 @@ async function main (): Promise<void> {
       [
         ...args,
         '--install-extension', `mads-hartmann.bash-ide-vscode@${bashVersion}`,
-        '--install-extension', `ms-python.python@${pythonVersion}`
+        '--install-extension', `ms-python.python@${pythonVersion}`,
+        '--install-extension', `timonwong.shellcheck@${shellcheckVersion}`,
+        '--install-extension', `ms-python.flake8@${flake8Version}`,
+        '--install-extension', `ms-python.pylint@${pylintVersion}`
       ],
       {
         encoding: 'utf-8',

integration-tests/src/tests/diagnostics.test.ts

@@ -35,10 +35,163 @@ suite('Bitbake Diagnostics Test Suite', () => {
   test('Diagnostics', async () => {
     void vscode.workspace.openTextDocument(docUri)
     await assertWillComeTrue(async () => {
-      const diagnostics = vscode.languages.getDiagnostics(docUri)
-      return diagnostics.length === 1 &&
-        diagnostics[0].source === 'Pylance, bitbake-python' &&
-        diagnostics[0].range.isEqual(new vscode.Range(1, 4, 1, 9))
+      const diagnosticsResult = vscode.languages.getDiagnostics()
+      // generateTestCases(diagnosticsResult)
+      const diagnostics = diagnosticsResult.find(
+        ([uri]) => uri.toString() === docUri.toString()
+      )?.at(1)
+
+      if (
+        !Array.isArray(diagnostics) || // For unknown reasons, Typescript thinks "diagnostics" could be an Uri
+        diagnostics.length !== expectedDiagnostics.length
+      ) {
+        return false
+      }
+
+      const hasAllExpectedDiagnostics = diagnostics.every((diagnostic) => {
+        return expectedDiagnostics.some((expectedDiagnostic) => {
+          return (
+            diagnostic.range.isEqual(expectedDiagnostic.range) &&
+            diagnostic.message === expectedDiagnostic.message &&
+            getCode(diagnostic) === expectedDiagnostic.code &&
+            diagnostic.source === expectedDiagnostic.source
+          )
+        })
+      })
+
+      // Make sure our test case has generated all the diagnostics that should be ignored
+      /* const hasGeneratedDiagnosticsToIgnore = diagnosticsResult.some(
+        // There is one file...
+        ([, diagnostics]) =>
+          // for which all ignored codes...
+          ignoredCodes.every(
+            // have at least one corresponding diagnostic
+            ([source, code]) => diagnostics.some(
+              (diagnostic) => hasSourceWithCode(diagnostic, source, code)
+            )
+          )
+      ) */
+
+      return hasAllExpectedDiagnostics // && hasGeneratedDiagnosticsToIgnore
     })
   }).timeout(BITBAKE_TIMEOUT)
 })
+
+const getCode = (diagnostic: vscode.Diagnostic): string | number | undefined => {
+  if (typeof diagnostic.code === 'string' || typeof diagnostic.code === 'number') {
+    return diagnostic.code
+  }
+  return diagnostic.code?.value
+}
+
+/* const hasSourceWithCode = (diagnostic: vscode.Diagnostic, source: string, code: string): boolean => {
+  if (diagnostic.source?.includes(source) !== true) {
+    return false
+  }
+  if (diagnostic.code === code) {
+    return true
+  }
+  if (typeof diagnostic.code === 'object' && diagnostic.code?.value === code) {
+    return true
+  }
+  return false
+}
+
+const ignoredCodes = [
+  ['Flake8', 'E203'],
+  ['Flake8', 'E211'],
+  ['Flake8', 'E302'],
+  ['Flake8', 'E303'],
+  ['Flake8', 'E501'],
+  ['Flake8', 'W391'],
+  ['Pylint', 'C0114:missing-module-docstring'],
+  ['Pylint', 'C0116:missing-function-docstring'],
+  ['Pylint', 'C0305:trailing-newlines'],
+  ['Pylint', 'C0415:import-outside-toplevel'],
+  ['Pylint', 'W0104:pointless-statement'],
+  ['Pylint', 'W0106:expression-not-assigned']
+] */
+
+const expectedDiagnostics = [
+  {
+    range: new vscode.Range(1, 19, 1, 23),
+    message: 'Using a conditional statement with a constant value',
+    code: 'W0125:using-constant-test',
+    source: 'Pylint, bitbake-python'
+  },
+  {
+    range: new vscode.Range(9, 4, 9, 13),
+    message: 'Redefining name \'os\' (imported by BitBake)',
+    code: 'W0621:redefined-outer-name',
+    source: 'Pylint, bitbake-python'
+  },
+  {
+    range: new vscode.Range(9, 4, 9, 13),
+    message: 'Reimport \'os\' (imported by BitBake)',
+    code: 'W0404:reimported',
+    source: 'Pylint, bitbake-python'
+  },
+  {
+    range: new vscode.Range(9, 4, 9, 13),
+    message: 'Unused import os',
+    code: 'W0611:unused-import',
+    source: 'Pylint, bitbake-python'
+  },
+  {
+    range: new vscode.Range(9, 4, 9, 4),
+    message: '\'os\' imported but unused',
+    code: 'F401',
+    source: 'Flake8, bitbake-python'
+  },
+  {
+    range: new vscode.Range(9, 4, 9, 4),
+    message: 'redefinition of unused \'os\' (imported by BitBake)',
+    code: 'F811',
+    source: 'Flake8, bitbake-python'
+  },
+  {
+    range: new vscode.Range(9, 11, 9, 13),
+    message: '"os" is not accessed',
+    code: undefined,
+    source: 'Pylance, bitbake-python'
+  },
+  {
+    range: new vscode.Range(6, 4, 6, 9),
+    message: 'Undefined variable \'error\'',
+    code: 'E0602:undefined-variable',
+    source: 'Pylint, bitbake-python'
+  },
+  {
+    range: new vscode.Range(6, 4, 6, 4),
+    message: 'undefined name \'error\'',
+    code: 'F821',
+    source: 'Flake8, bitbake-python'
+  },
+  {
+    range: new vscode.Range(6, 4, 6, 9),
+    message: '"error" is not defined',
+    code: 'reportUndefinedVariable',
+    source: 'Pylance, bitbake-python'
+  }
+]
+
+// Minimal effort helper to generate the code for the test cases when they need to be updated.
+// The output is intented to be copy-pasted into this file.
+export const generateTestCases = (diagnosticsResult: ReturnType<typeof vscode.languages.getDiagnostics>): void => {
+  for (const [uri, diagnostics] of diagnosticsResult) {
+    console.log(uri)
+    for (const diagnostic of diagnostics) {
+      const range = diagnostic.range
+      const rangeString = `${range.start.line}, ${range.start.character}, ${range.end.line}, ${range.end.character}`
+      const message = diagnostic.message.replace(/'/g, '\\\'')
+      const code = getCode(diagnostic)
+      console.log('  {')
+      console.log(`    range: new vscode.Range(${rangeString}),`)
+      console.log(`    message: '${message}',`)
+      console.log(`    code: ${typeof code === 'string' ? `'${code}'` : code},`)
+      console.log(`    source: '${diagnostic.source}'`)
+      console.log('  },')
+    }
+    console.log(']')
+  }
+}

integration-tests/src/utils/version.ts

@@ -5,4 +5,7 @@
 
 // Required extension versions
 export const bashVersion = '1.41.0'
-export const pythonVersion = '2024.6.0'
+export const pythonVersion = '2024.10.0'
+export const shellcheckVersion = '0.37.1'
+export const flake8Version = '2023.10.0'
+export const pylintVersion = '2023.10.1'

scripts/update-ref.sh

@@ -45,6 +45,27 @@ echo "export const pythonVersion = '$(git tag --sort=-v:refname | head -n 1 | se
 cd ..
 rm -rf vscode-python
 
+git clone --depth 1 --filter=blob:none --sparse https://github.com/vscode-shellcheck/vscode-shellcheck
+cd vscode-shellcheck
+git fetch --tags
+echo "export const shellcheckVersion = '$(git tag --sort=-v:refname | head -n 1 | sed "s/^v//")'" >> ../$TMP
+cd ..
+rm -rf vscode-shellcheck
+
+git clone --depth 1 --filter=blob:none --sparse https://github.com/microsoft/vscode-flake8
+cd vscode-flake8
+git fetch --tags
+echo "export const flake8Version = '$(git tag --sort=-v:refname | head -n 1 | sed "s/^v//")'" >> ../$TMP
+cd ..
+rm -rf vscode-flake8
+
+git clone --depth 1 --filter=blob:none --sparse https://github.com/microsoft/vscode-pylint
+cd vscode-pylint
+git fetch --tags
+echo "export const pylintVersion = '$(git tag --sort=-v:refname | head -n 1 | sed "s/^v//")'" >> ../$TMP
+cd ..
+rm -rf vscode-pylint
+
 cp $TMP $DEST
 rm $TMP