0.5.7

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+# [0.5.7] (2022-11-17)
+
+- Fixed an issue where `mypy` fails to show issues.
+- Added `Python Test Explorer` extension to Dev Container.
+- Changed pytest settings to `verbose` mode.
+- Maintenance and bug fixes.
+
 # [0.5.6] (2022-10-30)
 
 - Fixed broken installation of local development environment with python and poetry.

Scripts/setup_dependencies.sh

@@ -37,7 +37,8 @@ if [[ $dependencies == *"pyenv"* ]]; then
     export PATH="$PYENV_ROOT/bin":$PATH;
     eval "$(pyenv init -)"
     # get latest python version from the pyenv list
-    LATEST_PYTHON=$(pyenv install --list | grep --extended-regexp "^\s*3[0-9.]*[0-9]\s*$" | tail -1 | xargs);
+    # regex for python version 3.10.*
+    LATEST_PYTHON=$(pyenv install --list | grep --extended-regexp "^\s*3\.10\.[0-9]{1,2}\s*$" | tail -1 | xargs);
     pyenv install $LATEST_PYTHON --force
     pyenv install 2.7.18 --force
     pyenv global $LATEST_PYTHON 2.7.18;

Scripts/setup_venv.sh

@@ -10,8 +10,7 @@ eval "$(pyenv init -)" || echo "No pyenv, procceding without"
 dockerImage=$1
 name=$2
 dirPath=$3
-extraReqs=$4
-pythonPath=$5
+pythonPath=$4
 
 # workaround to support M1
 export PATH=/opt/homebrew/bin:$PATH
@@ -34,20 +33,19 @@ docker run --name "${name}" "$testImage" 'pip list --format=freeze > /requiremen
 docker cp "${name}":/requirements.txt .
 docker rm -f "${name}" || true
 
+
+
 # check if virtualenv module is installed
 isVirtualEnvInstalled=true
 $pythonPath -m virtualenv --version > /dev/null 2>&1 || isVirtualEnvInstalled=false
 if [ "$isVirtualEnvInstalled" = "false" ]; then
     $pythonPath -m pip install virtualenv
 fi
+
 $pythonPath -m virtualenv -p python"${pythonVersion}" venv
+
 venv/bin/pip --version || (echo "No pip, check your python"${pythonVersion}" installation" && exit 1)
+
 while read line; do
     venv/bin/pip install --disable-pip-version-check --no-cache-dir $line || echo "Could not install dependency $line, proceeding"
-done < requirements.txt
-venv/bin/pip install bandit --disable-pip-version-check --no-cache-dir || echo "Could not install bandit"
-venv/bin/pip install autopep8 --disable-pip-version-check --no-cache-dir || echo "Could not install autopep8"
-venv/bin/pip install flake8 --disable-pip-version-check --no-cache-dir || echo "Could not install flake8"
-if [ "${pythonVersion}" = "3" ]; then
-    venv/bin/pip install -r "$extraReqs" --disable-pip-version-check --no-cache-dir || echo "Could not install mypy"
-fi
+done < requirements.txt

Templates/integration_env/.devcontainer/Dockerfile

@@ -3,7 +3,7 @@ ARG IMAGENAME
 FROM ${IMAGENAME}
 
 ADD create_certs.sh /
-ADD extra-requirements-py3.txt /
+
 RUN chmod +x /create_certs.sh \ 
     && /create_certs.sh /usr/local/share/ca-certificates/certs.crt \
     && update-ca-certificates \
@@ -13,12 +13,12 @@ RUN chmod +x /create_certs.sh \
     && apk add build-base || apt-get install build-essential -y || echo "Could not install build-essential" \
     # check if pip command exists
     && if [ -x "$(command -v pip)" ]; then \
-    pip install autopep8 || echo "Could not install autopep8"; \
-    pip install flake8 || echo "Could not install flake8"; \
-    if [ $(python -c 'import sys; print(sys.version_info[0])') == "3" ]; then \
-        # mypy supports only py3
-        pip install -r /extra-requirements-py3.txt || echo "Could not install mypy"; \
+        pip install autopep8 flake8 || echo "Could not install autopep8 or flake8"; \
+        if [ $(python -c 'import sys; print(sys.version_info[0])') == "3" ]; then \
+            # mypy supports only py3
+            pip install mypy types-dateparser types-requests || echo "Could not install mypy"; \
+        fi \
     fi \
-    fi
+    && git config --system --add safe.directory /workspaces/content
 
 ENV NODE_EXTRA_CA_CERTS /usr/local/share/ca-certificates/certs.crt

Templates/integration_env/.devcontainer/devcontainer.json

@@ -6,15 +6,20 @@
 			"IMAGENAME": ""
 		}
 	},
+	"remoteEnv": {
+		"MYPYPATH": "/workspaces/content/Packs/Base/Scripts/CommonServerPython:${containerEnv:MYPYPATH}",
+		"PYTHONPATH": "/workspaces/content/Packs/Base/Scripts/CommonServerPython:${containerEnv:PYTHONPATH}"
+
+	},
 	"containerUser": "demisto",
-	"postCreateCommand": "git config --system --add safe.directory /workspaces/content",
 	"extensions": [
 		"ms-python.python",
 		"ms-vscode.PowerShell",
 		"cortexxsoarext.xsoar",
 		"eamodio.gitlens",
 		"GitHub.vscode-pull-request-github",
 		"njpwerner.autodocstring",
-		"streetsidesoftware.code-spell-checker"
+		"streetsidesoftware.code-spell-checker",
+		"littlefoxteam.vscode-python-test-adapter"
 	]
 }

Templates/settings.json

@@ -2,7 +2,7 @@
     "python.testing.cwd": "Packs/HelloWorld/Integrations/HelloWorld",
     "python.testing.pytestEnabled": true,
     "python.testing.pytestArgs": [
-        "."
+        "-v"
     ],
     "python.testing.unittestEnabled": false,
     "python.linting.mypyEnabled": true,
@@ -11,9 +11,18 @@
         "--ignore-missing-imports",
         "--show-column-numbers",
         "--no-pretty",
-        "--allow-redefinition",
-        "--namespace-packages"
+        "--allow-redefinition"
     ],
     "python.linting.flake8Enabled": true,
     "python.defaultInterpreterPath": ".venv/bin/python",
+    "cSpell.words": [
+        "demisto",
+        "xsoar",
+        "xsiam",
+        "fromversion",
+        "toversion",
+        "marketplacev2",
+        "ciac",
+        "whois"
+    ]
 }

package.json

@@ -2,7 +2,7 @@
   "name": "xsoar",
   "displayName": "Cortex XSOAR",
   "description": "Build, Format, and Validate Cortex XSOAR with ease.",
-  "version": "0.5.6",
+  "version": "0.5.7",
   "engines": {
     "vscode": "^1.54.0"
   },

src/devEnvs.ts

@@ -10,6 +10,27 @@ import { parse, stringify } from "envfile"
 import { installDemistoSDKGlobally, getContentPath } from "./tools";
 import glob from "glob";
 
+async function addPythonPath(): Promise<void>{
+    const contentPath = getContentPath()
+    if (!contentPath) {
+        return
+    }
+    let PYTHONPATH = `${contentPath}/Packs/Base/Scripts/CommonServerPython/:${contentPath}/Tests/demistomock/:`
+    const apiModules = execSync(`printf '%s:' ${contentPath}/Packs/ApiModules/Scripts/*`).toString().trim()
+    PYTHONPATH += apiModules
+    const envFilePath = path.join(contentPath, '.env')
+    if (!await fs.pathExists(envFilePath)) {
+        fs.createFileSync(envFilePath)
+    }
+    const envFile = fs.readFileSync(envFilePath, 'utf8')
+    Logger.info(`envFile: ${envFile}`)
+    const env = parse(envFile)
+    env["PYTHONPATH"] = PYTHONPATH
+    env["MYPYPATH"] = PYTHONPATH
+    Logger.info(stringify(env))
+    fs.writeFileSync(envFilePath, stringify(env))
+
+}
 
 export async function installDevEnv(): Promise<void> {
     const workspaces = vscode.workspace.workspaceFolders;
@@ -73,21 +94,7 @@ export async function installDevEnv(): Promise<void> {
     fs.copyFileSync(settingsFile, settingsFileOutput)
 
     // set up environment variables
-    let PYTHONPATH = `${dirPath}/Packs/Base/Scripts/CommonServerPython/:${dirPath}/Tests/demistomock/:`
-    const apiModules = execSync(`printf '%s:' ${dirPath}/Packs/ApiModules/Scripts/*`).toString().trim()
-    PYTHONPATH += apiModules
-    const envFilePath = path.join(dirPath, '.env')
-    if (!await fs.pathExists(envFilePath)) {
-        fs.createFileSync(envFilePath)
-    }
-    const envFile = fs.readFileSync(envFilePath, 'utf8')
-    Logger.info(`envFile: ${envFile}`)
-    const env = parse(envFile)
-    env["PYTHONPATH"] = PYTHONPATH
-    env["MYPYPATH"] = PYTHONPATH
-    Logger.info(JSON5.stringify(env))
-    Logger.info(stringify(env))
-    fs.writeFileSync(envFilePath, stringify(env))
+    await addPythonPath()
     fs.createFileSync(path.join(dirPath, 'Packs/Base/Scripts/CommonServerPython/CommonServerUserPython.py'))
 
     await vscode.window.showQuickPick(['Yes', 'No'], {
@@ -314,9 +321,8 @@ export async function openIntegrationDevContainer(dirPath: string): Promise<void
         fs.removeSync(CommonServerPython)
     }
     createLaunchJson(ymlObject.type, dirPath, filePath, vsCodePath);
-    createSettings(vsCodePath, dirPath, '/usr/local/bin/python')
+    createSettings(vsCodePath, dirPath, '/usr/local/bin/python', false)
 
-    fs.createFile(path.join(dirPath, 'DemistoClassApiModule.py'))
     await dsdk.lint(dirPath, false, false, false, true)
 
     // delete cache folders and *.pyc files
@@ -439,9 +445,11 @@ export async function openInVirtualenv(dirPath: string): Promise<void> {
     if (await fs.pathExists(CommonServerPython)) {
         fs.removeSync(CommonServerPython)
     }
-    fs.createFile(path.join(dirPath, 'DemistoClassApiModule.py'))
     createLaunchJson(ymlObject.type, dirPath, filePath, vsCodePath);
-    createSettings(vsCodePath, dirPath, path.join(dirPath, 'venv', 'bin', 'python'));
+    createSettings(vsCodePath, dirPath, path.join(dirPath, 'venv', 'bin', 'python'), true);
+    await addPythonPath()
+    fs.copySync(path.join(contentPath, '.env'), path.join(dirPath, '.env'))
+
     Logger.info('Run lint')
     await dsdk.lint(dirPath, false, false, false, true)
 
@@ -466,8 +474,12 @@ export async function openInVirtualenv(dirPath: string): Promise<void> {
     vscode.commands.executeCommand('vscode.openFolder', vscode.Uri.file(workspaceOutput), openInNewWindow)
 }
 
-async function createSettings(vsCodePath: string, dirPath: string, interpreterPath: string) {
+async function createSettings(vsCodePath: string, dirPath: string, interpreterPath: string, changeLinterPath: boolean) {
     Logger.info('Getting settings.json');
+    const contentPath = getContentPath()
+    if (!contentPath) {
+        throw new Error("No content path configured")
+    }
     const settingsPath = path.join(vsCodePath, 'settings.json')
     // create if not exists
     if (!await fs.pathExists(settingsPath)) {
@@ -477,8 +489,9 @@ async function createSettings(vsCodePath: string, dirPath: string, interpreterPa
 
     const settings = JSON5.parse(fs.readFileSync(settingsPath, 'utf-8'));
     Logger.info('Setting settings.json to venv');
+    const relativePath = '${workspaceFolder}/' + path.relative(path.join(vsCodePath, '..'), dirPath)
     settings['python.defaultInterpreterPath'] = interpreterPath;
-    settings['python.testing.cwd'] = '${workspaceFolder}/' + `${path.relative(path.join(vsCodePath, '..'), dirPath)}`;
+    settings['python.testing.cwd'] = relativePath;
     settings["python.testing.pytestEnabled"] = true;
     settings["python.testing.pytestArgs"] = ["."];
     settings["python.linting.mypyEnabled"] = true;
@@ -488,9 +501,21 @@ async function createSettings(vsCodePath: string, dirPath: string, interpreterPa
         "--show-column-numbers",
         "--no-pretty",
         "--allow-redefinition",
-        "--check-untyped-defs"
+        "--check-untyped-defs",
     ];
     settings["python.linting.flake8Enabled"] = true;
+
+    if (changeLinterPath && fs.pathExistsSync(path.join(contentPath, ".venv"))) {
+        settings["python.linting.mypyPath"] = `${contentPath}/.venv/bin/mypy`
+        settings["python.linting.flake8Path"] = `${contentPath}/.venv/bin/flake8`
+        settings["python.formatting.autopep8Path"] = `${contentPath}/.venv/bin/autopep8`
+    }
+    else {
+        settings["python.linting.mypyPath"] = 'mypy'
+        settings["python.linting.flake8Path"] = 'flake8'
+        settings["python.formatting.autopep8Path"] = 'autopep8'
+
+    }
     fs.writeJSONSync(settingsPath, settings, { spaces: 2 });
 }
 
@@ -538,9 +563,8 @@ async function createVirtualenv(name: string, dirPath: string, dockerImage: stri
     // this implemented in a script, should be a command in SDK.
     // When SDK added this command, change to use it as wrapper.
     Logger.info('Running virtualenv task')
-    const extraReqsPY3 = path.resolve(__dirname, '../Templates/integration_env/.devcontainer/extra-requirements-py3.txt')
     const setupVenvScript = path.resolve(__dirname, '../Scripts/setup_venv.sh')
-    const cmd = `${setupVenvScript} ${dockerImage} ${name} ${dirPath} ${extraReqsPY3} ` + "${command:python.interpreterPath}"
+    const cmd = `${setupVenvScript} ${dockerImage} ${name} ${dirPath} ` + "${command:python.interpreterPath}"
     const task = new vscode.Task(
         { type: 'virtualenv', name: 'Setup virtualenv' },
         vscode.TaskScope.Workspace,