diff --git a/CHANGELOG.md b/CHANGELOG.md index 2637b72..9f06e41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Change Log +# [0.5.7] (2022-11-17) + +- Fixed an issue where `mypy` fails to show issues. +- Added `Python Test Explorer` extension to Dev Container. +- Changed pytest settings to `verbose` mode. +- Maintenance and bug fixes. + # [0.5.6] (2022-10-30) - Fixed broken installation of local development environment with python and poetry. diff --git a/Scripts/setup_dependencies.sh b/Scripts/setup_dependencies.sh index 5b3c97e..a988d21 100755 --- a/Scripts/setup_dependencies.sh +++ b/Scripts/setup_dependencies.sh @@ -37,7 +37,8 @@ if [[ $dependencies == *"pyenv"* ]]; then export PATH="$PYENV_ROOT/bin":$PATH; eval "$(pyenv init -)" # get latest python version from the pyenv list - LATEST_PYTHON=$(pyenv install --list | grep --extended-regexp "^\s*3[0-9.]*[0-9]\s*$" | tail -1 | xargs); + # regex for python version 3.10.* + LATEST_PYTHON=$(pyenv install --list | grep --extended-regexp "^\s*3\.10\.[0-9]{1,2}\s*$" | tail -1 | xargs); pyenv install $LATEST_PYTHON --force pyenv install 2.7.18 --force pyenv global $LATEST_PYTHON 2.7.18; diff --git a/Scripts/setup_venv.sh b/Scripts/setup_venv.sh index 3f5e2da..98a4bdd 100755 --- a/Scripts/setup_venv.sh +++ b/Scripts/setup_venv.sh @@ -10,8 +10,7 @@ eval "$(pyenv init -)" || echo "No pyenv, procceding without" dockerImage=$1 name=$2 dirPath=$3 -extraReqs=$4 -pythonPath=$5 +pythonPath=$4 # workaround to support M1 export PATH=/opt/homebrew/bin:$PATH @@ -34,20 +33,19 @@ docker run --name "${name}" "$testImage" 'pip list --format=freeze > /requiremen docker cp "${name}":/requirements.txt . docker rm -f "${name}" || true + + # check if virtualenv module is installed isVirtualEnvInstalled=true $pythonPath -m virtualenv --version > /dev/null 2>&1 || isVirtualEnvInstalled=false if [ "$isVirtualEnvInstalled" = "false" ]; then $pythonPath -m pip install virtualenv fi + $pythonPath -m virtualenv -p python"${pythonVersion}" venv + venv/bin/pip --version || (echo "No pip, check your python"${pythonVersion}" installation" && exit 1) + while read line; do venv/bin/pip install --disable-pip-version-check --no-cache-dir $line || echo "Could not install dependency $line, proceeding" -done < requirements.txt -venv/bin/pip install bandit --disable-pip-version-check --no-cache-dir || echo "Could not install bandit" -venv/bin/pip install autopep8 --disable-pip-version-check --no-cache-dir || echo "Could not install autopep8" -venv/bin/pip install flake8 --disable-pip-version-check --no-cache-dir || echo "Could not install flake8" -if [ "${pythonVersion}" = "3" ]; then - venv/bin/pip install -r "$extraReqs" --disable-pip-version-check --no-cache-dir || echo "Could not install mypy" -fi \ No newline at end of file +done < requirements.txt \ No newline at end of file diff --git a/Templates/integration_env/.devcontainer/Dockerfile b/Templates/integration_env/.devcontainer/Dockerfile index caf5a6c..9315285 100644 --- a/Templates/integration_env/.devcontainer/Dockerfile +++ b/Templates/integration_env/.devcontainer/Dockerfile @@ -3,7 +3,7 @@ ARG IMAGENAME FROM ${IMAGENAME} ADD create_certs.sh / -ADD extra-requirements-py3.txt / + RUN chmod +x /create_certs.sh \ && /create_certs.sh /usr/local/share/ca-certificates/certs.crt \ && update-ca-certificates \ @@ -13,12 +13,12 @@ RUN chmod +x /create_certs.sh \ && apk add build-base || apt-get install build-essential -y || echo "Could not install build-essential" \ # check if pip command exists && if [ -x "$(command -v pip)" ]; then \ - pip install autopep8 || echo "Could not install autopep8"; \ - pip install flake8 || echo "Could not install flake8"; \ - if [ $(python -c 'import sys; print(sys.version_info[0])') == "3" ]; then \ - # mypy supports only py3 - pip install -r /extra-requirements-py3.txt || echo "Could not install mypy"; \ + pip install autopep8 flake8 || echo "Could not install autopep8 or flake8"; \ + if [ $(python -c 'import sys; print(sys.version_info[0])') == "3" ]; then \ + # mypy supports only py3 + pip install mypy types-dateparser types-requests || echo "Could not install mypy"; \ + fi \ fi \ - fi + && git config --system --add safe.directory /workspaces/content ENV NODE_EXTRA_CA_CERTS /usr/local/share/ca-certificates/certs.crt diff --git a/Templates/integration_env/.devcontainer/devcontainer.json b/Templates/integration_env/.devcontainer/devcontainer.json index 2b64f1f..b5b17e6 100644 --- a/Templates/integration_env/.devcontainer/devcontainer.json +++ b/Templates/integration_env/.devcontainer/devcontainer.json @@ -6,8 +6,12 @@ "IMAGENAME": "" } }, + "remoteEnv": { + "MYPYPATH": "/workspaces/content/Packs/Base/Scripts/CommonServerPython:${containerEnv:MYPYPATH}", + "PYTHONPATH": "/workspaces/content/Packs/Base/Scripts/CommonServerPython:${containerEnv:PYTHONPATH}" + + }, "containerUser": "demisto", - "postCreateCommand": "git config --system --add safe.directory /workspaces/content", "extensions": [ "ms-python.python", "ms-vscode.PowerShell", @@ -15,6 +19,7 @@ "eamodio.gitlens", "GitHub.vscode-pull-request-github", "njpwerner.autodocstring", - "streetsidesoftware.code-spell-checker" + "streetsidesoftware.code-spell-checker", + "littlefoxteam.vscode-python-test-adapter" ] } diff --git a/Templates/integration_env/.devcontainer/extra-requirements-py3.txt b/Templates/integration_env/.devcontainer/extra-requirements-py3.txt deleted file mode 100644 index 139c977..0000000 --- a/Templates/integration_env/.devcontainer/extra-requirements-py3.txt +++ /dev/null @@ -1,3 +0,0 @@ -mypy -types-dateparser -types-requests \ No newline at end of file diff --git a/Templates/settings.json b/Templates/settings.json index a64f7a6..020d318 100644 --- a/Templates/settings.json +++ b/Templates/settings.json @@ -2,7 +2,7 @@ "python.testing.cwd": "Packs/HelloWorld/Integrations/HelloWorld", "python.testing.pytestEnabled": true, "python.testing.pytestArgs": [ - "." + "-v" ], "python.testing.unittestEnabled": false, "python.linting.mypyEnabled": true, @@ -11,9 +11,18 @@ "--ignore-missing-imports", "--show-column-numbers", "--no-pretty", - "--allow-redefinition", - "--namespace-packages" + "--allow-redefinition" ], "python.linting.flake8Enabled": true, "python.defaultInterpreterPath": ".venv/bin/python", + "cSpell.words": [ + "demisto", + "xsoar", + "xsiam", + "fromversion", + "toversion", + "marketplacev2", + "ciac", + "whois" + ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index e4df93a..18bdd31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xsoar", - "version": "0.5.6", + "version": "0.5.7", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "xsoar", - "version": "0.5.6", + "version": "0.5.7", "license": "SEE LICENSE IN LICENSE", "dependencies": { "envfile": "^6.17.0", diff --git a/package.json b/package.json index 47df9f5..92471a5 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "xsoar", "displayName": "Cortex XSOAR", "description": "Build, Format, and Validate Cortex XSOAR with ease.", - "version": "0.5.6", + "version": "0.5.7", "engines": { "vscode": "^1.54.0" }, diff --git a/src/devEnvs.ts b/src/devEnvs.ts index 7e226fd..75f8963 100644 --- a/src/devEnvs.ts +++ b/src/devEnvs.ts @@ -10,6 +10,27 @@ import { parse, stringify } from "envfile" import { installDemistoSDKGlobally, getContentPath } from "./tools"; import glob from "glob"; +async function addPythonPath(): Promise{ + const contentPath = getContentPath() + if (!contentPath) { + return + } + let PYTHONPATH = `${contentPath}/Packs/Base/Scripts/CommonServerPython/:${contentPath}/Tests/demistomock/:` + const apiModules = execSync(`printf '%s:' ${contentPath}/Packs/ApiModules/Scripts/*`).toString().trim() + PYTHONPATH += apiModules + const envFilePath = path.join(contentPath, '.env') + if (!await fs.pathExists(envFilePath)) { + fs.createFileSync(envFilePath) + } + const envFile = fs.readFileSync(envFilePath, 'utf8') + Logger.info(`envFile: ${envFile}`) + const env = parse(envFile) + env["PYTHONPATH"] = PYTHONPATH + env["MYPYPATH"] = PYTHONPATH + Logger.info(stringify(env)) + fs.writeFileSync(envFilePath, stringify(env)) + +} export async function installDevEnv(): Promise { const workspaces = vscode.workspace.workspaceFolders; @@ -73,21 +94,7 @@ export async function installDevEnv(): Promise { fs.copyFileSync(settingsFile, settingsFileOutput) // set up environment variables - let PYTHONPATH = `${dirPath}/Packs/Base/Scripts/CommonServerPython/:${dirPath}/Tests/demistomock/:` - const apiModules = execSync(`printf '%s:' ${dirPath}/Packs/ApiModules/Scripts/*`).toString().trim() - PYTHONPATH += apiModules - const envFilePath = path.join(dirPath, '.env') - if (!await fs.pathExists(envFilePath)) { - fs.createFileSync(envFilePath) - } - const envFile = fs.readFileSync(envFilePath, 'utf8') - Logger.info(`envFile: ${envFile}`) - const env = parse(envFile) - env["PYTHONPATH"] = PYTHONPATH - env["MYPYPATH"] = PYTHONPATH - Logger.info(JSON5.stringify(env)) - Logger.info(stringify(env)) - fs.writeFileSync(envFilePath, stringify(env)) + await addPythonPath() fs.createFileSync(path.join(dirPath, 'Packs/Base/Scripts/CommonServerPython/CommonServerUserPython.py')) await vscode.window.showQuickPick(['Yes', 'No'], { @@ -314,9 +321,8 @@ export async function openIntegrationDevContainer(dirPath: string): Promise { if (await fs.pathExists(CommonServerPython)) { fs.removeSync(CommonServerPython) } - fs.createFile(path.join(dirPath, 'DemistoClassApiModule.py')) createLaunchJson(ymlObject.type, dirPath, filePath, vsCodePath); - createSettings(vsCodePath, dirPath, path.join(dirPath, 'venv', 'bin', 'python')); + createSettings(vsCodePath, dirPath, path.join(dirPath, 'venv', 'bin', 'python'), true); + await addPythonPath() + fs.copySync(path.join(contentPath, '.env'), path.join(dirPath, '.env')) + Logger.info('Run lint') await dsdk.lint(dirPath, false, false, false, true) @@ -466,8 +474,12 @@ export async function openInVirtualenv(dirPath: string): Promise { vscode.commands.executeCommand('vscode.openFolder', vscode.Uri.file(workspaceOutput), openInNewWindow) } -async function createSettings(vsCodePath: string, dirPath: string, interpreterPath: string) { +async function createSettings(vsCodePath: string, dirPath: string, interpreterPath: string, changeLinterPath: boolean) { Logger.info('Getting settings.json'); + const contentPath = getContentPath() + if (!contentPath) { + throw new Error("No content path configured") + } const settingsPath = path.join(vsCodePath, 'settings.json') // create if not exists if (!await fs.pathExists(settingsPath)) { @@ -477,8 +489,9 @@ async function createSettings(vsCodePath: string, dirPath: string, interpreterPa const settings = JSON5.parse(fs.readFileSync(settingsPath, 'utf-8')); Logger.info('Setting settings.json to venv'); + const relativePath = '${workspaceFolder}/' + path.relative(path.join(vsCodePath, '..'), dirPath) settings['python.defaultInterpreterPath'] = interpreterPath; - settings['python.testing.cwd'] = '${workspaceFolder}/' + `${path.relative(path.join(vsCodePath, '..'), dirPath)}`; + settings['python.testing.cwd'] = relativePath; settings["python.testing.pytestEnabled"] = true; settings["python.testing.pytestArgs"] = ["."]; settings["python.linting.mypyEnabled"] = true; @@ -488,9 +501,21 @@ async function createSettings(vsCodePath: string, dirPath: string, interpreterPa "--show-column-numbers", "--no-pretty", "--allow-redefinition", - "--check-untyped-defs" + "--check-untyped-defs", ]; settings["python.linting.flake8Enabled"] = true; + + if (changeLinterPath && fs.pathExistsSync(path.join(contentPath, ".venv"))) { + settings["python.linting.mypyPath"] = `${contentPath}/.venv/bin/mypy` + settings["python.linting.flake8Path"] = `${contentPath}/.venv/bin/flake8` + settings["python.formatting.autopep8Path"] = `${contentPath}/.venv/bin/autopep8` + } + else { + settings["python.linting.mypyPath"] = 'mypy' + settings["python.linting.flake8Path"] = 'flake8' + settings["python.formatting.autopep8Path"] = 'autopep8' + + } fs.writeJSONSync(settingsPath, settings, { spaces: 2 }); } @@ -538,9 +563,8 @@ async function createVirtualenv(name: string, dirPath: string, dockerImage: stri // this implemented in a script, should be a command in SDK. // When SDK added this command, change to use it as wrapper. Logger.info('Running virtualenv task') - const extraReqsPY3 = path.resolve(__dirname, '../Templates/integration_env/.devcontainer/extra-requirements-py3.txt') const setupVenvScript = path.resolve(__dirname, '../Scripts/setup_venv.sh') - const cmd = `${setupVenvScript} ${dockerImage} ${name} ${dirPath} ${extraReqsPY3} ` + "${command:python.interpreterPath}" + const cmd = `${setupVenvScript} ${dockerImage} ${name} ${dirPath} ` + "${command:python.interpreterPath}" const task = new vscode.Task( { type: 'virtualenv', name: 'Setup virtualenv' }, vscode.TaskScope.Workspace,