Webdriver tests in parallel (#695)

* Initial commit for webdriver tests in parallel

* fix linting'

* fixes to doc

cloudbuild.screenshot.yaml

@@ -33,25 +33,33 @@ steps:
 
   # Run the webdriver tests
   - id: flask_webdriver_test
-    name: gcr.io/datcom-ci/webdriver-chrome:2020-10-21
+    name: gcr.io/datcom-ci/webdriver-chrome:2020-11-25
     entrypoint: /bin/sh
     waitFor:
       - package_js
     args:
       - -c
       - |
-        ./run_test.sh -w
+        # Point SELENIUM_SERVER to the JAR file, needed to start Selenium Server/Grid.
+        export SELENIUM_SERVER=/resources/selenium-server-standalone-3.141.59.jar
+
+        # -t enables parallel testing flag.
+        ./run_test.sh -tw
 
   # Run screenshot test and save the images
   - id: screenshot_test
-    name: gcr.io/datcom-ci/webdriver-chrome:2020-10-21
+    name: gcr.io/datcom-ci/webdriver-chrome:2020-11-25
     entrypoint: /bin/sh
     waitFor:
       - flask_webdriver_test
     args:
       - -c
       - |
-        ./run_test.sh -s
+        # Point SELENIUM_SERVER to the JAR file, needed to start Selenium Server/Grid.
+        export SELENIUM_SERVER=/resources/selenium-server-standalone-3.141.59.jar
+
+        # -t enables parallel testing flag.
+        ./run_test.sh -ts
 
   # Copy over the screenshots to gcs
   - name: gcr.io/cloud-builders/gsutil

run_test.sh

@@ -16,6 +16,22 @@
 
 set -e
 
+# Starts Selenium Server/Grid.
+function start_selenium_server {
+
+if [[ ! $SELENIUM_SERVER ]]
+  then
+  echo -e "no SELENIUM_SERVER environment variable found"
+  echo -e "please point SELENIUM_SERVER to JAR file"
+  echo -e "download Selenium Server/Grid JAR file from https://www.selenium.dev/downloads/"
+  exit 1
+  fi
+
+  # Start the Selenium Server in the background.
+  # nohup and & (ampersand) is used for silent startup.
+  nohup java -Dwebdriver.chrome.whitelistedIps= -jar $SELENIUM_SERVER -port 4444&
+}
+
 # Run test for client side code.
 function run_npm_test {
   cd static
@@ -96,7 +112,14 @@ function run_webdriver_test {
   export FLASK_ENV=webdriver
   export GOOGLE_CLOUD_PROJECT=datcom-browser-staging
   pip3 install -r requirements.txt
-  python3 -m pytest webdriver_tests/*.py
+
+  if [ $PYTEST_PARALLEL ]
+  then
+    echo -e "#### Running webdriver tests in parallel"
+    python3 -m pytest --tests-per-worker auto webdriver_tests/*.py
+  else
+    python3 -m pytest webdriver_tests/*.py
+  fi
   cd ..
 }
 
@@ -109,16 +132,24 @@ function run_screenshot_test {
     echo "no dist folder, please run ./run_test.sh -b to build js first."
     exit 1
   fi
+
   export FLASK_ENV=webdriver
   export GOOGLE_CLOUD_PROJECT=datcom-browser-staging
   pip3 install -r requirements.txt
   if [  -d test_screenshots  ]
   then
-    echo "Delete the test_screenshots folder"
+    echo "delete the test_screenshots folder"
     rm -rf test_screenshots
   fi
   mkdir test_screenshots
-  python3 -m pytest webdriver_tests/screenshot/screenshot_test.py
+
+  if [ $PYTEST_PARALLEL ]
+  then
+    echo -e "#### Running screenshot tests in parallel"
+    python3 -m pytest --tests-per-worker auto webdriver_tests/screenshot/screenshot_test.py
+  else
+    python3 -m pytest webdriver_tests/screenshot/screenshot_test.py
+  fi
   cd ..
 }
 
@@ -132,7 +163,8 @@ function run_all_tests {
 }
 
 function help {
-  echo "Usage: $0 -pwblcsaf"
+  echo "Usage: $0 -tpwblcsaf"
+  echo "-t       Run tests in parallel"
   echo "-p       Run server python tests"
   echo "-w       Run webdriver tests"
   echo "-o       Build for production (ignores dev dependencies)"
@@ -145,8 +177,16 @@ function help {
   exit 1
 }
 
-while getopts pwoblcsaf OPTION; do
+# Always reset the variable null.
+unset PYTEST_PARALLEL
+while getopts tpwotblcsaf OPTION; do
   case $OPTION in
+    t)
+        echo -e "### Parallel flag enabled"
+        export PYTEST_PARALLEL=true
+        # Start Selenium Server/Grid on port 4444.
+        start_selenium_server
+        ;;
     p)
         echo -e "### Running server tests"
         run_py_test

server/requirements.txt

@@ -6,7 +6,7 @@ google-cloud-storage
 google-cloud-secret-manager==1.0.0
 jinja2
 parameterized
-pytest
+pytest-parallel
 requests
 six
 wheel

server/webdriver_tests/README.md

@@ -6,16 +6,37 @@ Run the following command from the parent directory:
 
     ./run_tests.sh -w
 
+To run the tests in parallel, add the -t (threading) flag:
+NOTE: -t flag requires that Selenium Server/Grid is downloaded and the environment variable SELENIUM_SERVER points to the JAR file.
+Download Selenium Server/Grid JAR file from https://www.selenium.dev/downloads/
+Verified to be working on selenium-server-standalone-3.141.59.jar
+
+    export SELENIUM_SERVER=/path/to/selenium-server-standalone*.jar
+    ./run_tests.sh -tw
+
 ## Things To Note
 
 Data Commons sites can take up to a few seconds to load; thus, WebDriver needs to be told what elements to wait for to know that the page has finished loading/rendering.
 
-- `SLEEP_SEC` represents the maximum time to wait. If this time is exceeded, the test will fail with a `TimeoutException`. By default, the test cases use 15 seconds.
+- `SLEEP_SEC` represents the maximum time for a test to finish. If this time is exceeded, the test will fail with a `TimeoutException`. By default, the test cases use 60 seconds.
 - WebDriver allows to select HTML elements using `By.ID, By.CSS_SELECTOR, By.CLASS_NAME, By.XPATH`.
 - Sometimes, it is more convenient to use CSS_SELECTOR or XPATH.
   - `By.CSS_SELECTOR('.myclass:nth-element(3)')`
   - `By.XPATH('//*[@id="my-id"]/span/button')`
 
+- LiveServerTestCase will run the methods in the following order:
+
+  - setUpClass()
+  - create_app()
+  - setUp()
+  - test1()
+  - tearDown()
+  - create_app()
+  - setUp()
+  - test2()
+  - tearDown()
+  - tearDownClass()
+
 ### 1. Wait Until HTML Element is Present in DOM
 
 WebDriver can wait for an element to be present in the DOM using `presence_of_element_located`. An example would be to wait for "my-button" to appear after an event has triggered.

server/webdriver_tests/base_test.py

@@ -14,32 +14,58 @@
 
 from flask_testing import LiveServerTestCase
 from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
 from main import app
+from os import environ
+
+# Read flag from OS' environment.
+# TODO(edumorales): Figure out a way to pass down an argument using Pytest.
+PYTEST_PARALLEL = environ.get("PYTEST_PARALLEL")
 
 
 # Base test class to setup the server.
+# Please refer to README.md to see the order of method execution during test.
 class WebdriverBaseTest(LiveServerTestCase):
 
-    @classmethod
-    def setUpClass(cls):
-        cls.port = 12345
-        super(WebdriverBaseTest, cls).setUpClass()
-
     def create_app(self):
-        return app
+        """Returns the Flask Server running Data Commons."""
+        app_instance = app
+        # Each test will start its own Flask Server.
+        # Port 0 is used to let Flask pick any available port.
+        # If no port is specified, port 5000 will be used for all tests which
+        # may cause some racing issue when running tests.
+        app_instance.config['LIVESERVER_PORT'] = 0
+        return app_instance
 
     def setUp(self):
-        """Will be called before every test"""
-        chrome_options = webdriver.ChromeOptions()
+        """Runs at the beginning of every individual test."""
+        # These options are needed to run ChromeDriver inside a Docker without a UI.
+        chrome_options = Options()
         chrome_options.add_argument('--headless')
         chrome_options.add_argument('--no-sandbox')
         chrome_options.add_argument('--disable-dev-shm-usage')
 
         # Maximum time, in seconds, before throwing a TimeoutException.
-        self.TIMEOUT_SEC = 25
+        self.TIMEOUT_SEC = 60
+
+        # If flag is enabled, connect to Selenium Grid.
+        if PYTEST_PARALLEL:
+            # Connect to port 4444, where Selenium Grid is running.
+            # Tell Selenium Grid you need a new ChromeDriver instance.
+            # Selenium Grid will be in charge of keeping track of all the ChromeDriver instances.
+            self.driver = webdriver.Remote(
+                command_executor="http://0.0.0.0:4444/wd/hub",
+                desired_capabilities=webdriver.DesiredCapabilities.CHROME,
+                options=chrome_options)
+        # Otherwise, start a simple WebDriver instance.
+        else:
+            self.driver = webdriver.Chrome(options=chrome_options)
 
-        self.driver = webdriver.Chrome(options=chrome_options)
+        # The URL of the Data Commons server.
         self.url_ = self.get_server_url()
 
     def tearDown(self):
+        """Runs at the end of every individual test."""
+        # Quit the ChromeDriver instance.
+        # NOTE: Every individual test starts a new ChromeDriver instance.
         self.driver.quit()

webdriver-chrome/Dockerfile

@@ -14,20 +14,25 @@
 
 FROM python:3.7
 
-# install google chrome
+WORKDIR /resources
+
+
+# Install Google Chrome.
 RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
 RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list'
-RUN apt-get -y update
-RUN apt-get install -y google-chrome-stable
+RUN apt -y update
+RUN apt install -y google-chrome-stable
 
-# install chromedriver
-RUN apt-get install -yqq unzip
+# Install ChromeDriver.
 RUN wget -O /tmp/chromedriver.zip http://chromedriver.storage.googleapis.com/`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE`/chromedriver_linux64.zip
 RUN unzip /tmp/chromedriver.zip chromedriver -d /usr/bin/
 RUN chown root:root /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
 
-# set display port to avoid crash
-ENV DISPLAY=:99
+# Download Selenium Server JAR, handles multiple instances of ChromeDriver in parallel.
+RUN wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar
 
-CMD ["--help"]
+# Install Java, needed to run Selenium Server JAR.
+RUN apt install -yqq software-properties-common
+RUN apt -y update
+RUN apt install -y default-jre

webdriver-chrome/README.md

@@ -2,7 +2,11 @@
 
 ## Description
 
-This is a Docker image based on python:3.7-slim, but with chrome and chromedriver preinstalled which can help using Selenium to do automation tests.
+This is a Docker image based on python:3.7, but comes with some other tools:
+    1. Google Chrome: the browser which is used to run the tests.
+    2. ChromeDriver: used to send commands to Google Chrome.
+    3. Java JDK: used to run the Selenium Server, which comes as a JAR file.
+    4. Selenium Server: used to start multiple ChromeDriver instances and run tests in parallel.
 
 ## How to build the Docker image
 
@@ -14,6 +18,6 @@ gcloud builds submit . --config=cloudbuild.yaml
 
 Note: You may need to contact Data Commons team to get permission to push image into `datcom-ci` project.
 
-## How to update the Docker image with newer chromedriver version
+## How to update the Docker image with a newer ChromeDriver version
 
-You can change the `VERS` argument in `Dockerfile` and `_VERS` argument in `cloudbuild.yaml` file with the chromedriver version you want, then run the above command.
+You can change the `VERS` argument in `Dockerfile` and `_VERS` argument in `cloudbuild.yaml` file with the ChromeDriver version you want, then run the above command.

webdriver-chrome/cloudbuild.yaml

@@ -14,7 +14,7 @@
 
 # Version can be found at: https://chromedriver.storage.googleapis.com/LATEST_RELEASE
 substitutions:
-  _VERS: "2020-10-21"
+  _VERS: "2020-11-25"
 
 steps:
   - name: "gcr.io/cloud-builders/docker"