Update Python packaging tools; Update Python dependencies; Prepare fo…

…r RHEL migration - convert selenium tests to use Firefox

CHANGELOG

@@ -1,3 +1,7 @@
+9.4.0   Update Python packaging tools
+        Update Python dependencies
+        Prepare for RHEL migration - convert selenium tests to use Firefox
+
 9.3.0   Make use of pypi index for django-clamd
         Update Cypress
         TMMA-506: Bug fix parsing OVID abstracts that include HTML tags at the start of a line.

deploy/Dockerfile

@@ -4,11 +4,11 @@ ARG REQUIREMENTS_FILE='requirements.txt'
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
-ENV PIP_VERSION='24.0'
-ENV SETUPTOOLS_VERSION='70.0.0'
+ENV PIP_VERSION='24.1.1'
+ENV SETUPTOOLS_VERSION='70.2.0'
 ENV PIP_TOOLS_VERSION='7.4.1'
-# As per Centos installation
-ENV VE_VERSION='20.24.5'
+# As per RHEL installation
+ENV VE_VERSION='15.1.0'
 
 # Update system packages and install dependencies
 RUN apt update -y && \
@@ -22,6 +22,7 @@ RUN apt update -y && \
             curl \
             default-mysql-client \
             default-libmysqlclient-dev \
+            firefox \
             gcc \
             g++ \
             libffi-dev  \

deploy/fabfile.py

@@ -21,8 +21,8 @@
 
 # Tools not handled by pip-tools and/or requirements installs using pip
 # Also update pip version in tests/build-test-env.sh and Dockerfile
-PIP_VERSION = '24.0'
-SETUPTOOLS_VERSION = '70.0.0'
+PIP_VERSION = '24.1.1'
+SETUPTOOLS_VERSION = '70.2.0'
 PIP_TOOLS_VERSION = '7.4.1'
 
 
@@ -143,35 +143,6 @@ def make_virtualenv(env="dev", configure_apache=False, clone_repo=False, branch=
     if restart_rqworker:
         start_rqworker_service(use_local_mode)
 
-    # Install local copy of the chromedriver
-    if env in ("dev", "test",):
-        with change_dir(venv_dir + "/bin"):            
-            # Only install the chrome driver if google chrome is installed.
-            if caller('which google-chrome'):
-                try:
-                    # Download the correct chrome driver version for the version of google chrome that is currently installed,
-                    # ref: https://chromedriver.chromium.org/downloads/version-selection
-                    google_chrome_version = caller('google-chrome --version').strip("Google Chrome ")
-                    print(f'Stripped chrome driver version: {google_chrome_version}.')
-                    google_chrome_version = google_chrome_version[:google_chrome_version.rindex(".")]
-                    print(f'Truncated chrome driver version: {google_chrome_version}.')
-                    version = urlopen(f'https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_{google_chrome_version}').read().decode()
-                    print(f'Look up version of chrome driver that should be compatible with this version of chrome {version}.')
-                    if int(google_chrome_version[:google_chrome_version.index(".")]) < 115:
-                        zip_name = 'chromedriver_linux64.zip'
-                        caller(f'wget https://chromedriver.storage.googleapis.com/{version}/{zip_name}')    
-                    else:
-                        zip_name = 'chromedriver-linux64.zip'
-                        caller(f'wget https://storage.googleapis.com/chrome-for-testing-public/{version}/linux64/{zip_name}')
-                    caller('ls -l')
-                    caller('rm -f chromedriver')
-                    caller(f'unzip -o -j {zip_name}')
-                    caller('ls -l')
-                    caller(f'rm {zip_name}*')
-                    caller('ls -l')
-                except Exception as e:
-                    print("Errors when trying to install the latest Chrome Driver: {e}")
-
 
 def deploy(env="dev", branch="master", using_apache=True, migrate_db=True, use_local_mode=False, use_pip_sync=False, requirements="requirements", project_dir=PROJECT_ROOT):
     """NB: env = dev|prod.  Optionally tag and merge the release env="dev", branch="master", using_apache=True, migrate_db=True, use_local_mode=False, use_pip_sync=False, requirements="requirements"."""

tests/base_selenium_test_case.py

@@ -29,9 +29,7 @@ def setUpClass(cls):
         super(SeleniumBaseTestCase, cls).setUpClass()
         cls.display = Display(visible=0, size=(1920, 1080))
         cls.display.start()
-        # ref: https://github.com/SeleniumHQ/selenium/issues/12746
-        service = webdriver.ChromeService(executable_path=settings.BASE_DIR + "/../../../bin/chromedriver")
-        cls.driver = webdriver.Chrome(service=service)
+        cls.driver = webdriver.Firefox()
         cls.driver.implicitly_wait(10)
 
     @classmethod
@@ -82,11 +80,3 @@ def tearDown(self):
         # self.create_debug_logs()
         delete_user_content(self.user)
         super(SeleniumBaseTestCase, self).tearDown()
-
-    # def create_debug_logs(self, also_print=False):
-    #     """Chrome console logs"""
-    #     for entry in self.driver.get_log('browser'):
-    #         logger.info(entry)
-    #         if also_print:
-    #             print(entry)
-

tests/build-test-env.sh

@@ -24,14 +24,14 @@ mkdir -p var/tmp
 echo "Check the version of pip that is installed"
 pip3 -V
 echo "Ensure using a consistent version of pip as per on premises CI server"
-pip3 install -U pip==24.0
+pip3 install -U pip==24.1.1
 pip3 -V
 
 echo "Install Fabric"
 pip3 install fabric==1.15.0 # NB: v1.15.0 supports Python 2, & 3.6, 3.7, & 3.8
 
 echo "Install virtualenv"
-pip3 install virtualenv==20.24.5 # As per CentOS installation
+pip3 install virtualenv==15.1.0 # As per RHEL installation
 
 echo "Install wheel"
 sudo apt-get install -y python3-wheel-whl
@@ -40,12 +40,9 @@ echo "Check aliases for python3.8"
 which python3
 which python3.8
 
-echo "Install Chrome"
-wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-sudo apt-get install -y ./google-chrome-stable_current_amd64.deb
-rm -f google-chrome-stable_current_amd64.deb
-which google-chrome
-google-chrome --version
+echo "Ensure Firefox is installed"
+which firefox
+firefox --version
 
 cd $GITHUB_WORKSPACE
 cd lib/test/src/temmpo