Python and dependency update bump (#175)

* update python to 3.11

* add landsat l2 for surface reflectance. ordering enhanced vegetation index sr

* adding landsat9 and updtaing masks from renamed tif files of collection 2

* update urls to re_path to prep for django 4.2

* remove logs on espa wrapper

* remove logs in adapter

* remove logs and new file name masks

* revert adapter to pull sr instead of aq_refl

* fix tests. cannot patch within a patch. update espa api to v0 AND PILLOW ANTIALIAS TO LANCZOS for pILLOW > 10

Dockerfile

@@ -1,16 +1,16 @@
-FROM python:3.7-stretch
+FROM python:3.11.5-bookworm
 
 LABEL maintainer="contact@zooniverse.org"
 
-RUN apt-get update \
+RUN apt-get upgrade && apt-get update \
   && apt-get install -y --no-install-recommends \
     postgresql-client \
     gdal-bin \
     libtiff5-dev \
     libgdal-dev \
     python3-gdal \
     binutils \
-    netcat \
+    ncat \
     libproj-dev \
     libgeoip1 \
     postgis \

Pipfile

@@ -55,7 +55,7 @@ urllib3 = "*"
 vine = "*"
 wcwidth = "*"
 zipp = "*"
-libtiff = "*"
+pylibtiff = "*"
 
 [dev-packages]
 pytest = "*"
@@ -67,7 +67,7 @@ pytest-pythonpath = "*"
 django-extensions = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3.11"
 
 [scripts]
 console = "python manage.py shell_plus"

tests/adapters/usgs/test_espa_wrapper.py

@@ -9,8 +9,8 @@
 
 class TestEspaWrapper:
     def test_api_url(self):
-        assert EspaWrapper.api_url('foo') == 'https://espa.cr.usgs.gov/api/v1/foo'
-        assert EspaWrapper.api_url('') == 'https://espa.cr.usgs.gov/api/v1/'
+        assert EspaWrapper.api_url('foo') == 'https://espa.cr.usgs.gov/api/v0/foo'
+        assert EspaWrapper.api_url('') == 'https://espa.cr.usgs.gov/api/v0/'
 
     def test_espa_credentials(self):
         assert EspaWrapper.espa_credentials(username='u', password='p') == HTTPBasicAuth('u', 'p')

tests/adapters/usgs/test_usgs_tasks.py

@@ -14,29 +14,26 @@ def test_wait_for_scene_already_done(mockStatus, mockGetScene):
 @patch('theia.adapters.usgs.EspaWrapper.download_urls', autospec=True)
 @patch('theia.adapters.usgs.tasks.wait_for_scene.apply_async', autospec=True)
 def test_wait_for_scene_pending(mockWait, mockUrls, mockStatus, mockGetScene):
-    with patch.object(mockGetScene.return_value, 'save', autospec=True) as mockSave:
-        mockGetScene.return_value.status=0
-        mockGetScene.return_value.scene_order_id='order id'
+    mockGetScene.return_value.status=0
+    mockGetScene.return_value.scene_order_id='order id'
 
-        theia.adapters.usgs.tasks.wait_for_scene(3)
+    theia.adapters.usgs.tasks.wait_for_scene(3)
 
-        mockGetScene.assert_called_once_with(pk=3)
-        mockStatus.assert_called_once_with('order id')
-        mockSave.assert_called_once()
-        mockWait.assert_called_once()
+    mockGetScene.assert_called_once_with(pk=3)
+    mockStatus.assert_called_once_with('order id')
+    mockGetScene.return_value.save.assert_called_once()
+    mockWait.assert_called_once()
 
 @patch('theia.api.models.RequestedScene.objects.get', autospec=True)
 @patch('theia.adapters.usgs.EspaWrapper.order_status', return_value='complete')
 @patch('theia.adapters.usgs.EspaWrapper.download_urls')
 @patch('theia.api.models.JobBundleManager.from_requested_scene', autospec=True)
 def test_wait_for_scene_ready(mockConstruct, mockUrls, mockStatus, mockGetScene):
-    with patch.object(mockGetScene.return_value, 'save', autospec=True) as mockSave:
-        mockGetScene.return_value.status=0
-        mockGetScene.return_value.scene_order_id='order id'
+    mockGetScene.return_value.status=0
+    mockGetScene.return_value.scene_order_id='order id'
+    theia.adapters.usgs.tasks.wait_for_scene(3)
 
-        theia.adapters.usgs.tasks.wait_for_scene(3)
-
-        mockGetScene.assert_called_once_with(pk=3)
-        mockStatus.assert_called_once_with('order id')
-        mockSave.assert_called_once()
-        mockConstruct.assert_called_once()
+    mockGetScene.assert_called_once_with(pk=3)
+    mockStatus.assert_called_once_with('order id')
+    mockGetScene.return_value.save.assert_called()
+    mockConstruct.assert_called_once()

tests/api/models/test_imagery_request.py

@@ -23,9 +23,9 @@ def test___str__(self):
     def test_post_create(self):
         '''it enqueues a job after being created'''
         with patch.object(theia.tasks, 'locate_scenes', autospec=True) as mockLocate:
-            with patch.object(mockLocate.return_value, 'delay', autospec=True) as mockDelay:
-                ImageryRequest.post_save(None, self.test_request, False)
-                mockDelay.assert_not_called()
+            ImageryRequest.post_save(None, self.test_request, False)
+            mockLocate.delay.assert_not_called()
+
+            ImageryRequest.post_save(None, self.test_request, True)
+            mockLocate.delay.assert_called_once_with(self.test_request.id)
 
-                ImageryRequest.post_save(None, self.test_request, True)
-                mockDelay.assert_called_once_with(self.test_request.id)

tests/api/models/test_job_bundle.py

@@ -34,13 +34,12 @@ def test___str__(self):
         bundle = JobBundle(scene_entity_id='entity')
         assert(bundle.__str__()=='[JobBundle entity on no host]')
 
-    def test_post_create(self):
+    @patch('theia.api.models.job_bundle.process_bundle')
+    def test_post_create(self, mock_process_bundle):
         '''it enqueues a job after being created'''
         bundle = JobBundle(id=7)
-        with patch.object(theia.tasks, 'process_bundle', autospec=True) as mockProcess:
-            with patch.object(mockProcess.return_value, 'delay', autospec=True) as mockDelay:
-                JobBundle.post_save(None, bundle, False)
-                mockDelay.assert_not_called()
+        JobBundle.post_save(None, bundle, False)
+        mock_process_bundle.delay.assert_not_called()
 
-                JobBundle.post_save(None, bundle, True)
-                mockDelay.assert_called_once_with(bundle.id)
+        JobBundle.post_save(None, bundle, True)
+        mock_process_bundle.delay.assert_called_once_with(bundle.id)

tests/operations/image_operations/test_resize_image.py

@@ -15,7 +15,7 @@ def test_apply(self, mock_open):
         operation.apply(["literal filename"])
 
         mock_open.assert_called_once_with('literal filename')
-        mock_open.return_value.thumbnail.assert_called_once_with((10, 20), Image.ANTIALIAS)
+        mock_open.return_value.thumbnail.assert_called_once_with((10, 20), Image.LANCZOS)
         mock_open.return_value.save.assert_called_once()
         first_method_call = mock_open.return_value.save.call_args_list[0]
         args = first_method_call[0]

tests/utils/test_file_utils.py

@@ -32,9 +32,8 @@ def test_locate_latest_version_exists(self, *args):
     @patch('os.mkdir')
     def test_untar(self, mockMkDir, mockIsDir):
         with patch.object(tarfile, 'open', autospec=True) as mockOpen:
-            with patch.object(mockOpen.return_value, 'extractall', autospec=True) as mockExtract:
-                FileUtils.untar('some/zip/path', 'some/dir/path')
-                mockOpen.assert_called_once_with('some/zip/path', 'r')
-                mockIsDir.assert_called_once_with('some/dir/path')
-                mockMkDir.assert_called_once_with('some/dir/path')
-                mockExtract.assert_called_once_with('some/dir/path')
+            FileUtils.untar('some/zip/path', 'some/dir/path')
+            mockOpen.assert_called_once_with('some/zip/path', 'r')
+            mockIsDir.assert_called_once_with('some/dir/path')
+            mockMkDir.assert_called_once_with('some/dir/path')
+            mockOpen.return_value.__enter__.return_value.extractall.assert_called_once_with('some/dir/path')

theia/adapters/usgs/adapter.py

@@ -44,6 +44,24 @@ class Adapter:
             'swir-1': 'band6',
             'swir-2': 'band7',
         },
+        'LANDSAT_OT_C2_L1': {
+            'coastal_aerosol': 'band1',
+            'blue': 'band2',
+            'green': 'band3',
+            'red': 'band4',
+            'nir': 'band5',
+            'swir-1': 'band6',
+            'swir-2': 'band7',
+        },
+        'LANDSAT_OT_C2_L2': {
+            'coastal_aerosol': 'band1',
+            'blue': 'band2',
+            'green': 'band3',
+            'red': 'band4',
+            'nir': 'band5',
+            'swir-1': 'band6',
+            'swir-2': 'band7',
+        },
     }
 
     def __init__(self):

theia/adapters/usgs/espa_wrapper.py

@@ -9,7 +9,7 @@ class EspaWrapper:
     @classmethod
     def api_url(cls, path):
         # return urljoin('https://demo1580318.mockable.io/', path)
-        return urljoin('https://espa.cr.usgs.gov/api/v1/', path)
+        return urljoin('https://espa.cr.usgs.gov/api/v0/', path)
 
     @classmethod
     def list_orders(cls):
@@ -35,13 +35,14 @@ def order_status(cls, order_id):
 
     @classmethod
     def order_one(cls, collection, scene_id, product_type):
-        return cls.espa_post('order', {
+        res =  cls.espa_post('order', {
             collection: {
                 'inputs': [scene_id],
                 'products': [product_type]
             },
             'format': 'gtiff'
-        })['orderid']
+        })
+        return res['orderid']
 
     @classmethod
     def order_all(cls, scene_id, product_type):

theia/api/models/pipeline_stage.py

@@ -1,5 +1,5 @@
 from django.db import models as models
-from django.contrib.postgres.fields import ArrayField, JSONField
+from django.contrib.postgres.fields import ArrayField
 from .pipeline import Pipeline
 
 
@@ -8,7 +8,7 @@ class PipelineStage(models.Model):
     output_format = models.CharField(max_length=8, null=True, blank=True)
     operation = models.CharField(max_length=64, null=False)
     select_images = ArrayField(models.CharField(max_length=64, null=False), null=True)
-    config = JSONField(blank=True)
+    config = models.JSONField(blank=True)
     pipeline = models.ForeignKey(Pipeline, related_name='pipeline_stages', on_delete=models.CASCADE)
 
     class Meta:

theia/celery.py

@@ -1,11 +1,8 @@
 from __future__ import absolute_import, unicode_literals
 import os
 from celery import Celery
-from libtiff import libtiff_ctypes
 import django
 
-libtiff_ctypes.suppress_warnings()
-
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'theia.settings')
 django.setup()

theia/operations/floating_forest_operations/floating_forest.py

@@ -14,6 +14,9 @@
 # https://docs.python.org/3/library/csv.html
 LANDSAT = {'red': 'band5', 'green': 'band2', 'blue': 'band3', 'infrared': 'band4'}
 LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'}
+# https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/LSDS-1822_Landsat8-9-OLI-TIRS-C2-L1-DFCB-v6.pdf
+# https://www.usgs.gov/faqs/what-are-band-designations-landsat-satellites
+LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' }
 
 
 ff_config = type('Config', (object,), {
@@ -66,6 +69,7 @@ def parse_options(filenames):
 
 LANDSAT = {'red': 'band5', 'green': 'band2', 'blue': 'band3', 'infrared': 'band4'}
 LANDSAT8 = {'red': 'band6', 'green': 'band3', 'blue': 'band4', 'infrared': 'band5'}
+LANDSAT9 = {'red': 'band4', 'green': 'band3', 'blue': 'band2', 'infrared': 'band5' }
 
 def usage():
     print("""
@@ -208,7 +212,6 @@ def run_ff(filenames, output_directory, manifest_directory):
 
     accepts = []
     rejects = []
-
     [ff_config.width, ff_config.height] = get_dimensions(ff_config.INPUT_FILE)
     ff_config.SCRATCH_PATH = manifest_directory
     ff_config.REJECTED_TILES_PATH = path.join(ff_config.SCRATCH_PATH + "/rejected")
@@ -222,6 +225,8 @@ def run_ff(filenames, output_directory, manifest_directory):
     ff_config.METADATA = metadata
     if ff_config.METADATA['spacecraft'] == 'LANDSAT_8':
         ff_config.SATELLITE = LANDSAT8
+    if ff_config.METADATA['spacecraft'] == 'LANDSAT_9':
+        ff_config.SATELLITE = LANDSAT9
 
     ff_config.RED_CHANNEL = path.join(
         ff_config.SCENE_DIR, ff_config.SCENE_NAME + "_sr_" + ff_config.SATELLITE['red'] + ".tif")
@@ -780,7 +785,7 @@ def compute_tile_coords(row, col, width, height, config):
     top = scene_top + ((row * config.GRID_SIZE) / config.height) * scene_span_y
     right = left + (width / config.width) * scene_span_x
     bottom = top + (height / config.height) * scene_span_y
-    
+
     if int(config.METADATA['#utm_zone']) < 0:
         top = top - 10000000
         bottom = bottom - 10000000

theia/operations/image_operations/resize_image.py

@@ -18,5 +18,5 @@ def apply(self, filenames):
             )
 
             im = Image.open(filename)
-            im.thumbnail(dimensions, Image.ANTIALIAS)
+            im.thumbnail(dimensions, Image.LANCZOS)
             im.save(self.output_directory + "/" + new_filename)

theia/urls.py

@@ -14,8 +14,7 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path, include
-from django.conf.urls import url
+from django.urls import path, include, re_path
 from rest_framework import routers
 from theia.api import views
 
@@ -35,9 +34,9 @@
     path('api/', include(router.urls)),
     path('admin/', admin.site.urls),
 
-    url(r'^$', views.home, name='home'),
-    url(r'^api-auth/', include('rest_framework.urls')),
-    url(r'oauth/', include('social_django.urls', namespace='social')),
-    url(r'^login$', auth_views.LoginView.as_view(), name='login'),
-    url(r'^logout$', auth_views.LogoutView.as_view(), name='logout'),
+    re_path(r'^$', views.home, name='home'),
+    re_path(r'^api-auth/', include('rest_framework.urls')),
+    re_path(r'oauth/', include('social_django.urls', namespace='social')),
+    re_path(r'^login$', auth_views.LoginView.as_view(), name='login'),
+    re_path(r'^logout$', auth_views.LogoutView.as_view(), name='logout'),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)