From 3ecd5a4e6a6badae04d57040ee1ba55669eae2ae Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:05:21 +0300 Subject: [PATCH 1/8] Add an aggregate stats to the plugin manager's view (#331) * Add plugin download event per country model * Add country colums to PluginVersionDownload * Add the stats to the plugin page * Fix dockerfile * Add stats url to the env variables * Fix docker-compose.yml * Update env template file --- REQUIREMENTS.txt | 3 ++- dockerize/.env.template | 5 +++- dockerize/docker-compose.yml | 2 ++ dockerize/docker/Dockerfile | 10 +++++++- dockerize/docker/REQUIREMENTS.txt | 3 ++- dockerize/production/Dockerfile | 10 ++++++++ .../migrations/0005_auto_20231214_2317.py | 23 +++++++++++++++++++ .../migrations/0010_merge_20240517_0729.py | 14 +++++++++++ qgis-app/plugins/models.py | 2 ++ .../templates/plugins/plugin_detail.html | 10 ++++++++ qgis-app/plugins/tests/test_download.py | 20 ++++++++++++++-- qgis-app/plugins/utils.py | 9 ++++++++ qgis-app/plugins/views.py | 18 +++++++++++++++ qgis-app/settings.py | 1 + qgis-app/settings_docker.py | 7 +++++- 15 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 qgis-app/plugins/migrations/0005_auto_20231214_2317.py create mode 100644 qgis-app/plugins/migrations/0010_merge_20240517_0729.py diff --git a/REQUIREMENTS.txt b/REQUIREMENTS.txt index a92fb708..c8375f10 100644 --- a/REQUIREMENTS.txt +++ b/REQUIREMENTS.txt @@ -67,4 +67,5 @@ pyjwt==1.7.1 djangorestframework-simplejwt==4.4 django-rest-auth==0.9.5 drf-yasg -django-matomo==0.1.6 \ No newline at end of file +geoip2==4.5.0 +django-matomo==0.1.6 diff --git a/dockerize/.env.template b/dockerize/.env.template index 43f8d6bb..38d4ca75 100644 --- a/dockerize/.env.template +++ b/dockerize/.env.template @@ -31,4 +31,7 @@ DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/' QGISPLUGINS_ENV=debug # Ldap -ENABLE_LDAP=True \ No newline at end of file +ENABLE_LDAP=True + +# Download stats URL +METABASE_DOWNLOAD_STATS_URL='https://plugins.qgis.org/metabase/public/dashboard/' \ No newline at end of file diff --git a/dockerize/docker-compose.yml b/dockerize/docker-compose.yml index 225f9784..387fd96d 100644 --- a/dockerize/docker-compose.yml +++ b/dockerize/docker-compose.yml @@ -46,6 +46,7 @@ services: - ENABLE_LDAP=${ENABLE_LDAP:-False} - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} - BROKER_URL=amqp://rabbitmq:5672 + - METABASE_DOWNLOAD_STATS_URL=${METABASE_DOWNLOAD_STATS_URL:-/metabase} - EMAIL_BACKEND=${EMAIL_BACKEND} - EMAIL_HOST=${EMAIL_HOST} - EMAIL_PORT=${EMAIL_PORT} @@ -53,6 +54,7 @@ services: - EMAIL_HOST_USER=${EMAIL_HOST_USER:-automation} - EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD} - DEFAULT_PLUGINS_SITE=${DEFAULT_PLUGINS_SITE:-https://plugins.qgis.org/} + volumes: - ../qgis-app:/home/web/django_project - ./docker/uwsgi.conf:/uwsgi.conf diff --git a/dockerize/docker/Dockerfile b/dockerize/docker/Dockerfile index dd51c53b..b635df70 100644 --- a/dockerize/docker/Dockerfile +++ b/dockerize/docker/Dockerfile @@ -16,7 +16,15 @@ RUN apt-get update && apt-get install -y \ build-essential \ libffi-dev gdal-bin\ libjpeg-dev libpq-dev \ - liblcms2-dev libblas-dev libatlas-base-dev + liblcms2-dev libblas-dev libatlas-base-dev \ + libmaxminddb0 libmaxminddb-dev mmdb-bin + +# GeoIp mmdb +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ RUN rm -rf /uwsgi.conf ADD dockerize/docker/uwsgi.conf /uwsgi.conf diff --git a/dockerize/docker/REQUIREMENTS.txt b/dockerize/docker/REQUIREMENTS.txt index e9ff83f3..3d738ad9 100644 --- a/dockerize/docker/REQUIREMENTS.txt +++ b/dockerize/docker/REQUIREMENTS.txt @@ -56,6 +56,7 @@ django-rest-multiple-models==2.1.3 django-preferences==1.0.0 PyWavefront==1.3.3 +geoip2==4.5.0 django-matomo==0.1.6 uwsgi~=2.0 -freezegun~=1.4 \ No newline at end of file +freezegun~=1.4 diff --git a/dockerize/production/Dockerfile b/dockerize/production/Dockerfile index 9453fffc..3631f46a 100644 --- a/dockerize/production/Dockerfile +++ b/dockerize/production/Dockerfile @@ -17,6 +17,9 @@ RUN mkdir -p /usr/src; mkdir -p /home/web && \ rm -rf /home/web/django_project && \ ln -s /usr/src/plugins/qgis-app /home/web/django_project +# Install C library for geoip2 +RUN apt-get install -y libmaxminddb0 libmaxminddb-dev mmdb-bin + RUN cd /usr/src/plugins/dockerize/docker && \ pip install --upgrade pip && \ pip install -r REQUIREMENTS.txt && \ @@ -24,6 +27,13 @@ RUN cd /usr/src/plugins/dockerize/docker && \ rm -rf /uwsgi.conf && \ ln -s ${PWD}/uwsgi.conf /uwsgi.conf +# GeoIp mmdb +RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \ + mkdir /var/opt/maxmind && \ + mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb + +ENV GEOIP_PATH=/var/opt/maxmind/ + # Open port 8080 as we will be running our uwsgi socket on that EXPOSE 8080 diff --git a/qgis-app/plugins/migrations/0005_auto_20231214_2317.py b/qgis-app/plugins/migrations/0005_auto_20231214_2317.py new file mode 100644 index 00000000..8cd7045b --- /dev/null +++ b/qgis-app/plugins/migrations/0005_auto_20231214_2317.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.25 on 2023-12-14 23:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0004_merge_20231122_0223'), + ] + + operations = [ + migrations.AddField( + model_name='pluginversiondownload', + name='country_code', + field=models.CharField(default='N/D', max_length=3), + ), + migrations.AddField( + model_name='pluginversiondownload', + name='country_name', + field=models.CharField(default='N/D', max_length=100), + ), + ] diff --git a/qgis-app/plugins/migrations/0010_merge_20240517_0729.py b/qgis-app/plugins/migrations/0010_merge_20240517_0729.py new file mode 100644 index 00000000..54355764 --- /dev/null +++ b/qgis-app/plugins/migrations/0010_merge_20240517_0729.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.13 on 2024-05-17 07:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('plugins', '0005_auto_20231214_2317'), + ('plugins', '0009_merge_20240321_0207'), + ] + + operations = [ + ] diff --git a/qgis-app/plugins/models.py b/qgis-app/plugins/models.py index 72e8cc5a..90245e85 100644 --- a/qgis-app/plugins/models.py +++ b/qgis-app/plugins/models.py @@ -943,6 +943,8 @@ class PluginVersionDownload(models.Model): download_date = models.DateField( default=timezone.now ) + country_code = models.CharField(max_length=3, default='N/D') + country_name = models.CharField(max_length=100, default='N/D') download_count = models.IntegerField( default=0 ) diff --git a/qgis-app/plugins/templates/plugins/plugin_detail.html b/qgis-app/plugins/templates/plugins/plugin_detail.html index a2ffcc37..726360f9 100644 --- a/qgis-app/plugins/templates/plugins/plugin_detail.html +++ b/qgis-app/plugins/templates/plugins/plugin_detail.html @@ -204,6 +204,7 @@

{{ object.name }}
  • {% trans "Versions" %}
    • {% if user.is_staff or user in object.editors %}
  • {% trans "Manage" %}
    • +
  • {% trans "Stats" %}
    • {% endif %} @@ -362,6 +363,15 @@

    {{ object.name }} +
    + +
    {% endif %} {# end admin #} diff --git a/qgis-app/plugins/tests/test_download.py b/qgis-app/plugins/tests/test_download.py index f18f0874..34f204bd 100644 --- a/qgis-app/plugins/tests/test_download.py +++ b/qgis-app/plugins/tests/test_download.py @@ -1,11 +1,11 @@ -from django.test import TestCase, RequestFactory +from django.test import Client, TestCase, RequestFactory from django.contrib.auth.models import User from django.utils import timezone from django.core.files.uploadedfile import SimpleUploadedFile from plugins.models import Plugin, PluginVersion, PluginVersionDownload from plugins.views import version_download - +from django.urls import reverse class TestVersionDownloadView(TestCase): def setUp(self): @@ -50,3 +50,19 @@ def test_version_download(self): self.assertEqual(self.version.downloads, 1) self.assertEqual(self.plugin.downloads, 1) self.assertEqual(download_record.download_count, 1) + + def test_version_download_per_country(self): + download_url = reverse('version_download', args=[self.plugin.package_name, self.version.version]) + c = Client(REMOTE_ADDR='180.247.213.170') + response = c.get(download_url) + + self.version.refresh_from_db() + self.plugin.refresh_from_db() + download_record = PluginVersionDownload.objects.get( + plugin_version=self.version, + download_date=timezone.now().date() + ) + + self.assertEqual(response.status_code, 200) + self.assertTrue(download_record.country_code == 'ID') + self.assertTrue(download_record.country_name == 'Indonesia') \ No newline at end of file diff --git a/qgis-app/plugins/utils.py b/qgis-app/plugins/utils.py index e180e22b..521a5797 100644 --- a/qgis-app/plugins/utils.py +++ b/qgis-app/plugins/utils.py @@ -1,5 +1,6 @@ import requests import re +from django.http import HttpRequest def extract_version(tag): @@ -47,3 +48,11 @@ def get_qgis_versions(): if version not in all_versions: all_versions.append(version) return all_versions + + +def parse_remote_addr(request: HttpRequest) -> str: + """Extract client IP from request.""" + x_forwarded_for = request.headers.get("X-Forwarded-For", "") + if x_forwarded_for: + return x_forwarded_for.split(",")[0] + return request.META.get("REMOTE_ADDR", "") \ No newline at end of file diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index 20dfca1c..9f8766a4 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -34,6 +34,8 @@ from plugins.forms import * from plugins.models import Plugin, PluginOutstandingToken, PluginVersion, PluginVersionDownload, vjust from plugins.validator import PLUGIN_REQUIRED_METADATA +from django.contrib.gis.geoip2 import GeoIP2 +from plugins.utils import parse_remote_addr from rest_framework_simplejwt.token_blacklist.models import OutstandingToken from rest_framework_simplejwt.tokens import RefreshToken, api_settings @@ -543,8 +545,10 @@ def get_context_data(self, **kwargs): "%s metadata is missing, this metadata entry is required. Please add %s to metadata.txt." ) % (md, md) messages.error(self.request, msg, fail_silently=True) + stats_url = f"{settings.METABASE_DOWNLOAD_STATS_URL}?package_name={plugin.package_name}#hide_parameters=package_name" context.update( { + "stats_url": stats_url, "rating": plugin.rating.get_rating(), "votes": plugin.rating.votes, } @@ -1519,8 +1523,22 @@ def version_download(request, package_name, version): plugin.downloads = plugin.downloads + 1 plugin.save(keep_date=True) + remote_addr = parse_remote_addr(request) + g = GeoIP2() + + if remote_addr: + try: + country_data = g.country(remote_addr) + country_code = country_data['country_code'] + country_name = country_data['country_name'] + except Exception as e: # AddressNotFoundErrors: + country_code = 'N/D' + country_name = 'N/D' + download_record, created = PluginVersionDownload.objects.get_or_create( plugin_version = version, + country_code = country_code, + country_name = country_name, download_date = now().date(), defaults = {'download_count': 1} ) diff --git a/qgis-app/settings.py b/qgis-app/settings.py index 23d50828..16d3cb2d 100644 --- a/qgis-app/settings.py +++ b/qgis-app/settings.py @@ -335,6 +335,7 @@ CELERY_BROKER_URL = BROKER_URL CELERY_RESULT_BACKEND = CELERY_BROKER_URL +GEOIP_PATH='/var/opt/maxmind/' # Token access and refresh validity SIMPLE_JWT = { 'ACCESS_TOKEN_LIFETIME': timedelta(days=15), diff --git a/qgis-app/settings_docker.py b/qgis-app/settings_docker.py index c0b7be1e..9d8fcacf 100644 --- a/qgis-app/settings_docker.py +++ b/qgis-app/settings_docker.py @@ -134,6 +134,11 @@ "TEST_REQUEST_DEFAULT_FORMAT": "json", } +GEOIP_PATH='/var/opt/maxmind/' +METABASE_DOWNLOAD_STATS_URL = os.environ.get( + "METABASE_DOWNLOAD_STATS_URL", + "/metabase" +) CELERY_RESULT_BACKEND = 'rpc://' CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672') CELERY_BEAT_SCHEDULE = { @@ -159,4 +164,4 @@ MATOMO_URL="//matomo.qgis.org/" # Default primary key type -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' \ No newline at end of file +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' From 9fe81ad972cbea2f37107ea96e69cd20143f4813 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:20:01 +0300 Subject: [PATCH 2/8] Explicitly list invalid metadata keys (#380) * List all invalid keys metadata * Update test for metadata url validator * Use inital function name for _check_url_link --- qgis-app/plugins/tests/test_validator.py | 8 +- qgis-app/plugins/validator.py | 116 ++++++++++++----------- 2 files changed, 65 insertions(+), 59 deletions(-) diff --git a/qgis-app/plugins/tests/test_validator.py b/qgis-app/plugins/tests/test_validator.py index 6b1e9051..a4261d88 100644 --- a/qgis-app/plugins/tests/test_validator.py +++ b/qgis-app/plugins/tests/test_validator.py @@ -114,13 +114,13 @@ def test_invalid_metadata_web_does_not_exist(self): @mock.patch("requests.get", side_effect=requests.exceptions.SSLError()) def test_check_url_link_ssl_error(self, mock_request): - url = "http://example.com/" - self.assertIsNone(_check_url_link(url, "forbidden_url", "metadata attribute")) + urls = [{'url': "http://example.com/", 'forbidden_url': "forbidden_url", 'metadata_attr': "metadata attribute"}] + self.assertIsNone(_check_url_link(urls)) @mock.patch("requests.get", side_effect=requests.exceptions.HTTPError()) def test_check_url_link_does_not_exist(self, mock_request): - url = "http://example.com/" - self.assertIsNone(_check_url_link(url, "forbidden_url", "metadata attribute")) + urls = [{'url': "http://example.com/", 'forbidden_url': "forbidden_url", 'metadata_attr': "metadata attribute"}] + self.assertIsNone(_check_url_link(urls)) class TestValidatorForbiddenFileFolder(TestCase): diff --git a/qgis-app/plugins/validator.py b/qgis-app/plugins/validator.py index f8ebd6ad..a31c5e72 100644 --- a/qgis-app/plugins/validator.py +++ b/qgis-app/plugins/validator.py @@ -86,65 +86,68 @@ def _check_required_metadata(metadata): """ Checks if required metadata are in place, raise ValidationError if not found """ - for md in PLUGIN_REQUIRED_METADATA: - if md not in dict(metadata) or not dict(metadata)[md]: - raise ValidationError( - _( - 'Cannot find metadata %s in metadata source %s.
    For further informations about metadata, please see: metadata documentation' - ) - % (md, dict(metadata).get("metadata_source")) - ) + missing_fields = [field for field in PLUGIN_REQUIRED_METADATA if field not in [item[0] for item in metadata]] + if len(missing_fields) > 0: + missing_fields_str = ', '.join(missing_fields) + raise ValidationError( + _( + f'Cannot find metadata {missing_fields_str} in metadata source {dict(metadata).get("metadata_source")}.
    For further informations about metadata, please see: metadata documentation' + ) + ) -def _check_url_link(url: str, forbidden_url: str, metadata_attr: str) -> None: +def _check_url_link(urls): """ - Checks if the url link is valid. + Checks if all the url link is valid. """ - error_check = ValidationError( - _("Please provide valid url link for %s in metadata.") % metadata_attr + def error_check(url: str, forbidden_url: str)->bool: + # Check against forbidden_url + if url == forbidden_url: + return True + + # Check if parsed URL is valid + try: + parsed_url = urlparse(url) + return not all([parsed_url.scheme, parsed_url.netloc]) + except Exception as e: + # Log the exception or handle it as per your requirement + print(f"Error occurred: {e}") + return True + + def error_check_if_exist(url: str)->bool: + # Check if url is exist + try: + # https://stackoverflow.com/a/41950438/10268058 + # add the headers parameter to make the request appears like coming + # from browser, otherwise some websites will return 403 + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/56.0.2924.76 Safari/537.36" + } + req = requests.head(url, headers=headers) + except requests.exceptions.SSLError: + req = requests.head(url, verify=False) + except Exception: + return True + return req.status_code >= 400 + + url_error = [item for item in [url_item['metadata_attr'] for url_item in urls if error_check(url_item['url'], url_item['forbidden_url'])]] + if len(url_error) > 0: + url_error_str = ", ".join(url_error) + raise ValidationError( + _(f"Please provide valid url link for the following key(s) in the metadata source: {url_error_str}. ") ) - error_check_if_exist = ValidationError( + exist_url_error = [item for item in [url_item['metadata_attr'] for url_item in urls if error_check_if_exist(url_item['url'])]] + if len(exist_url_error) > 0: + exist_url_error_str = ", ".join(exist_url_error) + raise ValidationError( _( - "Please provide valid url link for %s in metadata. " - "This website cannot be reached." - ) - % metadata_attr + f"Please provide valid url link for the following key(s) in the metadata source: {exist_url_error_str}. " + "The website(s) cannot be reached." + ) ) - # check against forbidden_url - is_forbidden_url = url == forbidden_url - if is_forbidden_url: - raise error_check - - # check if parsed url is valid - # https://stackoverflow.com/a/38020041 - try: - parsed_url = urlparse(url) # e.g https://plugins.qgis.org/ - if not ( - all([parsed_url.scheme, parsed_url.netloc]) # e.g http - ): # e.g www.qgis.org - raise error_check - except Exception: - raise error_check - - # Check if url is exist - try: - # https://stackoverflow.com/a/41950438/10268058 - # add the headers parameter to make the request appears like coming - # from browser, otherwise some websites will return 403 - headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/56.0.2924.76 Safari/537.36" - } - req = requests.head(url, headers=headers) - except requests.exceptions.SSLError: - req = requests.head(url, verify=False) - except Exception: - raise error_check_if_exist - if req.status_code >= 400: - raise error_check_if_exist - def validator(package): """ @@ -329,11 +332,14 @@ def validator(package): ) % (min_qgs_version, ",".join(e.messages)) ) - # check url_link - _check_url_link(dict(metadata).get("tracker"), "http://bugs", "Bug tracker") - _check_url_link(dict(metadata).get("repository"), "http://repo", "Repository") - _check_url_link(dict(metadata).get("homepage"), "http://homepage", "Home page") + urls_to_check = [ + {'url': dict(metadata).get("tracker"), 'forbidden_url': "http://bugs", 'metadata_attr': "tracker"}, + {'url': dict(metadata).get("repository"), 'forbidden_url': "http://repo", 'metadata_attr': "repository"}, + {'url': dict(metadata).get("homepage"), 'forbidden_url': "http://homepage", 'metadata_attr': "homepage"}, + ] + + _check_url_link(urls_to_check) # Checks for LICENCE file presence From a791c6afe24ba335278ff10297c8dc9d093ea255 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:20:31 +0300 Subject: [PATCH 3/8] Fix responsive navbar (#385) --- qgis-app/templates/base.html | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/qgis-app/templates/base.html b/qgis-app/templates/base.html index a702adc1..c8014abb 100644 --- a/qgis-app/templates/base.html +++ b/qgis-app/templates/base.html @@ -59,7 +59,7 @@ {% get_namedmenu Navigation as menu %} -
    +
    {% endif %} -
    +
    {% block content %} {% endblock %}
    @@ -179,6 +179,32 @@
    Silver Sponsors
    + {% tracking_code %} From 0e58058cdae4c9870b824c906123d9a46e0257ac Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:21:07 +0300 Subject: [PATCH 4/8] Show modified date in the model list and detail (#381) * Show modified date in the model list and detail * Show modified date in the gallery --- qgis-app/templates/base/includes/detail_object.html | 4 ++++ qgis-app/templates/base/list.html | 2 ++ qgis-app/templates/base/list_galery.html | 6 ++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/qgis-app/templates/base/includes/detail_object.html b/qgis-app/templates/base/includes/detail_object.html index e1ae3a66..1ad9e4ab 100644 --- a/qgis-app/templates/base/includes/detail_object.html +++ b/qgis-app/templates/base/includes/detail_object.html @@ -7,8 +7,12 @@
    {{ object_detail.description|md_to_html }}
    Uploaded Date
    {{ object_detail.upload_date|date:"d F Y" }}
    +{% if object_detail.review_set.last.review_date and object_detail.approved%}
    Approved Date
    {{ object_detail.review_set.last.review_date|date:"d F Y"}}
    +{% endif %} +
    Modified Date
    +
    {{ object_detail.modified_date|date:"d F Y" }}
    Creator
    {{ object_detail.get_creator_name|title }}
    {% if is_qlr %} diff --git a/qgis-app/templates/base/list.html b/qgis-app/templates/base/list.html index 6c5292b1..529ac62d 100644 --- a/qgis-app/templates/base/list.html +++ b/qgis-app/templates/base/list.html @@ -79,6 +79,7 @@

    {% if title %}{{title}}{% else %}{% trans "All" %} {{ resource_name }}s{% en {% anchor_sort_arrow 'Creator' 'creator' order_by queries %} {% anchor_sort_arrow 'Upload Date' 'upload_date' order_by queries %} + {% anchor_sort_arrow 'Modified Date' 'modified_date' order_by queries %} @@ -105,6 +106,7 @@

    {% if title %}{{title}}{% else %}{% trans "All" %} {{ resource_name }}s{% en {{ object.download_count }} {{ object.get_creator_name|title }} {{ object.upload_date|date:"d F Y" }} + {{ object.modified_date|date:"d F Y" }} {% if user.is_staff or user == style.creator %}   diff --git a/qgis-app/templates/base/list_galery.html b/qgis-app/templates/base/list_galery.html index 9c2f552f..5f230973 100644 --- a/qgis-app/templates/base/list_galery.html +++ b/qgis-app/templates/base/list_galery.html @@ -28,12 +28,12 @@ display: flex; align-items: center; justify-content: center; - height: 420px; + height: 380px; background-color: white; } img.image-demo { height: auto; - max-height: 100%; + max-height: 380px; width: auto; max-width: 100%; } @@ -78,6 +78,7 @@

    {% if title %}{{title}}{% else %}{% trans "All" %} {{ resource_name }}s{% en + {% if object_list.first.style_type %}{% endif %} @@ -106,6 +107,7 @@

    {{ object.get_creator_name|title }} | {{ object.upload_date|date:"d F Y" }} | {{ object.download_count }} downloads +

    Modified Date: {{ object.modified_date|date:"d F Y" }}

    From 8073926a2b3d0e74bb11e0c5b01a3f6753447f44 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:21:46 +0300 Subject: [PATCH 5/8] Show the number of results returned in search (#383) * Show the number of results returned in search * Fix typo --- qgis-app/templates/search/search.html | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/qgis-app/templates/search/search.html b/qgis-app/templates/search/search.html index 01f9f7e4..44c51b89 100644 --- a/qgis-app/templates/search/search.html +++ b/qgis-app/templates/search/search.html @@ -46,7 +46,7 @@

    Search

    {% if query %} -

    Search results

    +

    Search results for "{{query}}" ({{paginator.count}} items found)

    {% for result in page.object_list %}

    @@ -58,9 +58,27 @@

    Search results

    {% if page.has_previous or page.has_next %} {% endif %} {% else %} From 5c13bb1f14cd65d509d5663b64a88092a328fb2a Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:26:17 +0300 Subject: [PATCH 6/8] Add delay notification in the approval email (#384) * Add delay notification in the approval email * Improve delay notification message to 'up to 15 minutes' --- qgis-app/plugins/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qgis-app/plugins/views.py b/qgis-app/plugins/views.py index 9f8766a4..f6987f9c 100644 --- a/qgis-app/plugins/views.py +++ b/qgis-app/plugins/views.py @@ -1368,7 +1368,11 @@ def version_approve(request, package_name, version): return HttpResponseRedirect(version.get_absolute_url()) version.approved = True version.save() - msg = _('The plugin version "%s" is now approved' % version) + msg = _( + "The plugin version '%s' is now approved. " + "Please note that there may be a delay of up to 15 minutes " + "between the approval of the plugin and its actual availability in the XML." + ) % version messages.success(request, msg, fail_silently=True) plugin_approve_notify(version.plugin, msg, request.user) try: From f22c66e497c693c22825137e7144815f1797fd2e Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:27:26 +0300 Subject: [PATCH 7/8] Show the path for forbidden dir in child folder (#387) * Show the path for forbidden dir in child folder * Improve the error message for forbidden dir in the root of the archive --- qgis-app/plugins/tests/test_validator.py | 29 ++++++++++++++++++++---- qgis-app/plugins/validator.py | 12 ++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/qgis-app/plugins/tests/test_validator.py b/qgis-app/plugins/tests/test_validator.py index a4261d88..87c5a884 100644 --- a/qgis-app/plugins/tests/test_validator.py +++ b/qgis-app/plugins/tests/test_validator.py @@ -154,7 +154,10 @@ def test_zipfile_with_MACOSX(self, mock_namelist): mock_namelist.return_value = ["__MACOSX/"] with self.assertRaisesMessage( Exception, - ("For security reasons, zip file cannot contain " "'__MACOSX' directory"), + ( + "For security reasons, zip file cannot contain '__MACOSX' directory. " + "However, there is one present at the root of the archive." + ), ): validator(self.package) @@ -164,8 +167,20 @@ def test_zipfile_with_pycache(self, mock_namelist): with self.assertRaisesMessage( Exception, ( - "For security reasons, zip file cannot contain " - "'__pycache__' directory" + "For security reasons, zip file cannot contain '__pycache__' directory. " + "However, there is one present at the root of the archive." + ), + ): + validator(self.package) + + @mock.patch("zipfile.ZipFile.namelist") + def test_zipfile_with_pycache_in_children(self, mock_namelist): + mock_namelist.return_value = ["path/to/__pycache__/"] + with self.assertRaisesMessage( + Exception, + ( + "For security reasons, zip file cannot contain '__pycache__' directory. " + "However, it has been found at 'path/to/__pycache__/' ." ), ): validator(self.package) @@ -175,7 +190,10 @@ def test_zipfile_with_git(self, mock_namelist): mock_namelist.return_value = [".git"] with self.assertRaisesMessage( Exception, - ("For security reasons, zip file cannot contain " "'.git' directory"), + ( + "For security reasons, zip file cannot contain '.git' directory. " + "However, there is one present at the root of the archive." + ), ): validator(self.package) @@ -188,7 +206,8 @@ def test_zipfile_with_gitignore(self, mock_namelist): exception = cm.exception self.assertNotEqual( exception.message, - "For security reasons, zip file cannot contain '.git' directory", + "For security reasons, zip file cannot contain '.git' directory. ", + "However, there is one present at the root of the archive." ) diff --git a/qgis-app/plugins/validator.py b/qgis-app/plugins/validator.py index a31c5e72..1f075f2e 100644 --- a/qgis-app/plugins/validator.py +++ b/qgis-app/plugins/validator.py @@ -192,11 +192,19 @@ def validator(package): _("For security reasons, zip file cannot contain .pyc file") ) for forbidden_dir in ["__MACOSX", ".git", "__pycache__"]: - if forbidden_dir in zname.split("/"): + dir_name_list = zname.split("/") + if forbidden_dir in dir_name_list: + if forbidden_dir == dir_name_list[0]: + raise ValidationError( + _( + "For security reasons, zip file " + "cannot contain '%s' directory. However, there is one present at the root of the archive." % (forbidden_dir,) + ) + ) raise ValidationError( _( "For security reasons, zip file " - "cannot contain '%s' directory" % (forbidden_dir,) + "cannot contain '%s' directory. However, it has been found at '%s' ." % (forbidden_dir, zname) ) ) bad_file = zip.testzip() From 93ec9ee93c3c15bd1253557c14b7eb6fe9f187d3 Mon Sep 17 00:00:00 2001 From: Lova ANDRIARIMALALA <43842786+Xpirix@users.noreply.github.com> Date: Wed, 22 May 2024 17:27:44 +0300 Subject: [PATCH 8/8] Update and document makefile (#404) * Update and document makefile * Fix typo * Fix typo in make commands --- INSTALL.md | 2 + dockerize/Makefile | 169 +++++++++++++++++++++++++++-------------- dockerize/README.md | 181 ++++++++++++++++++++++++++++++++++++++++++++ readme.md | 2 +- 4 files changed, 296 insertions(+), 58 deletions(-) create mode 100644 dockerize/README.md diff --git a/INSTALL.md b/INSTALL.md index 3a2f7421..7cbcfe23 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -43,6 +43,8 @@ $ make rebuild_index ``` This command ensures that the search index accurately reflects the current state of the database, resolving the presence of 'None' in the search results. Automatic synchronization is currently managed in settings.py: `HAYSTACK_SIGNAL_PROCESSOR = "haystack.signals.RealtimeSignalProcessor"`. +For more information about make commands, please see the full docs [here](./dockerize/README.md). + --- ### Setup git-hooks and local linting diff --git a/dockerize/Makefile b/dockerize/Makefile index 43039c82..c58599c3 100644 --- a/dockerize/Makefile +++ b/dockerize/Makefile @@ -15,12 +15,6 @@ build: @echo "------------------------------------------------------------------" @docker compose -p $(PROJECT_ID) build -build-dev: - @echo - @echo "------------------------------------------------------------------" - @echo "Building in development mode only" - @echo "------------------------------------------------------------------" - @docker compose -p $(PROJECT_ID) build devweb db: @echo @@ -57,27 +51,6 @@ certbot: web @echo "------------------------------------------------------------------" @docker compose -p $(PROJECT_ID) up -d certbot -devweb-test: db - @echo - @echo "------------------------------------------------------------------" - @echo "Running in TESTING mode" - @echo "------------------------------------------------------------------" - @docker compose up --no-deps -d devweb - -devweb: db - @echo - @echo "------------------------------------------------------------------" - @echo "Running in DEVELOPMENT mode" - @echo "------------------------------------------------------------------" - @docker compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat - -devweb-runserver: devweb - @echo - @echo "------------------------------------------------------------------" - @echo "Running in DEVELOPMENT mode" - @echo "------------------------------------------------------------------" - @docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 - migrate: @echo @echo "------------------------------------------------------------------" @@ -105,26 +78,27 @@ collectstatic: @echo "Collecting static in production mode" @echo "------------------------------------------------------------------" @docker compose -p $(PROJECT_ID) run uwsgi python manage.py collectstatic --noinput - #We need to run collect static in the same context as the running - # uwsgi container it seems so I use docker exec here - # no -it flag so we can run over remote shell - # @docker exec $(PROJECT_ID)-web python manage.py collectstatic --noinput -reload: +start: @echo @echo "------------------------------------------------------------------" - @echo "Reload django project in production mode" + @echo "Starting a specific container(s) in production mode. Use web if you want to start all." @echo "------------------------------------------------------------------" - # no -it flag so we can run over remote shell - @docker exec $(PROJECT_ID)-web uwsgi --reload /tmp/django.pid + @docker compose -p $(PROJECT_ID) up -d $(c) +restart: + @echo + @echo "------------------------------------------------------------------" + @echo "Restarting all or a specific container(s) in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) restart $(c) kill: @echo @echo "------------------------------------------------------------------" - @echo "Killing in production mode" + @echo "Killing all or a specific container(s) in production mode" @echo "------------------------------------------------------------------" - @docker compose -p $(PROJECT_ID) kill + @docker compose -p $(PROJECT_ID) kill $(c) rm: rm-only @@ -135,20 +109,6 @@ rm-only: kill @echo "------------------------------------------------------------------" @docker compose -p $(PROJECT_ID) rm -maillogs: - @echo - @echo "------------------------------------------------------------------" - @echo "Showing smtp logs in production mode" - @echo "------------------------------------------------------------------" - @docker compose exec smtp tail -f /var/log/mail.log - -mailerrorlogs: - @echo - @echo "------------------------------------------------------------------" - @echo "Showing smtp error logs in production mode" - @echo "------------------------------------------------------------------" - @docker compose exec smtp tail -f /var/log/mail.err - dbrestore: @echo @echo "------------------------------------------------------------------" @@ -176,16 +136,111 @@ create-test-db: @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -c 'create database test_db;'" @docker compose -p $(PROJECT_ID) exec db su - postgres -c "psql -d test_db -c 'create extension postgis;'" -dbseed: +rebuild_index: @echo @echo "------------------------------------------------------------------" - @echo "Seed db with JSON data from /fixtures/*.json" + @echo "Rebuild search index in PRODUCTION mode" @echo "------------------------------------------------------------------" - @docker compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' -rebuild_index: +uwsgi-shell: @echo @echo "------------------------------------------------------------------" - @echo "Rebuild search index in PRODUCTION mode" + @echo "Shelling into the uwsgi container(s)" @echo "------------------------------------------------------------------" - @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'python manage.py rebuild_index' + @docker compose -p $(PROJECT_ID) exec uwsgi bash + +uwsgi-reload: + @echo + @echo "------------------------------------------------------------------" + @echo "Reload django project in production mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'uwsgi uwsgi --reload /tmp/django.pid' + +uwsgi-errors: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing errors in the uwsgi container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'tail -f /var/log/uwsgi-errors.log' + +uwsgi-logs: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing access logs in uwsgi container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec uwsgi bash -c 'tail -f /var/log/uwsgi-requests.log' + +web-shell: + @echo + @echo "------------------------------------------------------------------" + @echo "Shelling into the NGINX/WEB container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec web bash + +web-logs: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing logs in NGINX/WEB container(s)" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) logs -f web + +logs: + @echo + @echo "------------------------------------------------------------------" + @echo "Tailing all logs or a specific container" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) logs -f $(c) + +shell: + @echo + @echo "------------------------------------------------------------------" + @echo "Shelling into a specific container" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec $(c) bash + +exec: + @echo + @echo "------------------------------------------------------------------" + @echo "Execute a specific docker command" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) $(c) + +# ---------------------------------------------------------------------------- +# D E V E L O P M E N T C O M M A N D S +# ---------------------------------------------------------------------------- + +build-dev: + @echo + @echo "------------------------------------------------------------------" + @echo "Building in development mode only" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) build devweb + +devweb-test: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in TESTING mode" + @echo "------------------------------------------------------------------" + @docker compose up --no-deps -d devweb + +devweb: db + @echo + @echo "------------------------------------------------------------------" + @echo "Running in DEVELOPMENT mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) up --no-deps -d devweb rabbitmq worker beat + +devweb-runserver: devweb + @echo + @echo "------------------------------------------------------------------" + @echo "Running in DEVELOPMENT mode" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec devweb python manage.py runserver 0.0.0.0:8080 + +dbseed: + @echo + @echo "------------------------------------------------------------------" + @echo "Seed db with JSON data from /fixtures/*.json" + @echo "------------------------------------------------------------------" + @docker compose -p $(PROJECT_ID) exec devweb bash -c 'python manage.py loaddata fixtures/*.json' \ No newline at end of file diff --git a/dockerize/README.md b/dockerize/README.md new file mode 100644 index 00000000..aa59cea4 --- /dev/null +++ b/dockerize/README.md @@ -0,0 +1,181 @@ +# Docker compose commands Documentation + +## Overview +This doc is designed for managing a Docker-based project with the ID `qgis-plugins`. It includes various commands for building, running, and maintaining both production and development environments. Below is a detailed description of each command available in the Makefile. + +## Commands + +### Production Commands + +- **default**: Alias for the build command. +```sh +make +``` + +- **run**: Builds the project and runs the web, migrate, and collectstatic commands. +```sh +make run +``` + +- **build**: Builds Docker images for both production and development environments. +```sh +make build +``` + +- **db**: Starts the database container in production mode. +```sh +make db +``` + +- **metabase**: Starts the Metabase container after ensuring the database is running. +```sh +make metabase +``` + +- **web**: Starts the web container and scales the `uwsgi` service to 2 instances. +```sh +make web +``` + +- **dbbackups**: Starts the database backups container. +```sh +make dbbackups +``` + +- **certbot**: Starts the Certbot container for managing SSL certificates. +```sh +make certbot +``` + +- **migrate**: Runs database migrations, with the `auth` app being migrated first. +```sh +make migrate +``` + +- **update-migrations**: Creates new migration files based on changes in models. +```sh +make update-migrations +``` + +- **collectstatic**: Collects static files for the Django application. +```sh +make collectstatic +``` + +- **start**: Starts a specific container or all containers. Specify the container with the `c` variable. +```sh +make start c=container_name +``` + +- **restart**: Restarts a specific container or all containers. Specify the container with the `c` variable. +```sh +make restart c=container_name +``` + +- **kill**: Stops a specific container or all containers. Specify the container with the `c` variable. +```sh +make kill c=container_name +``` + +- **rm**: Removes all containers after stopping them. +```sh +make rm +``` + +- **rm-only:** Removes all containers without stopping them first. +```sh +make rm-only +``` + +- **dbrestore:** Restores the database from a backup file. +```sh +make dbrestore +``` + +- **wait-db:** Waits for the database to be ready. +```sh +make wait-db +``` + +- **create-test-db:** Creates a test database with PostGIS extension. +```sh +make create-test-db +``` + +- **rebuild_index:** Rebuilds the search index for the Django application. +```sh +make rebuild_index +``` + +- **uwsgi-shell:** Opens a shell in the `uwsgi` container. +```sh +make uwsgi-shell +``` + +- **uwsgi-reload:** Reloads the Django project in the `uwsgi` container. +```sh +make uwsgi-reload +``` + +- **uwsgi-errors:** Tails the error logs in the `uwsgi` container. +```sh +make uwsgi-errors +``` + +- **uwsgi-logs:** Tails the requests logs in the `uwsgi` container. +```sh +make uwsgi-logs +``` + +- **web-shell:** Opens a shell in the NGINX/web container. +```sh +make web-shell +``` + +- **web-logs:** Tails the logs in the NGINX/web container. +```sh +make web-logs +``` + +- **logs:** Tails logs for a specific container or all containers. Specify the container with the `c` variable. +```sh +make logs c=container_name +``` + +- **shell:** Opens a shell in a specific container. Specify the container with the `c` variable. +```sh +make shell c=container_name +``` + +- **exec:** Executes a specific Docker command. Specify the command with the `c` variable. +```sh +make exec c="command" +``` + +### Development Commands + +- **build-dev:** Builds Docker images for the development environment. +```sh +make build-dev +``` + +- **devweb-test:** Starts the `devweb` container for testing, ensuring the database is running. +```sh +make devweb-test +``` + +- **devweb:** Starts the `devweb` container for development, along with RabbitMQ, worker, and beat containers. +```sh +make devweb +``` + +- **devweb-runserver:** Runs the Django development server inside the `devweb` container. +```sh +make devweb-runserver +``` + +- **dbseed:** Seeds the database with initial data from JSON files in the `fixtures` directory. +```sh +make dbseed +``` + diff --git a/readme.md b/readme.md index ec2ea04c..56f730e2 100644 --- a/readme.md +++ b/readme.md @@ -33,7 +33,7 @@ docker and rancher. ## Token based authentication -Users have the ability to generate a Simple JWT token by providing their credentials, which can then be utilized to access endpoints requiring authentication. +Users can generate a Simple JWT token by providing their credentials, which can then be utilized to access endpoints requiring authentication. Users can create specific tokens for a plugin at `https://plugins.qgis.org//tokens/`.