Add an aggregate stats to the plugin manager's view (#331)

* Add plugin download event per country model

* Add country colums to PluginVersionDownload

* Add the stats to the plugin page

* Fix dockerfile

* Add stats url to the env variables

* Fix docker-compose.yml

* Update env template file

REQUIREMENTS.txt

@@ -67,4 +67,5 @@ pyjwt==1.7.1
 djangorestframework-simplejwt==4.4 
 django-rest-auth==0.9.5
 drf-yasg
-django-matomo==0.1.6
+geoip2==4.5.0
+django-matomo==0.1.6

dockerize/.env.template

@@ -31,4 +31,7 @@ DEFAULT_PLUGINS_SITE='https://plugins.qgis.org/'
 QGISPLUGINS_ENV=debug
 
 # Ldap
-ENABLE_LDAP=True
+ENABLE_LDAP=True
+
+# Download stats URL
+METABASE_DOWNLOAD_STATS_URL='https://plugins.qgis.org/metabase/public/dashboard/<dashboard_id>'

dockerize/docker-compose.yml

@@ -46,13 +46,15 @@ services:
       - ENABLE_LDAP=${ENABLE_LDAP:-False}
       - RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq}
       - BROKER_URL=amqp://rabbitmq:5672
+      - METABASE_DOWNLOAD_STATS_URL=${METABASE_DOWNLOAD_STATS_URL:-/metabase}
       - EMAIL_BACKEND=${EMAIL_BACKEND}
       - EMAIL_HOST=${EMAIL_HOST}
       - EMAIL_PORT=${EMAIL_PORT}
       - EMAIL_USE_TLS=${EMAIL_USE_TLS}
       - EMAIL_HOST_USER=${EMAIL_HOST_USER:-automation}
       - EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD}
       - DEFAULT_PLUGINS_SITE=${DEFAULT_PLUGINS_SITE:-https://plugins.qgis.org/}
+
     volumes:
       - ../qgis-app:/home/web/django_project
       - ./docker/uwsgi.conf:/uwsgi.conf

dockerize/docker/Dockerfile

@@ -16,7 +16,15 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     libffi-dev gdal-bin\
     libjpeg-dev libpq-dev \
-    liblcms2-dev libblas-dev libatlas-base-dev
+    liblcms2-dev libblas-dev libatlas-base-dev \
+    libmaxminddb0 libmaxminddb-dev mmdb-bin
+
+# GeoIp mmdb
+RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \
+    mkdir /var/opt/maxmind && \
+    mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb
+
+ENV GEOIP_PATH=/var/opt/maxmind/
 
 RUN rm -rf /uwsgi.conf
 ADD dockerize/docker/uwsgi.conf /uwsgi.conf

dockerize/docker/REQUIREMENTS.txt

@@ -56,6 +56,7 @@ django-rest-multiple-models==2.1.3
 
 django-preferences==1.0.0
 PyWavefront==1.3.3
+geoip2==4.5.0
 django-matomo==0.1.6
 uwsgi~=2.0
-freezegun~=1.4
+freezegun~=1.4

dockerize/production/Dockerfile

@@ -17,13 +17,23 @@ RUN mkdir -p /usr/src; mkdir -p /home/web && \
             rm -rf /home/web/django_project && \
 	        ln -s /usr/src/plugins/qgis-app /home/web/django_project
 
+# Install C library for geoip2
+RUN apt-get install -y libmaxminddb0 libmaxminddb-dev mmdb-bin
+
 RUN cd /usr/src/plugins/dockerize/docker && \
 	pip install --upgrade pip && \
 	pip install -r REQUIREMENTS.txt && \
 	pip install uwsgi && \
 	rm -rf /uwsgi.conf && \
 	ln -s ${PWD}/uwsgi.conf /uwsgi.conf
 
+# GeoIp mmdb
+RUN apt-get update && apt-get install -y curl && curl -LJO https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb && \
+    mkdir /var/opt/maxmind && \
+    mv GeoLite2-City.mmdb /var/opt/maxmind/GeoLite2-City.mmdb
+
+ENV GEOIP_PATH=/var/opt/maxmind/
+
 # Open port 8080 as we will be running our uwsgi socket on that
 EXPOSE 8080
 

qgis-app/plugins/migrations/0005_auto_20231214_2317.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.25 on 2023-12-14 23:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('plugins', '0004_merge_20231122_0223'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pluginversiondownload',
+            name='country_code',
+            field=models.CharField(default='N/D', max_length=3),
+        ),
+        migrations.AddField(
+            model_name='pluginversiondownload',
+            name='country_name',
+            field=models.CharField(default='N/D', max_length=100),
+        ),
+    ]

qgis-app/plugins/migrations/0010_merge_20240517_0729.py

@@ -0,0 +1,14 @@
+# Generated by Django 4.2.13 on 2024-05-17 07:29
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('plugins', '0005_auto_20231214_2317'),
+        ('plugins', '0009_merge_20240321_0207'),
+    ]
+
+    operations = [
+    ]

qgis-app/plugins/models.py

@@ -943,6 +943,8 @@ class PluginVersionDownload(models.Model):
     download_date = models.DateField(
         default=timezone.now
     )
+    country_code = models.CharField(max_length=3, default='N/D')
+    country_name = models.CharField(max_length=100, default='N/D')
     download_count = models.IntegerField(
         default=0
     )

qgis-app/plugins/templates/plugins/plugin_detail.html

@@ -204,6 +204,7 @@ <h2>{{ object.name }}
             <li><a href="#plugin-versions" data-toggle="tab">{% trans "Versions" %}</a></li>
             {% if user.is_staff or user in object.editors %}
             <li><a href="#plugin-manage" data-toggle="tab">{% trans "Manage" %}</a></li>
+            <li><a href="#plugin-stats" data-toggle="tab">{% trans "Stats" %}</a></li>
             {% endif %}
         </ul>
 
@@ -362,6 +363,15 @@ <h2>{{ object.name }}
                 </div>
                 </form>
             </div>
+            <div class="tab-pane" id="plugin-stats">
+                <iframe
+                    src="{{stats_url}}"
+                    frameborder="0"
+                    width="1350"
+                    height="810"
+                    allowtransparency
+                ></iframe>
+            </div>
             {% endif %}
             {# end admin #}
         </div><!-- tab content -->

qgis-app/plugins/tests/test_download.py

@@ -1,11 +1,11 @@
-from django.test import TestCase, RequestFactory
+from django.test import Client, TestCase, RequestFactory
 from django.contrib.auth.models import User
 from django.utils import timezone
 from django.core.files.uploadedfile import SimpleUploadedFile
 
 from plugins.models import Plugin, PluginVersion, PluginVersionDownload
 from plugins.views import version_download
-
+from django.urls import reverse
 
 class TestVersionDownloadView(TestCase):
     def setUp(self):
@@ -50,3 +50,19 @@ def test_version_download(self):
         self.assertEqual(self.version.downloads, 1)
         self.assertEqual(self.plugin.downloads, 1)
         self.assertEqual(download_record.download_count, 1)
+
+    def test_version_download_per_country(self):
+        download_url = reverse('version_download', args=[self.plugin.package_name, self.version.version])
+        c = Client(REMOTE_ADDR='180.247.213.170')
+        response = c.get(download_url)
+
+        self.version.refresh_from_db()
+        self.plugin.refresh_from_db()
+        download_record = PluginVersionDownload.objects.get(
+            plugin_version=self.version, 
+            download_date=timezone.now().date()
+        )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(download_record.country_code == 'ID')
+        self.assertTrue(download_record.country_name == 'Indonesia')

qgis-app/plugins/utils.py

@@ -1,5 +1,6 @@
 import requests
 import re
+from django.http import HttpRequest
 
 
 def extract_version(tag):
@@ -47,3 +48,11 @@ def get_qgis_versions():
         if version not in all_versions:
             all_versions.append(version)
     return all_versions
+
+
+def parse_remote_addr(request: HttpRequest) -> str:
+    """Extract client IP from request."""
+    x_forwarded_for = request.headers.get("X-Forwarded-For", "")
+    if x_forwarded_for:
+        return x_forwarded_for.split(",")[0]
+    return request.META.get("REMOTE_ADDR", "")

qgis-app/plugins/views.py

@@ -34,6 +34,8 @@
 from plugins.forms import *
 from plugins.models import Plugin, PluginOutstandingToken, PluginVersion, PluginVersionDownload, vjust
 from plugins.validator import PLUGIN_REQUIRED_METADATA
+from django.contrib.gis.geoip2 import GeoIP2
+from plugins.utils import parse_remote_addr
 
 from rest_framework_simplejwt.token_blacklist.models import OutstandingToken
 from rest_framework_simplejwt.tokens import RefreshToken, api_settings
@@ -543,8 +545,10 @@ def get_context_data(self, **kwargs):
                         "<strong>%s</strong> metadata is missing, this metadata entry is <strong>required</strong>. Please add <strong>%s</strong> to <code>metadata.txt</code>."
                     ) % (md, md)
                     messages.error(self.request, msg, fail_silently=True)
+        stats_url = f"{settings.METABASE_DOWNLOAD_STATS_URL}?package_name={plugin.package_name}#hide_parameters=package_name"
         context.update(
             {
+                "stats_url": stats_url,
                 "rating": plugin.rating.get_rating(),
                 "votes": plugin.rating.votes,
             }
@@ -1519,8 +1523,22 @@ def version_download(request, package_name, version):
     plugin.downloads = plugin.downloads + 1
     plugin.save(keep_date=True)
 
+    remote_addr = parse_remote_addr(request)
+    g = GeoIP2()
+
+    if remote_addr:
+        try:
+            country_data = g.country(remote_addr)
+            country_code = country_data['country_code']
+            country_name = country_data['country_name']
+        except Exception as e:  # AddressNotFoundErrors:
+            country_code = 'N/D'
+            country_name = 'N/D'
+
     download_record, created = PluginVersionDownload.objects.get_or_create(
         plugin_version = version, 
+        country_code = country_code,
+        country_name = country_name,
         download_date = now().date(), 
         defaults = {'download_count': 1}
     )

qgis-app/settings.py

@@ -335,6 +335,7 @@
 CELERY_BROKER_URL = BROKER_URL
 CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 
+GEOIP_PATH='/var/opt/maxmind/'
 # Token access and refresh validity
 SIMPLE_JWT = {
     'ACCESS_TOKEN_LIFETIME': timedelta(days=15),

qgis-app/settings_docker.py

@@ -134,6 +134,11 @@
     "TEST_REQUEST_DEFAULT_FORMAT": "json",
 }
 
+GEOIP_PATH='/var/opt/maxmind/'
+METABASE_DOWNLOAD_STATS_URL = os.environ.get(
+    "METABASE_DOWNLOAD_STATS_URL", 
+    "/metabase"
+)
 CELERY_RESULT_BACKEND = 'rpc://'
 CELERY_BROKER_URL = os.environ.get('BROKER_URL', 'amqp://rabbitmq:5672')
 CELERY_BEAT_SCHEDULE = {
@@ -159,4 +164,4 @@
 MATOMO_URL="//matomo.qgis.org/"
 
 # Default primary key type
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'