Advanced submission evaluation (docker)

.github/workflows/test.yml

@@ -19,6 +19,7 @@ jobs:
           pip install -r ./backend/requirements.txt
       - name: Running Django tests
         run: |
+          source /home/selab2/hosting/.env.test
           sh ./backend/runtests.sh
 
   frontend-test:

Dockerfile

@@ -10,7 +10,7 @@ ENV PYTHONUNBUFFERED 1
 
 # install psycopg2 dependencies
 RUN apk update \
-    && apk add postgresql-dev gcc python3-dev musl-dev
+    && apk add postgresql-dev gcc python3-dev musl-dev docker-cli
 
 # install dependencies
 RUN pip install --upgrade pip

Dockerfile.prod

@@ -47,7 +47,7 @@ RUN mkdir $APP_HOME/uploads
 WORKDIR $APP_HOME
 
 # install dependencies
-RUN apk update && apk add libpq
+RUN apk update && apk add libpq docker-cli
 COPY --from=builder /usr/src/app/wheels /wheels
 COPY --from=builder /usr/src/app/requirements.txt .
 RUN pip install --no-cache /wheels/*

Makefile

@@ -14,6 +14,23 @@ superuser:
 mockdata:
 	docker exec -it pigeonhole-backend python manage.py runscript mockdata
 
+registry:
+	docker build examples/advanced-evaluation/always-succeed -t test-always-succeed
+	docker tag test-always-succeed localhost:5000/test-always-succeed
+	docker push localhost:5000/test-always-succeed
+	docker build examples/advanced-evaluation/helloworld -t test-helloworld
+	docker tag test-helloworld localhost:5000/test-helloworld
+	docker push localhost:5000/test-helloworld
+	docker build examples/advanced-evaluation/fibonacci-python -t fibonacci-python
+	docker tag fibonacci-python localhost:5000/fibonacci-python
+	docker push localhost:5000/fibonacci-python
+
+evaltest:
+	docker exec -it pigeonhole-backend python manage.py runscript eval_test
+
+submit:
+	docker exec -it pigeonhole-backend python manage.py runscript submit
+
 reset:
 	docker image prune -af
 	docker system prune
@@ -39,3 +56,14 @@ silentcomponenttest:
 resetdb:
 	docker exec pigeonhole-backend python manage.py flush --noinput
 	docker exec -it pigeonhole-backend python manage.py runscript mockdata
+
+prodregistry:
+	docker build examples/advanced-evaluation/always-succeed -t test-always-succeed
+	docker tag test-always-succeed sel2-1.ugent.be:2002/test-always-succeed
+	docker push sel2-1.ugent.be:2002/test-always-succeed
+	docker build examples/advanced-evaluation/helloworld -t test-helloworld
+	docker tag test-helloworld sel2-1.ugent.be:2002/test-helloworld
+	docker push sel2-1.ugent.be:2002/test-helloworld
+	docker build examples/advanced-evaluation/fibonacci-python -t fibonacci-python
+	docker tag fibonacci-python sel2-1.ugent.be:2002/fibonacci-python
+	docker push sel2-1.ugent.be:2002/fibonacci-python

backend/entrypoint.prod.sh

@@ -17,5 +17,6 @@ python manage.py runscript push_site
 
 python manage.py collectstatic --noinput
 
+docker login $REGISTRY_URL -u $REGISTRY_USER -p $REGISTRY_PASSWORD
 
 exec "$@"

backend/entrypoint.sh

@@ -19,4 +19,6 @@ python manage.py runscript push_site
 
 python manage.py createsuperuser --noinput --email $DJANGO_SUPERUSER_EMAIL
 
+docker login $REGISTRY_URL -u $REGISTRY_USER -p $REGISTRY_PASSWORD
+
 exec "$@"

backend/pigeonhole/apps/projects/admin.py

@@ -63,6 +63,14 @@ class ProjectAdmin(admin.ModelAdmin):
                 )
             }
         ),
+        (
+            'test docker image',
+            {
+                'fields': (
+                    'test_docker_image',
+                )
+            }
+        ),
     )
 
     raw_id_fields = (

backend/pigeonhole/apps/projects/migrations/0002_project_test_dockerfile.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.13 on 2024-05-19 11:35
+
+import backend.pigeonhole.apps.projects.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('projects', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='test_dockerfile',
+            field=models.FileField(blank=True, null=True,
+                                   upload_to=backend.pigeonhole.apps.projects.models.get_upload_to),
+        ),
+    ]

backend/pigeonhole/apps/projects/migrations/0003_alter_project_test_dockerfile_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.13 on 2024-05-19 16:38
+
+import backend.pigeonhole.apps.projects.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('projects', '0002_project_test_dockerfile'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='test_dockerfile',
+            field=models.FileField(blank=True, null=True,
+                                   upload_to=backend.pigeonhole.apps.projects.models.dockerfile_path),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='test_files',
+            field=models.FileField(blank=True, null=True,
+                                   upload_to=backend.pigeonhole.apps.projects.models.testfile_path),
+        ),
+    ]

backend/pigeonhole/apps/projects/migrations/0003_project_test_entrypoint.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.11 on 2024-05-19 14:55
+
+import backend.pigeonhole.apps.projects.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('projects', '0002_project_test_dockerfile'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='test_entrypoint',
+            field=models.FileField(blank=True, null=True,
+                                   upload_to=backend.pigeonhole.apps.projects.models.get_upload_to),
+        ),
+    ]

backend/pigeonhole/apps/projects/migrations/0004_merge_20240519_1640.py

@@ -0,0 +1,14 @@
+# Generated by Django 4.2.13 on 2024-05-19 16:40
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0003_alter_project_test_dockerfile_and_more'),
+        ('projects', '0003_project_test_entrypoint'),
+    ]
+
+    operations = [
+    ]

backend/pigeonhole/apps/projects/migrations/0005_remove_project_test_dockerfile_and_more.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.2.13 on 2024-05-20 18:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0004_merge_20240519_1640'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='project',
+            name='test_dockerfile',
+        ),
+        migrations.RemoveField(
+            model_name='project',
+            name='test_entrypoint',
+        ),
+        migrations.RemoveField(
+            model_name='project',
+            name='test_files',
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='test_docker_image',
+            field=models.CharField(blank=True, max_length=1024, null=True),
+        ),
+    ]

backend/pigeonhole/apps/projects/models.py

@@ -4,6 +4,21 @@
 from backend.pigeonhole.apps.courses.models import Course
 
 
+def dockerfile_path(self, _):
+    if not self.pk:
+        nextpk = Project.objects.order_by('-project_id').first().project_id + 1
+        self.id = self.pk = nextpk
+    return f'courses/id_{str(self.course_id.course_id)}/project/id_{str(self.project_id)}/DOCKERFILE'
+
+
+def testfile_path(self, filename):
+    if not self.pk:
+        nextpk = Project.objects.order_by('-project_id').first().project_id + 1
+        self.id = self.pk = nextpk
+    return f'courses/id_{str(self.course_id.course_id)}/project/id_{str(self.project_id)}/{filename}'
+
+
+# legacy code
 def get_upload_to(self, filename):
     return 'projects/' + str(self.project_id) + '/' + filename
 
@@ -21,14 +36,17 @@ class Project(models.Model):
     group_size = models.IntegerField(default=1)
     file_structure = models.TextField(blank=True, null=True)
     conditions = models.TextField(blank=True, null=True)
-    test_files = models.FileField(blank=True, null=True, upload_to=get_upload_to)
+
+    # test_files = models.FileField(blank=True, null=True, upload_to=testfile_path)
+    # test_dockerfile = models.FileField(blank=True, null=True, upload_to=dockerfile_path)
+    test_docker_image = models.CharField(max_length=1024, blank=True, null=True)
 
 
 class ProjectSerializer(serializers.ModelSerializer):
     class Meta:
         model = Project
         fields = ["project_id", "course_id", "name", "description", "deadline", "visible", "number_of_groups",
-                  "group_size", "max_score", "file_structure", "conditions", "test_files"]
+                  "group_size", "max_score", "file_structure", "conditions", "test_docker_image"]
 
 
 class Test(models.Model):

backend/pigeonhole/apps/submissions/admin.py

@@ -36,7 +36,23 @@ class SubmissionAdmin(admin.ModelAdmin):
             }
         ),
         (
-            'Files',
+            'eval output',
+            {
+                'fields': (
+                    'eval_output',
+                )
+            }
+        ),
+        (
+            'eval result',
+            {
+                'fields': (
+                    'eval_result',
+                )
+            }
+        ),
+        (
+            'file urls',
             {
                 'fields': (
                     'file_urls',

backend/pigeonhole/apps/submissions/migrations/0005_submissions_eval_result.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-05-05 02:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('submissions', '0004_remove_submissions_file_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='submissions',
+            name='eval_result',
+            field=models.BooleanField(default=False),
+        ),
+    ]

backend/pigeonhole/apps/submissions/migrations/0006_submissions_eval_output.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.13 on 2024-05-19 11:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('submissions', '0005_submissions_eval_result'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='submissions',
+            name='eval_output',
+            field=models.TextField(null=True),
+        ),
+    ]

backend/pigeonhole/apps/submissions/migrations/0007_alter_submissions_eval_result.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-05-23 02:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('submissions', '0006_submissions_eval_output'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='submissions',
+            name='eval_result',
+            field=models.BooleanField(default=None, null=True),
+        ),
+    ]

backend/pigeonhole/apps/submissions/migrations/0008_merge_20240523_0823.py

@@ -0,0 +1,14 @@
+# Generated by Django 4.2.13 on 2024-05-23 08:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('submissions', '0005_submissions_feedback_simple_test_and_more'),
+        ('submissions', '0007_alter_submissions_eval_result'),
+    ]
+
+    operations = [
+    ]