Fix Redis queues, Defect-Dojo integration, Telegram notifications and…

… filters

.gitignore

@@ -132,9 +132,9 @@ dmypy.json
 .DS_Store
 .vscode/
 .scannerwork/
-./reports/
-./wordlists/
-./logs/
+reports/
+wordlists/
+logs/
 /static/
 /src/backend/tests/home/
 

src/backend/authentications/models.py

@@ -7,7 +7,6 @@
 from framework.models import BaseEncrypted, BaseInput
 from security.input_validator import Regex, Validator
 from target_ports.models import TargetPort
-from targets.models import Target
 
 # Create your models here.
 

src/backend/executions/queues.py

@@ -22,9 +22,7 @@
 
 
 class ExecutionsQueue(BaseQueue):
-    def __init__(self) -> None:
-        super().__init__("executions-queue")
-        self.findings_queue = FindingsQueue()
+    name = "executions-queue"
 
     def enqueue(
         self,
@@ -37,8 +35,8 @@ def enqueue(
         dependencies: List[Job] = [],
         at_front: bool = False,
     ) -> Job:
-        job = self.queue.enqueue(
-            self.consume.__func__,
+        job = self._get_queue().enqueue(
+            self.consume,
             execution=execution,
             findings=findings,
             target_ports=target_ports,
@@ -63,9 +61,9 @@ def enqueue(
         execution.save(update_fields=["rq_job_id"])
         return job
 
+    @staticmethod
     @job("executions-queue")
     def consume(
-        self,
         execution: Execution,
         findings: List[Finding],
         target_ports: List[TargetPort],
@@ -84,7 +82,7 @@ def consume(
                 input_vulnerabilities,
                 input_technologies,
                 wordlists,
-            ) = self._get_findings_from_dependencies(
+            ) = ExecutionsQueue._get_findings_from_dependencies(
                 executor,
                 target_ports,
                 input_vulnerabilities,
@@ -98,11 +96,11 @@ def consume(
             executor, execution.output_plain
         )
         parser.parse()
-        self.findings_queue.enqueue(execution, parser.findings)
+        FindingsQueue().enqueue(execution, parser.findings)
         return execution, parser.findings
 
+    @staticmethod
     def _get_findings_from_dependencies(
-        self,
         executor: BaseExecutor,
         target_ports: List[TargetPort],
         input_vulnerabilities: List[InputVulnerability],
@@ -111,15 +109,16 @@ def _get_findings_from_dependencies(
         current_job: Job,
     ) -> Dict[int, List[BaseInput]]:
         findings = []
+        queue = ExecutionsQueue._get_queue()
         for dependency_id in current_job._dependency_ids:
-            dependency = self.queue.fetch_job(dependency_id)
+            dependency = queue.fetch_job(dependency_id)
             if dependency and dependency.result:
                 findings.extend(dependency.result[1])
         if not findings:
             return findings
         executions = [
             e
-            for e in self._calculate_executions(
+            for e in ExecutionsQueue._calculate_executions(
                 executor.execution.configuration.tool,
                 findings,
                 target_ports,
@@ -141,7 +140,7 @@ def _get_findings_from_dependencies(
                 configuration=executor.execution.configuration,
                 group=executor.execution.group,
             )
-            job = self.enqueue(
+            job = queue.enqueue(
                 new_execution,
                 execution.get(0, []),
                 execution.get(1, []),
@@ -153,15 +152,16 @@ def _get_findings_from_dependencies(
             )
             new_jobs.append(job.id)
         if new_jobs:
-            registry = DeferredJobRegistry(queue=self.queue)
+            instance = ExecutionsQueue()
+            registry = DeferredJobRegistry(queue=queue)
             for pending_job_id in registry.get_job_ids():
-                pending_job = self.queue.fetch_job(pending_job_id)
+                pending_job = queue.fetch_job(pending_job_id)
                 if pending_job and current_job.id in pending_job._dependency_ids:
                     dependencies = pending_job._dependency_ids
                     meta = pending_job.get_meta()
-                    self.cancel_job(pending_job_id)
-                    self.delete_job(pending_job_id)
-                    self.enqueue(
+                    instance.cancel_job(pending_job_id)
+                    instance.delete_job(pending_job_id)
+                    instance.enqueue(
                         meta["execution"],
                         [],
                         meta["target_ports"],

src/backend/findings/framework/models.py

@@ -1,7 +1,6 @@
-from typing import Any, Dict, List
+from typing import Any, Dict
 
-from django.core.exceptions import NON_FIELD_ERRORS
-from django.db import connection, models
+from django.db import models
 from executions.models import Execution
 from findings.enums import TriageStatus
 from framework.models import BaseInput

src/backend/findings/models.py

@@ -199,14 +199,34 @@ def parse(self, accumulated: Dict[str, Any] = {}) -> Dict[str, Any]:
             InputKeyword.ENDPOINT.name.lower(): path,
         }
 
-    def defect_dojo(self) -> Dict[str, Any]:
+    def defect_dojo_endpoint(self, target: Target) -> Dict[str, Any]:
         return {
             "protocol": self.port.service if self.port else None,
-            "host": self.port.host.address if self.port and self.port.host else None,
+            "host": self.port.host.address
+            if self.port and self.port.host
+            else target.target,
             "port": self.port.port if self.port else None,
             "path": self.path,
         }
 
+    def defect_dojo(self) -> Dict[str, Any]:
+        description = f"Path: {self.path}\nType: {self.type}"
+        for key, value in [("Status", self.status), ("Info", self.extra_info)]:
+            if value:
+                description = f"{description}\n{key}: {value}"
+        if self.port:
+            description = f"Port: {self.port.port}\n{description}"
+            if self.port.host:
+                description = f"Host: {self.port.host.address}\n{description}"
+        return {
+            "title": "Path discovered",
+            "description": description,
+            "severity": Severity.INFO,
+            "date": self.last_seen.strftime(
+                DefectDojoSettings.objects.first().date_format
+            ),
+        }
+
     def __str__(self) -> str:
         return f"{f'{self.port.__str__()} - ' if self.port else ''}{self.path}"
 

src/backend/findings/queues.py

@@ -15,8 +15,7 @@
 
 
 class FindingsQueue(BaseQueue):
-    def __init__(self) -> None:
-        super().__init__("findings-queue")
+    name = "findings-queue"
 
     def enqueue(self, execution: Execution, findings: List[Finding]) -> Job:
         job = super().enqueue(execution=execution, findings=findings)
@@ -25,7 +24,9 @@ def enqueue(self, execution: Execution, findings: List[Finding]) -> Job:
         )
         return job
 
+    @staticmethod
     @job("findings-queue")
-    def consume(self, execution: Execution, findings: List[Finding]) -> List[Finding]:
-        for platform in [NvdNist, DefectDojo, SMTP, Telegram]:
-            platform.process_findings(execution, findings)
+    def consume(execution: Execution, findings: List[Finding]) -> List[Finding]:
+        if findings:
+            for platform in [NvdNist, DefectDojo, SMTP, Telegram]:
+                platform().process_findings(execution, findings)

src/backend/framework/models.py

@@ -152,7 +152,7 @@ def _compare_filter(
             comparison(filter, value) if not negative else not comparison(filter, value)
         )
 
-    def filter(self, input: Any, target: Any = None) -> bool:
+    def filter(self, argument_input: Any, target: Any = None) -> bool:
         """Check if this instance is valid based on input filter.
 
         Args:
@@ -161,34 +161,46 @@ def filter(self, input: Any, target: Any = None) -> bool:
         Returns:
             bool: Indicate if this instance match the input filter or not
         """
-        if not input.filter:
+        if not argument_input.filter:
             return True
-        for filter_value in input.filter.split(" or "):
-            negative = filter_value.startswith("!")
-            if negative:
-                filter_value = filter_value[1:]
-            for filter in self.filters:
-                field_value = getattr(self, filter.field)
-                if filter.processor:
-                    field_value = filter.processor(field_value)
-                try:
-                    if (
-                        issubclass(filter.type, models.TextChoices)
-                        and self._compare_filter(
-                            filter.type[filter_value.upper()], field_value, negative
-                        )
-                    ) or (
-                        hasattr(self, filter_value)
-                        and self._compare_filter(
-                            filter.type(getattr(self, filter_value)),
-                            field_value,
-                            negative,
-                            filter.contains,
-                        )
-                    ):
+        filter_value = argument_input.filter
+        for split, or_condition in [(" or ", True), (" and ", False)]:
+            if split not in filter_value and or_condition:
+                continue
+            for match_value in filter_value.split(split):
+                negative = match_value.startswith("!")
+                if negative:
+                    match_value = match_value[1:]
+                for filter in self.filters:
+                    and_condition = False
+                    field_value = getattr(self, filter.field)
+                    if filter.processor:
+                        field_value = filter.processor(field_value)
+                    try:
+                        if (
+                            issubclass(filter.type, models.TextChoices)
+                            and self._compare_filter(
+                                filter.type[match_value.upper()], field_value, negative
+                            )
+                        ) or (
+                            hasattr(self, match_value)
+                            and self._compare_filter(
+                                filter.type(getattr(self, match_value)),
+                                field_value,
+                                negative,
+                                filter.contains,
+                            )
+                        ):
+                            if or_condition:
+                                return True
+                            else:
+                                and_condition = True
+                        elif not or_condition:
+                            return False
+                    except (ValueError, KeyError) as ex:
+                        continue
+                    if not or_condition and and_condition:
                         return True
-                except (ValueError, KeyError):
-                    pass
         return False
 
     def parse(self, accumulated: Dict[str, Any] = {}) -> Dict[str, Any]:

src/backend/framework/queues.py

@@ -8,6 +8,7 @@
 from input_types.models import InputType
 from parameters.models import InputTechnology, InputVulnerability
 from rq.job import Job
+from rq.queue import Queue
 from target_ports.models import TargetPort
 from tools.models import Input, Tool
 from wordlists.models import Wordlist
@@ -16,30 +17,33 @@
 
 
 class BaseQueue:
-    def __init__(self, name: str) -> None:
-        self.name = name
-        self.queue = django_rq.get_queue(name)
+    name = ""
+
+    def _get_queue(self) -> Queue:
+        return django_rq.get_queue(self.name)
 
     def cancel_job(self, job_id: str) -> Job:
-        job = self.queue.fetch_job(job_id)
+        job = self._get_queue().fetch_job(job_id)
         if job:
             logger.info(f"[{self.name}] Job {job_id} has been cancelled")
             job.cancel()
 
     def delete_job(self, job_id: str) -> Job:
-        job = self.queue.fetch_job(job_id)
+        job = self._get_queue().fetch_job(job_id)
         if job:
             logger.info(f"[{self.name}] Job {job_id} has been deleted")
             job.delete()
 
     def enqueue(self, **kwargs: Any) -> Job:
-        return self.queue.enqueue(self.consume.__func__, **kwargs)
+        return self._get_queue().enqueue(self.consume, **kwargs)
 
-    def consume(self, **kwargs: Any) -> Any:
+    @staticmethod
+    def consume(**kwargs: Any) -> Any:
         pass
 
+    @staticmethod
     def _get_findings_by_type(
-        self, findings: List[Finding]
+        findings: List[Finding],
     ) -> Dict[InputType, List[Finding]]:
         findings_by_type = {}
         for finding in findings:
@@ -55,8 +59,8 @@ def _get_findings_by_type(
             )
         )
 
+    @staticmethod
     def _calculate_executions(
-        self,
         tool: Tool,
         findings: List[Finding],
         target_ports: List[TargetPort],
@@ -66,7 +70,7 @@ def _calculate_executions(
     ) -> List[Dict[int, List[BaseInput]]]:
         executions = [{0: []}]
         input_types_used = set()
-        findings_by_type = self._get_findings_by_type(findings)
+        findings_by_type = BaseQueue._get_findings_by_type(findings)
         for index, input_type, source in [
             (0, t, list(f)) for t, f in (findings_by_type or {}).items() if f
         ] + [