From b57a04a6a2bb2149bfe1c9915037f43e2a1409d6 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Wed, 11 Sep 2024 16:24:07 +0200 Subject: [PATCH 01/18] Use aiosmtpd instead of smtpd --- .gitignore | 2 +- .../files/external_mail_handler.py | 81 +++++++++---------- requirements.txt | 1 + setup.py | 5 +- 4 files changed, 46 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index 6089dc3..e082b1f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ __pycache__/ ### Cache ### .cache/* -# DS_STORE
 +# DS_STORE .DS_STORE ### PyCharm ### diff --git a/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py b/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py index c3892a0..4e1c3f6 100644 --- a/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py +++ b/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py @@ -20,9 +20,10 @@ import email from email.mime.text import MIMEText -from smtpd import SMTPServer +from aiosmtpd.controller import Controller +from aiosmtpd.smtp import Envelope +import asyncio from smtplib import SMTP -import asyncore import time @@ -43,39 +44,31 @@ def setup_logging(): format="%(asctime)s" + gmt_offset_string + " %(name)s %(levelname)s %(message)s", ) +class Server: + def __init__(self, server_ip=None, server_port=None): + self.server_port: None | int = server_port + self.server_ip: None | str = server_ip logger = logging.getLogger(__name__) - - -class Responder: - def __init__(self): - self.smtp_out = Server("172.18.0.2", 25) - self.smtp_in = Server("0.0.0.0", 25) - self.smtp_server = None - - def run(self): - logger.info("Starting Mail Responder listening at " + - str(self.smtp_in.server_ip) + - ":" + str(self.smtp_in.server_port) - ) - logger.info("Sending responses to " + - str(self.smtp_out.server_ip) + - ":" + str(self.smtp_out.server_port) - ) - self.init_smtp_server() - asyncore.loop() - - def init_smtp_server(self): - self.smtp_server = SMTPServer((self.smtp_in.server_ip, self.smtp_in.server_port), None) - self.smtp_server.process_message = self.process_message - - def process_message(self, peer, mailfrom, rcpttos, data): - mail = mime_string_to_text_mail(data) +smtp_out = Server("127.0.0.1", 1025) +smtp_in = Server("0.0.0.0", 1025) + + +class CustomHandler: + async def handle_DATA(self, server, session, envelope: Envelope): + # Contents are available raw bytes, thus requiring decoding + self.process(envelope.content.decode("utf8"), server) + # if error_occurred: + # return '500 Could not process your message' + return '250 OK' + + def process(self, content, server): + mail = mime_string_to_text_mail(content) logger.info("Received mail from " + str(mail.sender) + " addressed to " + str(mail.receiver)) self.swap_sender_receiver(mail) self.modify_text(mail) - self.send_mail(mail) - + self.send_mail(mail, smtp_instance=server) + @staticmethod def swap_sender_receiver(mail): tmp = mail.sender @@ -86,13 +79,13 @@ def swap_sender_receiver(mail): def modify_text(mail): mail.text = "You sent me the following text:\n" + mail.text - def send_mail(self, mail): - con = SMTP() - try: - con.connect(host=self.smtp_out.server_ip, port=self.smtp_out.server_port) - con.send_message(mail.to_mime_text()) - finally: - con.quit() + @staticmethod + def send_mail(mail, smtp_instance): + ip = smtp_out.server_ip + port = smtp_out.server_port + if ip and port: + smtp_instance.connect(host=ip, port=port) + smtp_instance.send_message(mail.to_mime_text()) def mime_string_to_text_mail(mime_string): @@ -131,11 +124,17 @@ def to_mime_text(self): class Server: def __init__(self, server_ip=None, server_port=None): - self.server_port = server_port - self.server_ip = server_ip + self.server_port: None | int = server_port + self.server_ip: None | str = server_ip if __name__ == "__main__": setup_logging() - responder = Responder() - responder.run() + logger.info("Starting Mail Responder listening at " + + str(smtp_in.server_ip) + ":" + str(smtp_in.server_port)) + logger.info("Sending responses to " + + str(smtp_out.server_ip) + ":" + str(smtp_out.server_port)) + controller = Controller(CustomHandler(), hostname=smtp_in.server_ip, port=smtp_in.server_port) + while True: + # don't care about stopping + pass diff --git a/requirements.txt b/requirements.txt index a98bcc3..51f1a3c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ pywinrm==0.4.1 selenium==4.3.0 tox==3.7.0 veryprettytable==0.8.1 +asyncio==3.4.3 diff --git a/setup.py b/setup.py index 124ef70..bc29613 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,10 @@ "paramiko", "pyvmomi", "veryprettytable", - "selenium" + "selenium", + "aiosmtpd", + "pytest-asyncio", + "pytest-mock" ], entry_points={ 'console_scripts': [ From 6a81a079d61c226b530a2d1a1b8c6995bb564732 Mon Sep 17 00:00:00 2001 From: Maspital Date: Thu, 12 Sep 2024 17:56:25 +0200 Subject: [PATCH 02/18] Update tests after switching from smptd to aiosmtpd --- .../files/external_mail_handler.py | 83 +++++++++------- .../files/test_external_mail_handler.py | 99 +++++++++---------- requirements.txt | 4 +- setup.py | 3 +- tox.ini | 1 + 5 files changed, 97 insertions(+), 93 deletions(-) diff --git a/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py b/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py index 4e1c3f6..3137f44 100644 --- a/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py +++ b/provisioning/ansible/roles/external_mail_handler/files/external_mail_handler.py @@ -20,10 +20,8 @@ import email from email.mime.text import MIMEText -from aiosmtpd.controller import Controller -from aiosmtpd.smtp import Envelope -import asyncio from smtplib import SMTP +from aiosmtpd.controller import Controller import time @@ -44,31 +42,25 @@ def setup_logging(): format="%(asctime)s" + gmt_offset_string + " %(name)s %(levelname)s %(message)s", ) -class Server: - def __init__(self, server_ip=None, server_port=None): - self.server_port: None | int = server_port - self.server_ip: None | str = server_ip logger = logging.getLogger(__name__) -smtp_out = Server("127.0.0.1", 1025) -smtp_in = Server("0.0.0.0", 1025) class CustomHandler: - async def handle_DATA(self, server, session, envelope: Envelope): - # Contents are available raw bytes, thus requiring decoding - self.process(envelope.content.decode("utf8"), server) - # if error_occurred: - # return '500 Could not process your message' - return '250 OK' - - def process(self, content, server): - mail = mime_string_to_text_mail(content) + def __init__(self, smtp_out): + self.smtp_out = smtp_out + + async def handle_DATA(self, server, session, envelope): + # peer, mail_from, mail_to, rcpt_tos and data are now all encapsulated in 'envelope' + # keep in mind that envelope.data contains raw bytes which first have to be decoded + mail = mime_string_to_text_mail(envelope.data.decode("utf-8")) logger.info("Received mail from " + str(mail.sender) + " addressed to " + str(mail.receiver)) self.swap_sender_receiver(mail) self.modify_text(mail) - self.send_mail(mail, smtp_instance=server) - + self.send_mail(mail) + # A return message is mandatory + return '250 OK' + @staticmethod def swap_sender_receiver(mail): tmp = mail.sender @@ -79,13 +71,36 @@ def swap_sender_receiver(mail): def modify_text(mail): mail.text = "You sent me the following text:\n" + mail.text - @staticmethod - def send_mail(mail, smtp_instance): - ip = smtp_out.server_ip - port = smtp_out.server_port - if ip and port: - smtp_instance.connect(host=ip, port=port) - smtp_instance.send_message(mail.to_mime_text()) + def send_mail(self, mail): + con = SMTP() + try: + con.connect(host=self.smtp_out.server_ip, port=self.smtp_out.server_port) + con.send_message(mail.to_mime_text()) + finally: + con.quit() + + +class Responder: + def __init__(self): + self.smtp_out = Server("172.18.0.2", 25) + self.smtp_in = Server("0.0.0.0", 25) + self.controller: Controller|None = None + + def run(self): + logger.info("Starting Mail Responder listening at " + + str(self.smtp_in.server_ip) + + ":" + str(self.smtp_in.server_port)) + logger.info("Sending responses to " + + str(self.smtp_out.server_ip) + + ":" + str(self.smtp_out.server_port)) + self.init_controller() + self.controller.start() # detaches from current thread + input('SMTP server running. Press Return to stop server and exit.') + self.controller.stop() + + def init_controller(self): + handler = CustomHandler(self.smtp_out) + self.controller = Controller(handler, hostname=self.smtp_in.server_ip, port=self.smtp_in.server_port) def mime_string_to_text_mail(mime_string): @@ -124,17 +139,11 @@ def to_mime_text(self): class Server: def __init__(self, server_ip=None, server_port=None): - self.server_port: None | int = server_port - self.server_ip: None | str = server_ip + self.server_port = server_port + self.server_ip = server_ip if __name__ == "__main__": setup_logging() - logger.info("Starting Mail Responder listening at " + - str(smtp_in.server_ip) + ":" + str(smtp_in.server_port)) - logger.info("Sending responses to " + - str(smtp_out.server_ip) + ":" + str(smtp_out.server_port)) - controller = Controller(CustomHandler(), hostname=smtp_in.server_ip, port=smtp_in.server_port) - while True: - # don't care about stopping - pass + responder = Responder() + responder.run() \ No newline at end of file diff --git a/provisioning/ansible/roles/external_mail_handler/files/test_external_mail_handler.py b/provisioning/ansible/roles/external_mail_handler/files/test_external_mail_handler.py index 0fc51e4..4225d5f 100644 --- a/provisioning/ansible/roles/external_mail_handler/files/test_external_mail_handler.py +++ b/provisioning/ansible/roles/external_mail_handler/files/test_external_mail_handler.py @@ -21,15 +21,21 @@ from unittest.mock import Mock, patch import pytest +from aiosmtpd.controller import Controller -from external_mail_handler import TextMail, mime_string_to_text_mail, Responder +from external_mail_handler import TextMail, mime_string_to_text_mail, Responder, CustomHandler, Server @pytest.fixture() -def r(): +def responder(): return Responder() +@pytest.fixture() +def handler(): + return CustomHandler(Server("172.18.0.2", 25)) + + @pytest.fixture() def patch_smtp(request): mock = MockSMTP() @@ -40,31 +46,6 @@ def patch_smtp(request): return mock -@pytest.fixture() -def patch_smtp_server(request): - mock = MockSMTPServer() - - def modify_mock(local_address, remote_address): - mock.local_address = local_address - mock.remote_address = remote_address - return mock - - mock_smtp_server_creator = Mock(side_effect=modify_mock) - p = patch(Responder.__module__ + ".SMTPServer", mock_smtp_server_creator) - p.start() - request.addfinalizer(p.stop) - return mock - - -@pytest.fixture() -def patch_asyncore_loop(request): - mock = Mock() - p = patch(Responder.__module__ + ".asyncore.loop", mock) - p.start() - request.addfinalizer(p.stop) - return mock - - class MockSMTP: def __init__(self): self.quit = Mock() @@ -79,43 +60,53 @@ def __init__(self, local_address=None, remote_address=None): self.process_message = None -class TestResponder: - def test_swap_sender_receiver(self, r: Responder): +class TestCustomHandler: + def test_swap_sender_receiver(self, handler: CustomHandler): mail = TextMail(sender="some@sender", receiver="some@receiver") - r.swap_sender_receiver(mail) + handler.swap_sender_receiver(mail) assert mail.sender == "some@receiver" assert mail.receiver == "some@sender" - def test_send_mail(self, r: Responder, patch_smtp: MockSMTP): + def test_send_mail(self, handler: CustomHandler, patch_smtp: MockSMTP): mail = TextMail(sender="some@domain", receiver="some@other", text="Some text") - r.send_mail(mail) + handler.send_mail(mail) assert patch_smtp.connect.called assert patch_smtp.send_message.called assert patch_smtp.quit.called - def test_handler(self, r: Responder): - r.swap_sender_receiver = Mock() - r.send_mail = Mock() + @pytest.mark.asyncio + async def test_handler(self, handler: CustomHandler): + handler.swap_sender_receiver = Mock() + handler.send_mail = Mock() + envelope = Mock() mail = TextMail(sender="some@domain", receiver="other@domain", text="Hallo Welt") - peer = "127.0.0.1" - mailfrom = mail.sender - rcpttos = [mail.receiver] - data = mail.to_mime_text().as_string() - r.process_message(peer, mailfrom, rcpttos, data) - assert r.swap_sender_receiver.called - assert r.send_mail.called - - def test_init_smtp_server(self, r: Responder, patch_smtp_server): - r.init_smtp_server() - assert r.smtp_server.local_address == ("0.0.0.0", 25) - assert r.smtp_server.remote_address is None - assert r.smtp_server.process_message == r.process_message - - def test_run(self, r: Responder, patch_asyncore_loop): - r.init_smtp_server = Mock() - r.run() - assert r.init_smtp_server.called - assert patch_asyncore_loop.called + envelope.peer = "127.0.0.1" + envelope.mail_to = mail.sender + envelope.rcpt_to = [mail.receiver] + envelope.data = mail.to_mime_text().as_string().encode('utf-8') + await handler.handle_DATA(None, None, envelope) + assert handler.swap_sender_receiver.called + assert handler.send_mail.called + + def test_init_controller(self, responder: Responder): + responder.init_controller() + + assert isinstance(responder.controller, Controller) + assert isinstance(responder.controller.handler, CustomHandler) + assert responder.controller.hostname == responder.smtp_in.server_ip + assert responder.controller.port == responder.smtp_in.server_port + + def test_run(self, responder: Responder): + responder.init_controller = Mock() + responder.controller = Mock() # Mock the controller object itself + responder.controller.start = Mock() + responder.controller.stop = Mock() + + with patch("builtins.input", return_value=""): + responder.run() + assert responder.init_controller.called + assert responder.controller.start.called + assert responder.controller.stop.called @pytest.fixture() diff --git a/requirements.txt b/requirements.txt index 51f1a3c..236c171 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,12 @@ ansible==5.1.0 colorama==0.4.1 paramiko==2.11.0 -pytest==5.2.0 +pytest==8.3.3 pyvmomi==6.7.1.2018.12 pywinrm==0.4.1 selenium==4.3.0 tox==3.7.0 veryprettytable==0.8.1 asyncio==3.4.3 +setuptools~=74.1.2 +aiosmtpd~=1.4.6 \ No newline at end of file diff --git a/setup.py b/setup.py index bc29613..305519b 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,8 @@ "selenium", "aiosmtpd", "pytest-asyncio", - "pytest-mock" + "pytest-mock", + "pytest-print" ], entry_points={ 'console_scripts': [ diff --git a/tox.ini b/tox.ini index 60407e9..93a3e7b 100644 --- a/tox.ini +++ b/tox.ini @@ -16,3 +16,4 @@ markers = systest: mark a system test, i.e., virtual machines will be run. unstable: mark a system test that does not (yet) reliably work. longtest: mark a unit test that takes longer than 10 seconds. +asyncio_default_fixture_loop_scope = function \ No newline at end of file From 8776599a9bf890a6206c3fcb290fa777a7af1059 Mon Sep 17 00:00:00 2001 From: Maspital Date: Thu, 12 Sep 2024 17:59:18 +0200 Subject: [PATCH 03/18] Update sqlmap_command test to use fixed terminal width --- src/attacks/tests/test_attack_sqlmap.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/attacks/tests/test_attack_sqlmap.py b/src/attacks/tests/test_attack_sqlmap.py index 1ca0a99..1f5d7ba 100644 --- a/src/attacks/tests/test_attack_sqlmap.py +++ b/src/attacks/tests/test_attack_sqlmap.py @@ -16,7 +16,7 @@ # along with SOCBED. If not, see . -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest @@ -31,10 +31,15 @@ def attack(): class TestSQLMapAttack: + + def test_sqlmap_command(self, attack: SQLMapAttack): - assert attack._sqlmap_command() == ( - "export COLUMNS=80; echo \"http://172.18.0.2/dvwa/vulnerabilities/sqli/?id=&Submit=Submit\" | " - "sqlmap --purge --batch --dump") + with patch("attacks.attack_sqlmap.get_terminal_size") as mock_get_terminal_size: + # on larger monitors, this value will be != 80 + mock_get_terminal_size.return_value.columns = 80 + assert attack._sqlmap_command() == ( + "export COLUMNS=80; echo \"http://172.18.0.2/dvwa/vulnerabilities/sqli/?id=&Submit=Submit\" | " + "sqlmap --purge --batch --dump") def test_raise_exception_bad_output(self, attack: SQLMapAttack): attack.exec_commands_on_target = lambda _: attack.printer.print("Bad output") From 47ec2305479c9d8f20075164eddc0078ad1c6b1d Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 08:30:39 +0200 Subject: [PATCH 04/18] Manage python dependencies within workflow --- .github/workflows/socbed-unittest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/socbed-unittest.yml b/.github/workflows/socbed-unittest.yml index d77221b..c20ae21 100644 --- a/.github/workflows/socbed-unittest.yml +++ b/.github/workflows/socbed-unittest.yml @@ -9,5 +9,9 @@ jobs: runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install tox - run: tox -- -m "not systest" From 29b5a75d6a05e8fd5be45ebba8c91345519b7cd6 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 08:39:58 +0200 Subject: [PATCH 05/18] Manage python dependencies within workflow --- .github/workflows/socbed-systemtest-dev.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index 0062a1e..e97c38e 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -9,17 +9,10 @@ jobs: runs-on: [self-hosted, linux] steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - ref: dev - - - name: Create virtual environment - run: python -m venv /usr/share/runner-dependencies/socbed_env - - - name: Activate virtual environment - run: source /usr/share/runner-dependencies/socbed_env/bin/activate - - - name: Install requirements in virtual environment (without using cached packages) - run: pip install -r requirements.txt --no-cache-dir + python-version: '3.12' + - run: pip install -r requirements.txt build-machines: runs-on: [self-hosted, linux] From 4f38e203c12bedd5ad237f3a56667ca46edd5739 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 08:49:31 +0200 Subject: [PATCH 06/18] Update workflow to manage dependencies per individual task --- .github/workflows/socbed-systemtest-dev.yml | 39 ++++++++------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index e97c38e..a189aae 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -5,26 +5,20 @@ on: - cron: "0 1 * * THU" # At 01:00 on Thursday jobs: - prepare-environment: - runs-on: [self-hosted, linux] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - run: pip install -r requirements.txt - build-machines: runs-on: [self-hosted, linux] - needs: [prepare-environment] timeout-minutes: 480 steps: - uses: actions/checkout@v4 with: ref: dev - - - name: Activate virtual environment - run: source /usr/share/runner-dependencies/socbed_env/bin/activate + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Install dependencies + run: pip install -r requirements.txt - name: Build Internet Router uses: nick-invision/retry@v3 @@ -84,14 +78,16 @@ jobs: test-machines: runs-on: [self-hosted, linux] - needs: [prepare-environment, build-machines] + needs: [build-machines] steps: - uses: actions/checkout@v4 with: ref: dev - - - name: Activate virtual environment - run: source /usr/share/runner-dependencies/socbed_env/bin/activate + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install tox - name: Ensure all machines are powered off run: ./tools/cleanup_failed_session @@ -107,7 +103,7 @@ jobs: delete-machines: runs-on: [self-hosted, linux] if: always() - needs: [prepare-environment, build-machines, test-machines] + needs: [build-machines, test-machines] steps: - uses: actions/checkout@v4 with: @@ -115,10 +111,3 @@ jobs: - name: Delete created VMs run: ./tools/delete_vms - - - name: Deactivate virtual environment - run: deactivate || true - - - name: Delete virtual environment - run: rm -rf /usr/share/runner-dependencies/socbed_env - From 273cffe88be34d932814b870bc115e6a953d1175 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 08:53:18 +0200 Subject: [PATCH 07/18] Add workflow dispatch to dev workflow --- .github/workflows/socbed-systemtest-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index a189aae..10e358f 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -3,6 +3,7 @@ name: Nightly Build with System Tests (Development Branch) on: schedule: - cron: "0 1 * * THU" # At 01:00 on Thursday + workflow_dispatch: jobs: build-machines: From 50318d451f55687fb8a4dc9ca51ab8554fc769bb Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 08:57:09 +0200 Subject: [PATCH 08/18] Add temporary push trigger to dev workflow --- .github/workflows/socbed-systemtest-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index 10e358f..bdb55fb 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -1,6 +1,7 @@ name: Nightly Build with System Tests (Development Branch) on: + push: schedule: - cron: "0 1 * * THU" # At 01:00 on Thursday workflow_dispatch: From b56a38eb2e8721a92d7ee4d5fffffd89d2b321a0 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 09:46:49 +0200 Subject: [PATCH 09/18] Bump ansible version --- .github/workflows/socbed-systemtest-dev.yml | 2 +- .../ansible/roles/auditbeat/tasks/main.yml | 6 +-- .../configure_company_router/tasks/main.yml | 34 +++++++-------- .../configure_internet_router/tasks/main.yml | 10 ++--- .../tasks/install_firefox.yml | 2 +- .../configure_win10_client/tasks/main.yml | 42 +++++++++---------- .../rsyslog_install_ipfire/tasks/main.yml | 12 ++++-- requirements.txt | 6 +-- 8 files changed, 59 insertions(+), 55 deletions(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index bdb55fb..58ffddd 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -89,7 +89,7 @@ jobs: with: python-version: '3.12' - name: Install dependencies - run: pip install tox + run: pip install tox==4.18.1 - name: Ensure all machines are powered off run: ./tools/cleanup_failed_session diff --git a/provisioning/ansible/roles/auditbeat/tasks/main.yml b/provisioning/ansible/roles/auditbeat/tasks/main.yml index 4c5b7fd..0b7ec22 100644 --- a/provisioning/ansible/roles/auditbeat/tasks/main.yml +++ b/provisioning/ansible/roles/auditbeat/tasks/main.yml @@ -1,4 +1,4 @@ --- -- include: add_auditbeat_apt_repository.yml -- include: install_auditbeat.yml -- include: configure_auditbeat.yml +- include_tasks: add_auditbeat_apt_repository.yml +- include_tasks: install_auditbeat.yml +- include_tasks: configure_auditbeat.yml diff --git a/provisioning/ansible/roles/configure_company_router/tasks/main.yml b/provisioning/ansible/roles/configure_company_router/tasks/main.yml index 6c7daee..7f23075 100644 --- a/provisioning/ansible/roles/configure_company_router/tasks/main.yml +++ b/provisioning/ansible/roles/configure_company_router/tasks/main.yml @@ -1,17 +1,17 @@ -- include: configure_suricata.yml -- include: configure_dhcpd.yml -- include: configure_fwhosts_customservices.yml -- include: configure_hosts.yml -- include: configure_K00squid.yml -- include: configure_loggings_settings.yml -- include: configure_ntp.yml -- include: configure_proxy_advanced.yml -- include: configure_proxy_cachemgr.yml -- include: configure_proxy_settings.yml -- include: configure_proxy_viewersettings.yml -- include: configure_squid.yml -- include: configure_squid_cachemgr.yml -- include: configure_syslog.yml -- include: configure_time_settings.yml -- include: create_ipfire_proxy_enable.yml -- include: create_symbolic_link_for_syslog.yml +- include_tasks: configure_suricata.yml +- include_tasks: configure_dhcpd.yml +- include_tasks: configure_fwhosts_customservices.yml +- include_tasks: configure_hosts.yml +- include_tasks: configure_K00squid.yml +- include_tasks: configure_loggings_settings.yml +- include_tasks: configure_ntp.yml +- include_tasks: configure_proxy_advanced.yml +- include_tasks: configure_proxy_cachemgr.yml +- include_tasks: configure_proxy_settings.yml +- include_tasks: configure_proxy_viewersettings.yml +- include_tasks: configure_squid.yml +- include_tasks: configure_squid_cachemgr.yml +- include_tasks: configure_syslog.yml +- include_tasks: configure_time_settings.yml +- include_tasks: create_ipfire_proxy_enable.yml +- include_tasks: create_symbolic_link_for_syslog.yml diff --git a/provisioning/ansible/roles/configure_internet_router/tasks/main.yml b/provisioning/ansible/roles/configure_internet_router/tasks/main.yml index 5762450..27ef1d7 100644 --- a/provisioning/ansible/roles/configure_internet_router/tasks/main.yml +++ b/provisioning/ansible/roles/configure_internet_router/tasks/main.yml @@ -1,5 +1,5 @@ -- include: configure_etc_hosts.yml -- include: configure_etc_unbound_hosts.yml -- include: configure_hosts.yml -- include: configure_ntp.yml -- include: create_symbolic_link_for_syslog.yml +- include_tasks: configure_etc_hosts.yml +- include_tasks: configure_etc_unbound_hosts.yml +- include_tasks: configure_hosts.yml +- include_tasks: configure_ntp.yml +- include_tasks: create_symbolic_link_for_syslog.yml diff --git a/provisioning/ansible/roles/configure_win10_client/tasks/install_firefox.yml b/provisioning/ansible/roles/configure_win10_client/tasks/install_firefox.yml index 6fe7ee4..dabb3d7 100644 --- a/provisioning/ansible/roles/configure_win10_client/tasks/install_firefox.yml +++ b/provisioning/ansible/roles/configure_win10_client/tasks/install_firefox.yml @@ -23,4 +23,4 @@ src: files/win10_firefox/mozilla.cfg dest: C:\Program Files (x86)\Mozilla Firefox\mozilla.cfg -- include: set_default_browser.yml +- include_tasks: set_default_browser.yml diff --git a/provisioning/ansible/roles/configure_win10_client/tasks/main.yml b/provisioning/ansible/roles/configure_win10_client/tasks/main.yml index 679726e..4926ad9 100644 --- a/provisioning/ansible/roles/configure_win10_client/tasks/main.yml +++ b/provisioning/ansible/roles/configure_win10_client/tasks/main.yml @@ -1,21 +1,21 @@ -- include: setup_user.yml -- include: set_timezone.yml -- include: add_exclusion_folder.yml -- include: disable_firewall.yml -- include: disable_screen_lock.yml -- include: install_python.yml -- include: update_pip.yml -- include: install_pip_selenium.yml -- include: upload_assets.yml -- include: upload_userbehavior_src.yml -- include: install_firefox.yml -- include: disable_automatic_updates.yml -- include: run_init_tbf_client_on_boot.yml -- include: install_ssh_server.yml -- include: disable_ipv6.yml -- include: set_autostart.yml -- include: install_imdisk.yml -- include: set_logging_config.yml -- include: install_sysmon.yml -- include: install_winlogbeat.yml -- include: disable_defender.yml +- include_tasks: setup_user.yml +- include_tasks: set_timezone.yml +- include_tasks: add_exclusion_folder.yml +- include_tasks: disable_firewall.yml +- include_tasks: disable_screen_lock.yml +- include_tasks: install_python.yml +- include_tasks: update_pip.yml +- include_tasks: install_pip_selenium.yml +- include_tasks: upload_assets.yml +- include_tasks: upload_userbehavior_src.yml +- include_tasks: install_firefox.yml +- include_tasks: disable_automatic_updates.yml +- include_tasks: run_init_tbf_client_on_boot.yml +- include_tasks: install_ssh_server.yml +- include_tasks: disable_ipv6.yml +- include_tasks: set_autostart.yml +- include_tasks: install_imdisk.yml +- include_tasks: set_logging_config.yml +- include_tasks: install_sysmon.yml +- include_tasks: install_winlogbeat.yml +- include_tasks: disable_defender.yml diff --git a/provisioning/ansible/roles/rsyslog_install_ipfire/tasks/main.yml b/provisioning/ansible/roles/rsyslog_install_ipfire/tasks/main.yml index 399c132..de8f556 100644 --- a/provisioning/ansible/roles/rsyslog_install_ipfire/tasks/main.yml +++ b/provisioning/ansible/roles/rsyslog_install_ipfire/tasks/main.yml @@ -1,4 +1,8 @@ -- include: install.yml packet=libestr-0.1.11-1.ipfire -- include: install.yml packet=libfastjson-0.99.8-1.ipfire -- include: install.yml packet=liblogging-1.0.6-1.ipfire -- include: install.yml packet=rsyslog-8.1907.0-1.ipfire +- include_tasks: install.yml + with_items: + - libestr-0.1.11-1.ipfire + - libfastjson-0.99.8-1.ipfire + - liblogging-1.0.6-1.ipfire + - rsyslog-8.1907.0-1.ipfire + vars: + packet: "{{ item }}" diff --git a/requirements.txt b/requirements.txt index 236c171..76ea108 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -ansible==5.1.0 -colorama==0.4.1 +ansible==10.4.0 +colorama==0.4.6 paramiko==2.11.0 pytest==8.3.3 pyvmomi==6.7.1.2018.12 pywinrm==0.4.1 selenium==4.3.0 -tox==3.7.0 +tox==4.18.1 veryprettytable==0.8.1 asyncio==3.4.3 setuptools~=74.1.2 From 7283c2894f54147971b399869ec8ca81220e6219 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 09:57:19 +0200 Subject: [PATCH 10/18] Debug ansible errors --- .github/workflows/socbed-systemtest-dev.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index 58ffddd..19cd97b 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -22,6 +22,12 @@ jobs: - name: Install dependencies run: pip install -r requirements.txt + - name: Check Python and Ansible path + run: | + which python + which ansible-playbook + ansible-playbook --version + - name: Build Internet Router uses: nick-invision/retry@v3 with: From e1ef25e9ad4408448f4d0e9d6283ac67dc26d3ce Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 10:00:30 +0200 Subject: [PATCH 11/18] Temporarily change target branch to custom one --- .github/workflows/socbed-systemtest-dev.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index 19cd97b..cf46b4a 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: dev + ref: replace-smtpd-with-aiosmtpd - uses: actions/setup-python@v5 with: @@ -21,12 +21,6 @@ jobs: cache: 'pip' - name: Install dependencies run: pip install -r requirements.txt - - - name: Check Python and Ansible path - run: | - which python - which ansible-playbook - ansible-playbook --version - name: Build Internet Router uses: nick-invision/retry@v3 From dae12cb74b9e4a99ce7e802fdea036ce4ee7e1ef Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 10:19:47 +0200 Subject: [PATCH 12/18] Move preparations to separate job --- .github/workflows/socbed-systemtest-dev.yml | 30 ++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index cf46b4a..fe207fc 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -6,21 +6,31 @@ on: - cron: "0 1 * * THU" # At 01:00 on Thursday workflow_dispatch: + jobs: - build-machines: + prepare-environment: runs-on: [self-hosted, linux] - timeout-minutes: 480 steps: - uses: actions/checkout@v4 with: - ref: replace-smtpd-with-aiosmtpd - + ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + - uses: actions/setup-python@v5 with: python-version: '3.12' cache: 'pip' - name: Install dependencies - run: pip install -r requirements.txt + run: pip install -r requirements.txt --no-cache-dir + - name: Install SOCBED + run: pip install --editable . + + build-machines: + runs-on: [self-hosted, linux] + timeout-minutes: 480 + steps: + - uses: actions/checkout@v4 + with: + ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name - name: Build Internet Router uses: nick-invision/retry@v3 @@ -84,7 +94,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: dev + ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name - uses: actions/setup-python@v5 with: python-version: '3.12' @@ -109,7 +119,13 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: dev + ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name - name: Delete created VMs run: ./tools/delete_vms + + - name: Uninstall SOCBED + run: pip uninstall socbed -y + + - name: Remove all installed packages + run: pip freeze | xargs pip uninstall -y From ecfb306d91c431f3b793f25d8168ba5eb4d4e79a Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 10:22:52 +0200 Subject: [PATCH 13/18] Add 'needs' keyword ensuring intended order of execution --- .github/workflows/socbed-systemtest-dev.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index fe207fc..aee4ee6 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -4,7 +4,7 @@ on: push: schedule: - cron: "0 1 * * THU" # At 01:00 on Thursday - workflow_dispatch: + workflow_dispatch: # Remove this jobs: @@ -26,6 +26,7 @@ jobs: build-machines: runs-on: [self-hosted, linux] + needs: [prepare-environment] timeout-minutes: 480 steps: - uses: actions/checkout@v4 @@ -95,11 +96,6 @@ jobs: - uses: actions/checkout@v4 with: ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install dependencies - run: pip install tox==4.18.1 - name: Ensure all machines are powered off run: ./tools/cleanup_failed_session @@ -115,7 +111,7 @@ jobs: delete-machines: runs-on: [self-hosted, linux] if: always() - needs: [build-machines, test-machines] + needs: [test-machines] steps: - uses: actions/checkout@v4 with: From 11a0a40259fd14670833896735c33f832479ee75 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 10:31:21 +0200 Subject: [PATCH 14/18] Include python setup in every individual job --- .github/workflows/socbed-systemtest-dev.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index aee4ee6..97b0f1b 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -26,13 +26,17 @@ jobs: build-machines: runs-on: [self-hosted, linux] - needs: [prepare-environment] timeout-minutes: 480 steps: - uses: actions/checkout@v4 with: ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Build Internet Router uses: nick-invision/retry@v3 with: @@ -96,6 +100,11 @@ jobs: - uses: actions/checkout@v4 with: ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' - name: Ensure all machines are powered off run: ./tools/cleanup_failed_session @@ -116,6 +125,11 @@ jobs: - uses: actions/checkout@v4 with: ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' - name: Delete created VMs run: ./tools/delete_vms From cf8caff636b8c65f1f47ad5eac0219a9221f52b7 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 10:33:22 +0200 Subject: [PATCH 15/18] Add missing 'needs' keyword --- .github/workflows/socbed-unittest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/socbed-unittest.yml b/.github/workflows/socbed-unittest.yml index c20ae21..a6cfbf6 100644 --- a/.github/workflows/socbed-unittest.yml +++ b/.github/workflows/socbed-unittest.yml @@ -12,6 +12,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - run: pip install tox + - run: pip install tox==4.18.1 - run: tox -- -m "not systest" From a001ac56d69c6d0864d1613d5bae52a6f2f5e80d Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 10:37:03 +0200 Subject: [PATCH 16/18] Add missing 'needs' keyword --- .github/workflows/socbed-systemtest-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index 97b0f1b..bf9800a 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -26,6 +26,7 @@ jobs: build-machines: runs-on: [self-hosted, linux] + needs: [prepare-environment] timeout-minutes: 480 steps: - uses: actions/checkout@v4 From 22e40ab67eedeadd230ca57f75baca1af3ceb808 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 14:26:47 +0200 Subject: [PATCH 17/18] Update requirements --- README.md | 14 +++++++------- requirements.txt | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c061d6c..7266ecc 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ After the build process is finished, SOCBED sessions can be started, controlled, ## System Requirements * Physical host with Linux or macOS. Note: Running SOCBED (and therefore VirtualBox) in a virtual machine might work as well but was not tested. -* Python v3.8 or newer +* Python v3.9 or newer * RAM: 16 GB minimum, 32 GB recommended * CPU: Quad-core with hardware support for virtualization * HDD: 50 GB free, SSD strongly recommended @@ -165,12 +165,12 @@ The domain name is `BREACH`. The following table shows all available web interfaces and their logins: -| Machine | Service | Username | Password | URL | -| --- | --- | --- | --- | --- | -| Company Router | IPFire | admin | breach | https://192.168.56.10:444/ | -| DMZ Server | phpMyAdmin | root | breach | http://192.168.56.20/phpmyadmin/ | -| Log Server | Kibana | - | - | http://192.168.56.12:5601/app/kibana | -| Internet Router | IPFire | admin | breach | https://192.168.56.30:444/ | +| Machine | Service | Username | Password | URL | +| --------------- | ---------- | -------- | -------- | ------------------------------------ | +| Company Router | IPFire | admin | breach | https://192.168.56.10:444/ | +| DMZ Server | phpMyAdmin | root | breach | http://192.168.56.20/phpmyadmin/ | +| Log Server | Kibana | - | - | http://192.168.56.12:5601/app/kibana | +| Internet Router | IPFire | admin | breach | https://192.168.56.30:444/ | ## Documentation diff --git a/requirements.txt b/requirements.txt index 76ea108..0191641 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -ansible==10.4.0 +ansible==8.7.0 colorama==0.4.6 -paramiko==2.11.0 +paramiko==3.4.1 pytest==8.3.3 pyvmomi==6.7.1.2018.12 pywinrm==0.4.1 From ff9c347f5abe9b4ca5d4c1ff7feb40bb7a27d685 Mon Sep 17 00:00:00 2001 From: Philipp Boenninghausen Date: Fri, 13 Sep 2024 17:25:22 +0200 Subject: [PATCH 18/18] Remove temporary workflow changes --- .github/workflows/socbed-systemtest-dev.yml | 10 ++-- .github/workflows/socbed-systemtest.yml | 55 +++++++++++---------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/.github/workflows/socbed-systemtest-dev.yml b/.github/workflows/socbed-systemtest-dev.yml index bf9800a..b13a563 100644 --- a/.github/workflows/socbed-systemtest-dev.yml +++ b/.github/workflows/socbed-systemtest-dev.yml @@ -1,10 +1,8 @@ name: Nightly Build with System Tests (Development Branch) on: - push: schedule: - cron: "0 1 * * THU" # At 01:00 on Thursday - workflow_dispatch: # Remove this jobs: @@ -13,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + ref: dev - uses: actions/setup-python@v5 with: @@ -31,7 +29,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + ref: dev - uses: actions/setup-python@v5 with: @@ -100,7 +98,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + ref: dev - uses: actions/setup-python@v5 with: @@ -125,7 +123,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: replace-smtpd-with-aiosmtpd # Change this to the development branch name + ref: dev - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/socbed-systemtest.yml b/.github/workflows/socbed-systemtest.yml index d6c637a..b07c4a5 100644 --- a/.github/workflows/socbed-systemtest.yml +++ b/.github/workflows/socbed-systemtest.yml @@ -11,19 +11,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Create virtual environment - run: python3 -m venv /usr/share/runner-dependencies/socbed_env - - - name: Activate virtual environment - run: source /usr/share/runner-dependencies/socbed_env/bin/activate - - - name: Upgrade pip3 inside virtual environment - run: pip3 install --upgrade pip - - - name: Install requirements in virtual environment (without using cached packages) - run: pip3 install -r requirements.txt --no-cache-dir - - - name: Install socbed + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Install dependencies + run: pip install -r requirements.txt --no-cache-dir + - name: Install SOCBED run: pip install --editable . build-machines: @@ -33,9 +27,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Activate virtual environment - run: source /usr/share/runner-dependencies/socbed_env/bin/activate - + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Build Internet Router uses: nick-invision/retry@v3 with: @@ -94,12 +90,14 @@ jobs: test-machines: runs-on: [self-hosted, linux] - needs: [prepare-environment, build-machines] + needs: [build-machines] steps: - uses: actions/checkout@v4 - - name: Activate virtual environment - run: source /usr/share/runner-dependencies/socbed_env/bin/activate + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' - name: Ensure all machines are powered off run: ./tools/cleanup_failed_session @@ -115,16 +113,21 @@ jobs: delete-machines: runs-on: [self-hosted, linux] if: always() - needs: [prepare-environment, build-machines, test-machines] + needs: [test-machines] steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - name: Delete created VMs run: ./tools/delete_vms - - - name: Deactivate virtual environment - run: deactivate || true - - - name: Delete virtual environment - run: rm -rf /usr/share/runner-dependencies/socbed_env + + - name: Uninstall SOCBED + run: pip uninstall socbed -y + + - name: Remove all installed packages + run: pip freeze | xargs pip uninstall -y