From 99a955c42bafc8b6d36bd751a66178a4b4eacdd2 Mon Sep 17 00:00:00 2001 From: Tobias Pfandzelter Date: Tue, 26 Mar 2024 11:15:51 +0100 Subject: [PATCH] make using docker for satgen possible --- Makefile | 5 +- TODO.md | 14 --- docs/setup.md | 17 ++++ proto/celestial/celestial.pb.go | 4 +- proto/celestial/celestial_grpc.pb.go | 41 ++++++-- proto/celestial/celestial_pb2.py | 1 - proto/celestial/celestial_pb2_grpc.pyi | 50 +++++++--- satgen.Dockerfile | 32 +++++++ test.py | 126 ------------------------- 9 files changed, 126 insertions(+), 164 deletions(-) delete mode 100644 TODO.md create mode 100644 satgen.Dockerfile delete mode 100644 test.py diff --git a/Makefile b/Makefile index 19bbae4..eb2f05a 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ PROJECT_NAME := "celestial" PKG := "github.com/OpenFogStack/$(PROJECT_NAME)" GO_FILES := $(shell find . -name '*.go' | grep -v _test.go) -.PHONY: build proto ebpf celestial-make rootfsbuilder +.PHONY: build proto ebpf celestial-make satgen-docker rootfsbuilder build: celestial.bin @@ -47,5 +47,8 @@ celestial.bin: go.mod go.sum celestial.go ${GO_FILES} ## build go binary celestial-make: compile.Dockerfile ## build the compile container @docker build --platform ${OS}/${ARCH} -f $< -t $@ . +satgen-docker: satgen.Dockerfile satgen.py requirements.txt celestial/*.py ## build the satgen container + @docker build -f $< -t $@ . + rootfsbuilder: builder/build-script.sh builder/Dockerfile builder/fcinit.c builder/inittab builder/interfaces builder/run-user-script builder/prepare.sh builder/ceinit ## build the rootfs builder container @docker build --platform=linux/amd64 -t $@:latest builder/ diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 6c39460..0000000 --- a/TODO.md +++ /dev/null @@ -1,14 +0,0 @@ -# Celestial++ TODO List - -- [x] Update all dependencies -- [x] Create a test process -- [x] Re-structure host-side code with packages and interfaces -- [x] Move database communication to websockets (or similar) -- [x] find a good solution for DNS -- [] Battery emulation -- [] Different topologies -- [] Support for OCI images -- [x] Add NetEm stuff -- [] Add fault injection -- [x] Update documentation -- [x] git cloning in rootfs builder not working as git has no access to dev/urandom (needs symlink into chroot) diff --git a/docs/setup.md b/docs/setup.md index 5f8fbea..7356051 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -101,6 +101,13 @@ source .venv/bin/activate python3 -m pip install -r requirements.txt ``` +Alternatively, you can also run this with Docker. +Build the `satgen-docker` image: + +```sh +make satgen-docker +``` + #### Running Satellite Trajectory Generation We can then use the `satgen.py` script to generate trajectories: @@ -112,6 +119,16 @@ python3 satgen.py [PATH_TO_CONFIG] [OUTPUT_PATH] Replace `PATH_TO_CONFIG` with the path to your configuration file and the optional `OUTPUT_PATH` with a path to your output file. +If you want to use the Docker image instead: + +```sh +docker run --rm \ + -v ${pwd}:/app \ + satgen-docker \ + /app/[PATH_TO_CONFIG] \ + /app/[OUTPUT_PATH] \ +``` + After a few seconds to minutes (depending on the size of the constellation you want to emulate) you will end up with a `.zip` file that you can use for further emulation. diff --git a/proto/celestial/celestial.pb.go b/proto/celestial/celestial.pb.go index ac2227e..3f4e710 100644 --- a/proto/celestial/celestial.pb.go +++ b/proto/celestial/celestial.pb.go @@ -16,8 +16,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v4.25.2 +// protoc-gen-go v1.31.0 +// protoc v4.25.1 // source: celestial.proto package celestial diff --git a/proto/celestial/celestial_grpc.pb.go b/proto/celestial/celestial_grpc.pb.go index f6b220a..751d3a5 100644 --- a/proto/celestial/celestial_grpc.pb.go +++ b/proto/celestial/celestial_grpc.pb.go @@ -1,7 +1,23 @@ +// +// This file is part of Celestial (https://github.com/OpenFogStack/celestial). +// Copyright (c) 2024 Tobias Pfandzelter, The OpenFogStack Team. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v4.25.2 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.25.1 // source: celestial.proto package celestial @@ -18,6 +34,13 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + Celestial_Register_FullMethodName = "/openfogstack.celestial.celestial.Celestial/Register" + Celestial_Init_FullMethodName = "/openfogstack.celestial.celestial.Celestial/Init" + Celestial_Update_FullMethodName = "/openfogstack.celestial.celestial.Celestial/Update" + Celestial_Stop_FullMethodName = "/openfogstack.celestial.celestial.Celestial/Stop" +) + // CelestialClient is the client API for Celestial service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -38,7 +61,7 @@ func NewCelestialClient(cc grpc.ClientConnInterface) CelestialClient { func (c *celestialClient) Register(ctx context.Context, in *RegisterRequest, opts ...grpc.CallOption) (*RegisterResponse, error) { out := new(RegisterResponse) - err := c.cc.Invoke(ctx, "/openfogstack.celestial.celestial.Celestial/Register", in, out, opts...) + err := c.cc.Invoke(ctx, Celestial_Register_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -47,7 +70,7 @@ func (c *celestialClient) Register(ctx context.Context, in *RegisterRequest, opt func (c *celestialClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/openfogstack.celestial.celestial.Celestial/Init", in, out, opts...) + err := c.cc.Invoke(ctx, Celestial_Init_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -55,7 +78,7 @@ func (c *celestialClient) Init(ctx context.Context, in *InitRequest, opts ...grp } func (c *celestialClient) Update(ctx context.Context, opts ...grpc.CallOption) (Celestial_UpdateClient, error) { - stream, err := c.cc.NewStream(ctx, &Celestial_ServiceDesc.Streams[0], "/openfogstack.celestial.celestial.Celestial/Update", opts...) + stream, err := c.cc.NewStream(ctx, &Celestial_ServiceDesc.Streams[0], Celestial_Update_FullMethodName, opts...) if err != nil { return nil, err } @@ -90,7 +113,7 @@ func (x *celestialUpdateClient) CloseAndRecv() (*Empty, error) { func (c *celestialClient) Stop(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/openfogstack.celestial.celestial.Celestial/Stop", in, out, opts...) + err := c.cc.Invoke(ctx, Celestial_Stop_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -145,7 +168,7 @@ func _Celestial_Register_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/openfogstack.celestial.celestial.Celestial/Register", + FullMethod: Celestial_Register_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CelestialServer).Register(ctx, req.(*RegisterRequest)) @@ -163,7 +186,7 @@ func _Celestial_Init_Handler(srv interface{}, ctx context.Context, dec func(inte } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/openfogstack.celestial.celestial.Celestial/Init", + FullMethod: Celestial_Init_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CelestialServer).Init(ctx, req.(*InitRequest)) @@ -207,7 +230,7 @@ func _Celestial_Stop_Handler(srv interface{}, ctx context.Context, dec func(inte } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/openfogstack.celestial.celestial.Celestial/Stop", + FullMethod: Celestial_Stop_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CelestialServer).Stop(ctx, req.(*Empty)) diff --git a/proto/celestial/celestial_pb2.py b/proto/celestial/celestial_pb2.py index 7ece3ee..c3cf58d 100644 --- a/proto/celestial/celestial_pb2.py +++ b/proto/celestial/celestial_pb2.py @@ -19,7 +19,6 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'celestial_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'Z\014./;celestial' _globals['_VMSTATE']._serialized_start=1449 diff --git a/proto/celestial/celestial_pb2_grpc.pyi b/proto/celestial/celestial_pb2_grpc.pyi index 8128a2f..9869da9 100644 --- a/proto/celestial/celestial_pb2_grpc.pyi +++ b/proto/celestial/celestial_pb2_grpc.pyi @@ -21,9 +21,19 @@ import abc import celestial_pb2 import collections.abc import grpc +import grpc.aio +import typing + +_T = typing.TypeVar('_T') + +class _MaybeAsyncIterator(collections.abc.AsyncIterator[_T], collections.abc.Iterator[_T], metaclass=abc.ABCMeta): + ... + +class _ServicerContext(grpc.ServicerContext, grpc.aio.ServicerContext): # type: ignore + ... class CelestialStub: - def __init__(self, channel: grpc.Channel) -> None: ... + def __init__(self, channel: typing.Union[grpc.Channel, grpc.aio.Channel]) -> None: ... Register: grpc.UnaryUnaryMultiCallable[ celestial_pb2.RegisterRequest, celestial_pb2.RegisterResponse, @@ -41,30 +51,48 @@ class CelestialStub: celestial_pb2.Empty, ] +class CelestialAsyncStub: + Register: grpc.aio.UnaryUnaryMultiCallable[ + celestial_pb2.RegisterRequest, + celestial_pb2.RegisterResponse, + ] + Init: grpc.aio.UnaryUnaryMultiCallable[ + celestial_pb2.InitRequest, + celestial_pb2.Empty, + ] + Update: grpc.aio.StreamUnaryMultiCallable[ + celestial_pb2.StateUpdateRequest, + celestial_pb2.Empty, + ] + Stop: grpc.aio.UnaryUnaryMultiCallable[ + celestial_pb2.Empty, + celestial_pb2.Empty, + ] + class CelestialServicer(metaclass=abc.ABCMeta): @abc.abstractmethod def Register( self, request: celestial_pb2.RegisterRequest, - context: grpc.ServicerContext, - ) -> celestial_pb2.RegisterResponse: ... + context: _ServicerContext, + ) -> typing.Union[celestial_pb2.RegisterResponse, collections.abc.Awaitable[celestial_pb2.RegisterResponse]]: ... @abc.abstractmethod def Init( self, request: celestial_pb2.InitRequest, - context: grpc.ServicerContext, - ) -> celestial_pb2.Empty: ... + context: _ServicerContext, + ) -> typing.Union[celestial_pb2.Empty, collections.abc.Awaitable[celestial_pb2.Empty]]: ... @abc.abstractmethod def Update( self, - request_iterator: collections.abc.Iterator[celestial_pb2.StateUpdateRequest], - context: grpc.ServicerContext, - ) -> celestial_pb2.Empty: ... + request_iterator: _MaybeAsyncIterator[celestial_pb2.StateUpdateRequest], + context: _ServicerContext, + ) -> typing.Union[celestial_pb2.Empty, collections.abc.Awaitable[celestial_pb2.Empty]]: ... @abc.abstractmethod def Stop( self, request: celestial_pb2.Empty, - context: grpc.ServicerContext, - ) -> celestial_pb2.Empty: ... + context: _ServicerContext, + ) -> typing.Union[celestial_pb2.Empty, collections.abc.Awaitable[celestial_pb2.Empty]]: ... -def add_CelestialServicer_to_server(servicer: CelestialServicer, server: grpc.Server) -> None: ... +def add_CelestialServicer_to_server(servicer: CelestialServicer, server: typing.Union[grpc.Server, grpc.aio.Server]) -> None: ... diff --git a/satgen.Dockerfile b/satgen.Dockerfile new file mode 100644 index 0000000..db4b453 --- /dev/null +++ b/satgen.Dockerfile @@ -0,0 +1,32 @@ +# +# This file is part of Celestial (https://github.com/OpenFogStack/celestial). +# Copyright (c) 2024 Tobias Pfandzelter, The OpenFogStack Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +FROM python:3.11-slim + +RUN apt update && apt install -y \ + --no-install-recommends \ + --no-install-suggests \ + git && \ + apt clean && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY satgen.py . +COPY celestial/*.py celestial/ + +ENTRYPOINT ["python", "satgen.py"] \ No newline at end of file diff --git a/test.py b/test.py deleted file mode 100644 index acd750d..0000000 --- a/test.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# This file is part of Celestial (https://github.com/OpenFogStack/celestial). -# Copyright (c) 2024 Tobias Pfandzelter, The OpenFogStack Team. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 3. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import logging -import sys -import time -import typing - -import celestial.host -import celestial.types -import celestial.zip_serializer - -DEBUG = True -DEFAULT_PORT = 1969 - -if __name__ == "__main__": - if len(sys.argv) < 3: - exit( - "Usage: python3 celestial.py [celestial.zip] [host1_addr] [host2_addr] ... [hostN_addr]" - ) - - if DEBUG: - logging.basicConfig(level=logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) - - celestial_zip = sys.argv[1] - - serializer = celestial.zip_serializer.ZipDeserializer(celestial_zip) - - config = serializer.config() - - host_addrs = sys.argv[2:] - - for i in range(len(host_addrs)): - if ":" not in host_addrs[i]: - host_addrs[i] = f"{host_addrs[i]}:{DEFAULT_PORT}" - - hosts: typing.List[celestial.host.Host] = [ - celestial.host.Host(num=i, addr=host_addrs[i]) for i in range(len(host_addrs)) - ] - - # init the constellation - # register the hosts - logging.info("Registering hosts...") - # with concurrent.futures.ThreadPoolExecutor() as e: - # for h in hosts: - # e.submit(h.register) - logging.info("Hosts registered!") - - inits = serializer.init_machines() - - machines: typing.Dict[ - int, - typing.List[ - typing.Tuple[ - celestial.types.MachineID_dtype, celestial.config.MachineConfig - ] - ], - ] = {h: [] for h in range(len(hosts))} - count = 0 - - for m_id, m_config in inits: - # this is the logic that assigns machines to hosts - # we just do a round robin assignment for now - # this might be suboptimal, because it forces any neighbor - # communication to go across the hosts - # assinging by shells might be better - m_host = count % len(hosts) - - machines[m_host].append((m_id, m_config)) - count += 1 - - # init the hosts - logging.info("Initializing hosts...") - # init_request = celestial.host.make_init_request(hosts, machines) - # with concurrent.futures.ThreadPoolExecutor() as e: - # for h in hosts: - # e.submit(h.init, args=(init_request,)) - logging.info("Hosts initialized!") - - def get_diff( - t: celestial.types.timestamp_s, - ) -> typing.Tuple[ - typing.List[ - typing.Tuple[celestial.types.MachineID_dtype, celestial.types.VMState] - ], - typing.List[ - typing.Tuple[ - celestial.types.MachineID_dtype, - celestial.types.MachineID_dtype, - celestial.types.Link_dtype, - ] - ], - ]: - machine_diff = serializer.diff_machines(t) - link_diff = serializer.diff_links(t) - - return (machine_diff, link_diff) - - # start the simulation - timestep: celestial.types.timestamp_s = 0 - machine_diff, link_diff = get_diff(timestep) - start_time = time.perf_counter() - logging.info("Starting emulation...") - - # install sigterm handler - # signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) - update_request = celestial.host.make_update_request(machine_diff, link_diff) - - with open("test.bin", "wb") as f: - f.write(update_request.SerializeToString())