Merge branch 'develop' into feat/use-effect-submitref

DEVELOPMENT.md

@@ -0,0 +1,18 @@
+# Development resources
+
+The internal details of dyrector.io's components.
+
+## Agent versioning
+
+Crux considers two things to determine if an agent is outdated:
+ - The `AGENT_PROTO_COMPATIBILITY_MINIMUM_VERSION` in the `web/crux/shared/const.ts` file
+ - And the version number in Crux's `package.json`
+
+When the agent version is between `AGENT_PROTO_COMPATIBILITY_MINIMUM_VERSION` and the `package.json`'s version
+the agent's status will be connected, and it will be fully functional. Otherwise it going to be outdated,
+and can not be used unless an update is performed.
+
+### Releasing a new agent
+While creating a new release, the `AGENT_PROTO_COMPATIBILITY_MINIMUM_VERSION` should be incremented to the version number of the new release when:
+ - The agent proto changed in a way that is incompatible with the old agents
+ - The internal working of the agent changed in a way that is incompatible with the new Crux

GETTING_STARTED.md

@@ -32,3 +32,7 @@ Stack:
 2. Execute `make up` in the project root
 3. Open the `localhost:8000`
 4. Happy deploying! 🎬
+
+## Development resources
+
+[Development](./DEVELOPMENT.md)

docker-compose.yaml

@@ -8,9 +8,10 @@ services:
       - --providers.docker=true
       - --providers.docker.exposedbydefault=false
       - --entrypoints.web.address=:80
-      - --entrypoints.websecure.address=:443
+      - --entrypoints.web.http.redirections.entryPoint.to=websecure
       - --entrypoints.web.http.redirections.entrypoint.scheme=https
       - --entrypoints.web.http.redirections.entrypoint.permanent=true
+      - --entrypoints.websecure.address=:443
       - --certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web
       - --certificatesresolvers.letsencrypt.acme.email=${ACME_EMAIL}
       - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json

golang/pkg/builder/container/container_builder.go

@@ -305,7 +305,7 @@ func builderToDockerConfig(dc *DockerContainerBuilder) (hostConfig *container.Ho
 	}
 
 	portListNat := portListToNatBinding(dc.portRanges, dc.portList)
-	exposedPortSet := getPortSet(dc.portList)
+	exposedPortSet := getPortSet(dc.portRanges, dc.portList)
 	hostConfig = &container.HostConfig{
 		Mounts:       dc.mountList,
 		PortBindings: portListNat,
@@ -544,7 +544,7 @@ func portListToNatBinding(portRanges []PortRangeBinding, portList []PortBinding)
 		// the latest go-connections pkg provides on more return value, not yet needed imported by docker-cli
 
 		portMapping, _ := nat.ParsePortSpec(
-			fmt.Sprintf("0.0.0.0:%v-%v:%v-%v/tcp",
+			fmt.Sprintf("%v-%v:%v-%v",
 				p.External.From,
 				p.External.To,
 				p.Internal.From,
@@ -562,7 +562,7 @@ func portListToNatBinding(portRanges []PortRangeBinding, portList []PortBinding)
 			portExternal, _ := nat.NewPort("tcp", fmt.Sprint(*p.PortBinding))
 			portBinding = []nat.PortBinding{
 				{
-					HostIP:   "0.0.0.0",
+					HostIP:   "", // this way docker binds booth ipv4 and ipv6
 					HostPort: portExternal.Port(),
 				},
 			}
@@ -574,16 +574,26 @@ func portListToNatBinding(portRanges []PortRangeBinding, portList []PortBinding)
 	return portMap
 }
 
-func getPortSet(portList []PortBinding) nat.PortSet {
-	if len(portList) == 0 {
+func getPortSet(portRanges []PortRangeBinding, portList []PortBinding) nat.PortSet {
+	if len(portRanges) == 0 && len(portList) == 0 {
 		return nil
 	}
 	portSet := nat.PortSet{}
 
+	for _, portRange := range portRanges {
+		port := portRange.Internal.From
+		for port <= portRange.Internal.To && port != 0 {
+			exposedPort, _ := nat.NewPort("tcp", fmt.Sprint(port))
+			portSet[exposedPort] = struct{}{}
+			port++
+		}
+	}
+
 	for _, port := range portList {
 		exposedPort, _ := nat.NewPort("tcp", fmt.Sprint(port.ExposedPort))
 		portSet[exposedPort] = struct{}{}
 	}
+
 	return portSet
 }
 

web/crux-ui/e2e/utils/node-helper.ts

@@ -34,8 +34,8 @@ export const installDagent = async (page: Page) => {
   try {
     await page.waitForSelector('div.bg-dyo-green')
   } catch (err) {
-    console.log('[E2E] Agent install failed, script output:')
-    installOutput.forEach(it => console.log(it))
+    console.info('[E2E] Agent install failed, script output:')
+    installOutput.forEach(it => console.error(it))
     throw err
   }
 

web/crux-ui/src/components/projects/versions/images/config/common-config-section.tsx

@@ -109,19 +109,22 @@ const CommonConfigSection = (props: CommonConfigSectionProps) => {
     })
 
   const onPortsChanged = (ports: ContainerConfigPort[]) => {
-    const externalPorts = ports.filter(it => !!it.external)
-    const metricsPortGone = !!config.metrics?.port && externalPorts.some(it => it.external === config.metrics.port)
-    onChange({
+    let patch: Partial<InstanceCommonConfigDetails & Pick<InstanceCraneConfigDetails, 'metrics'>> = {
       ports,
-      ...(metricsPortGone
-        ? null
-        : {
-            metrics: {
-              ...config.metrics,
-              port: null,
-            },
-          }),
-    })
+    }
+
+    if (config.metrics) {
+      const metricsPort = ports.find(it => it.internal === config.metrics.port)
+      patch = {
+        ...patch,
+        metrics: {
+          ...config.metrics,
+          port: metricsPort?.internal ?? null,
+        },
+      }
+    }
+
+    onChange(patch)
   }
 
   return !filterEmpty([...COMMON_CONFIG_PROPERTIES], selectedFilters) ? null : (

web/crux-ui/src/models/container.spec.ts

@@ -33,8 +33,7 @@ describe('container model tests', () => {
   it('given undefined ports when calling portToString()', () => {
     const data: ContainerPort = { internal: undefined, external: undefined }
 
-    expect(() => {
-      portToString(data)
-    }).toThrow('Missing Port Information, provide either an internal or external port number.')
+    const actual = portToString(data)
+    expect(actual).toBe('?')
   })
 })

web/crux-ui/src/models/container.ts

@@ -744,7 +744,7 @@ export const portToString = (port: ContainerPort): string => {
     return `${external}->None`
   }
 
-  throw new Error('Missing Port Information, provide either an internal or external port number.')
+  return '?'
 }
 
 export const containerPortsToString = (ports: ContainerPort[], truncateAfter: number = 2): string => {

web/crux-ui/src/validations/container.ts

@@ -1,19 +1,20 @@
 import { UID_MAX } from '@app/const'
 import {
-  ContainerConfigExposeStrategy,
-  ContainerConfigPortRange,
-  ContainerDeploymentStrategyType,
-  ContainerLogDriverType,
-  ContainerNetworkMode,
-  ContainerRestartPolicyType,
   CONTAINER_DEPLOYMENT_STRATEGY_VALUES,
   CONTAINER_EXPOSE_STRATEGY_VALUES,
   CONTAINER_LOG_DRIVER_VALUES,
   CONTAINER_NETWORK_MODE_VALUES,
   CONTAINER_RESTART_POLICY_TYPE_VALUES,
   CONTAINER_VOLUME_TYPE_VALUES,
-  VolumeType,
+  ContainerConfigExposeStrategy,
+  ContainerConfigPortRange,
+  ContainerDeploymentStrategyType,
+  ContainerLogDriverType,
+  ContainerNetworkMode,
   ContainerPort,
+  ContainerRestartPolicyType,
+  Metrics,
+  VolumeType,
 } from '@app/models/container'
 import * as yup from 'yup'
 
@@ -331,17 +332,24 @@ const createMetricsPortRule = (ports: ContainerPort[]) => {
   )
 }
 
-const metricsRule = yup.mixed().when('ports', ([ports]) =>
-  yup
+const metricsRule = yup.mixed().when(['ports'], ([ports]) => {
+  const portRule = createMetricsPortRule(ports)
+
+  return yup
     .object()
-    .shape({
-      path: yup.string().nullable(),
-      port: createMetricsPortRule(ports),
+    .when({
+      is: (it: Metrics) => it?.enabled,
+      then: schema =>
+        schema.shape({
+          enabled: yup.boolean(),
+          path: yup.string().nullable(),
+          port: portRule,
+        }),
     })
-    .default({})
     .nullable()
-    .optional(),
-)
+    .optional()
+    .default(null)
+})
 
 const containerConfigBaseSchema = yup.object().shape({
   name: yup.string().required().matches(/^\S+$/g),

web/crux/src/app/agent/agent.service.ts

@@ -17,7 +17,7 @@ import {
   startWith,
   takeUntil,
 } from 'rxjs'
-import { coerce, major, minor } from 'semver'
+import { coerce } from 'semver'
 import { Agent, AgentConnectionMessage, AgentTokenReplacement } from 'src/domain/agent'
 import AgentInstaller from 'src/domain/agent-installer'
 import { generateAgentToken } from 'src/domain/agent-token'
@@ -39,8 +39,8 @@ import DomainNotificationService from 'src/services/domain.notification.service'
 import PrismaService from 'src/services/prisma.service'
 import GrpcNodeConnection from 'src/shared/grpc-node-connection'
 import AgentMetrics from 'src/shared/metrics/agent.metrics'
-import { getAgentVersionFromPackage } from 'src/shared/package'
-import { PRODUCTION } from '../../shared/const'
+import { getAgentVersionFromPackage, getPackageVersion } from 'src/shared/package'
+import { AGENT_SUPPORTED_MINIMUM_VERSION } from '../../shared/const'
 import DeployService from '../deploy/deploy.service'
 import { DagentTraefikOptionsDto, NodeConnectionStatus, NodeScriptTypeDto } from '../node/node.dto'
 import AgentConnectionStrategyProvider from './agent.connection-strategy.provider'
@@ -380,10 +380,6 @@ export default class AgentService {
   }
 
   agentVersionSupported(version: string): boolean {
-    if (this.configService.get<string>('NODE_ENV') !== PRODUCTION) {
-      return true
-    }
-
     if (!version.includes('-')) {
       return false
     }
@@ -393,13 +389,12 @@ export default class AgentService {
       return false
     }
 
-    const majorMinor = `${major(agentVersion)}.${minor(agentVersion)}`
+    const packageVersion = coerce(getPackageVersion(this.configService))
 
-    return getAgentVersionFromPackage(this.configService) === majorMinor
-  }
-
-  getAgentImageTag() {
-    return this.configService.get<string>('CRUX_AGENT_IMAGE') ?? getAgentVersionFromPackage(this.configService)
+    return (
+      agentVersion.compare(AGENT_SUPPORTED_MINIMUM_VERSION) >= 0 && // agent version is newer (bigger) or the same
+      agentVersion.compare(packageVersion) <= 0
+    )
   }
 
   generateConnectionTokenFor(nodeId: string, startedBy: string): AgentTokenReplacement {
@@ -513,6 +508,10 @@ export default class AgentService {
     }
   }
 
+  private getAgentImageTag() {
+    return this.configService.get<string>('CRUX_AGENT_IMAGE') ?? getAgentVersionFromPackage(this.configService)
+  }
+
   private logServiceInfo(): void {
     this.logger.debug(`Agents: ${this.agents.size}`)
     this.agents.forEach(it => it.debugInfo(this.logger))

web/crux/src/app/image/image.service.ts

@@ -44,7 +44,6 @@ export default class ImageService {
         registry: true,
       },
     })
-
     return this.mapper.toDto(image)
   }
 

web/crux/src/domain/validation.ts

@@ -16,6 +16,7 @@ import {
   ContainerNetworkMode,
   ContainerRestartPolicyType,
   ContainerVolumeType,
+  Metrics,
   PORT_MAX,
 } from './container'
 
@@ -334,17 +335,24 @@ const createMetricsPortRule = (ports: ContainerPort[]) => {
   )
 }
 
-const metricsRule = yup.mixed().when('ports', ([ports]) =>
-  yup
+const metricsRule = yup.mixed().when(['ports'], ([ports]) => {
+  const portRule = createMetricsPortRule(ports)
+
+  return yup
     .object()
-    .shape({
-      path: yup.string().nullable(),
-      port: createMetricsPortRule(ports),
+    .when({
+      is: (it: Metrics) => it?.enabled,
+      then: schema =>
+        schema.shape({
+          enabled: yup.boolean(),
+          path: yup.string().nullable(),
+          port: portRule,
+        }),
     })
-    .default({})
     .nullable()
-    .optional(),
-)
+    .optional()
+    .default(null)
+})
 
 export const containerConfigSchema = yup.object().shape({
   name: yup.string().required().matches(/^\S+$/g),

web/crux/src/shared/const.ts

@@ -1,3 +1,5 @@
+import { coerce } from 'semver'
+
 export const REGISTRY_HUB_URL = 'hub.docker.com'
 export const REGISTRY_GITHUB_URL = 'ghcr.io'
 export const PRODUCTION = 'production'
@@ -16,6 +18,10 @@ export const CONTAINER_DELETE_TIMEOUT = 1000 // millis
 
 export const DEFAULT_CONTAINER_LOG_TAIL = 40
 
+// NOTE(@m8vago): This should be incremented, when a new release includes a proto file change
+const AGENT_PROTO_COMPATIBILITY_MINIMUM_VERSION = '0.8.0'
+export const AGENT_SUPPORTED_MINIMUM_VERSION = coerce(AGENT_PROTO_COMPATIBILITY_MINIMUM_VERSION)
+
 export const API_CREATED_LOCATION_HEADERS = {
   Location: {
     description: 'URL of the created object.',

web/crux/src/websockets/route.ts

@@ -135,7 +135,9 @@ export default class WsRoute {
     const { token } = client
     if (this.callbacks.has(token)) {
       this.logger.error(`Client already connected ${token}`)
-      // TODO(@m8vago): check when this error could occour
+
+      // NOTE (@m8vago): This normally never happens, unless we unintentionally
+      // overwrite the clients' token to the same uuid
       throw new Error('Duplicated client')
     }