Merge pull request #61 from mstg/module-v3-support

Module v3 support and misc fixes

.envrc

@@ -9,3 +9,5 @@ export IBAZEL_USE_LEGACY_WATCHER=0
 export STABLE_REGISTRY_SECRET="none"
 export STABLE_OCI_REGISTRY_NO_NESTED_SUPPORT_IN_2022_SHAME_ON_YOU_AWS="true"
 PATH_add hack
+
+[[ -f .envrc.local ]] && source .envrc.local

.gitignore

@@ -114,3 +114,5 @@ fabric.properties
 .ijwb/.idea/runConfigurations.xml
 .ijwb/.idea/google-java-format.xml
 .ijwb/.idea/dataSources.xml
+
+.envrc.local

ci/resfdeploy.jsonnet

@@ -109,11 +109,12 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
       image: image,
       tag: tag,
     };
+    local istio_mode = if helm_mode then false else if utils.local_image then false else true;
 
     {
       [nssa]: (if helm_mode then '{{ if not .Values.serviceAccountName }}\n' else '') + manifestYamlStream([
         // disable namespace creation in helm mode
-        if !helm_mode then kubernetes.define_namespace(metadata.namespace, infolabels),
+        if !helm_mode then kubernetes.define_namespace(metadata.namespace, infolabels + { annotations: { 'linkerd.io/inject': 'enabled' } }),
         kubernetes.define_service_account(
           metadata {
             name: fixed.name,
@@ -285,22 +286,22 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
             selector=metadata.name,
             env=mappings.get_env_from_svc(srv.name)
           ) for srv in services] +
-          if !helm_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_ingress(
+          if istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_ingress(
             metadata {
               name: srv.name,
               annotations: ingress_annotations + {
-                'kubernetes.io/ingress.class': '{{ .Values.ingressClass | default !"!" }}',
+                'kubernetes.io/ingress.class': if helm_mode then '{{ .Values.ingressClass | default !"!" }}' else 'kong',
                 // Secure only by default
                 // This produces https, grpcs, etc.
                 // todo(mustafa): check if we need to add an exemption to a protocol (TCP comes to mind)
-                'konghq.com/protocols': '{{ .Values.kongProtocols | default !"%ss!"' % std.strReplace(std.strReplace(std.strReplace(srv.name, metadata.name, ''), stage, ''), '-', ''),
+                'konghq.com/protocols': (if helm_mode then '{{ .Values.kongProtocols | default !"%ss!" }}' else '%ss') % std.strReplace(std.strReplace(std.strReplace(srv.name, metadata.name, ''), stage, ''), '-', ''),
               }
             },
             host=if helm_mode then '{{ .Values.%s.ingressHost }}' % srv.portName else mappings.get(srv.name, user),
             port=srv.port,
             srvName=srv.name + '-service',
           ) else null for srv in services] +
-          if helm_mode then [] else [kubernetes.define_virtual_service(metadata { name: srv.name + '-internal' }, {
+          if !istio_mode then [] else [kubernetes.define_virtual_service(metadata { name: srv.name + '-internal' }, {
             hosts: [vshost(srv)],
             gateways: [],
             http: [
@@ -317,7 +318,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
               },
             ],
           },) for srv in services] +
-          if helm_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_virtual_service(
+          if !istio_mode then [] else [if std.objectHas(srv, 'expose') && srv.expose then kubernetes.define_virtual_service(
             metadata {
               name: srv.name,
               annotations: {
@@ -342,7 +343,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
               ],
             }
           ) else null for srv in services] +
-          if helm_mode then [] else [{
+          if !istio_mode then [] else [{
               apiVersion: 'security.istio.io/v1beta1',
               kind: 'RequestAuthentication',
               metadata: metadata {
@@ -363,7 +364,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
                 }] else [],
             },
           } for srv in services] +
-          if helm_mode then [] else [{
+          if !istio_mode then [] else [{
             apiVersion: 'security.istio.io/v1beta1',
             kind: 'AuthorizationPolicy',
             metadata: metadata {
@@ -388,7 +389,7 @@ local manifestYamlStream = function (value, indent_array_in_object=false, c_docu
               }],
             },
           } for srv in services] +
-          if helm_mode then [] else [kubernetes.define_destination_rule(metadata { name: srv.name }, {
+          if !istio_mode then [] else [kubernetes.define_destination_rule(metadata { name: srv.name }, {
             host: vshost(srv),
             trafficPolicy: {
               tls: {

ci/service_mappings.jsonnet

@@ -1,6 +1,7 @@
-# sync-ignore-file: true
+local local_domain = std.extVar("local_domain");
+
 {
-  local_domain: '.pdev.resf.localhost',
+  local_domain: local_domain,
   default_domain: '.build.resf.org',
   service_mappings: {
     'peridotserver-http': {

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/antchfx/xmlquery v1.3.6 // indirect
 	github.com/authzed/authzed-go v0.3.0
 	github.com/authzed/grpcutil v0.0.0-20211115181027-063820eb2511
-	github.com/aws/aws-sdk-go v1.36.12
+	github.com/aws/aws-sdk-go v1.44.129
 	github.com/cavaliergopher/rpm v1.2.0
 	github.com/coreos/go-oidc/v3 v3.0.0
 	github.com/fatih/color v1.12.0
@@ -37,7 +37,7 @@ require (
 	github.com/pelletier/go-toml v1.8.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.13.0
-	github.com/rocky-linux/srpmproc v0.3.16
+	github.com/rocky-linux/srpmproc v0.4.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
@@ -50,6 +50,7 @@ require (
 	go.temporal.io/sdk v1.13.1
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	google.golang.org/genproto v0.0.0-20211104193956-4c6863e31247
 	google.golang.org/grpc v1.44.0

go.sum

@@ -113,6 +113,8 @@ github.com/aws/aws-sdk-go v1.34.13/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/aws/aws-sdk-go v1.36.12 h1:YJpKFEMbqEoo+incs5qMe61n1JH3o4O1IMkMexLzJG8=
 github.com/aws/aws-sdk-go v1.36.12/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.44.129 h1:yld8Rc8OCahLtenY1mnve4w1jVeBu/rSiscGzodaDOs=
+github.com/aws/aws-sdk-go v1.44.129/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -663,6 +665,8 @@ github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
 github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
 github.com/rocky-linux/srpmproc v0.3.16 h1:kxJEiQsZ0DcMhX0vY482n82XvjPiP2WifxI3NYuyLLM=
 github.com/rocky-linux/srpmproc v0.3.16/go.mod h1:vWZzxPTfxh4pmfr5Mw20FyrqyKsbGHzDwOlN+W5EMpw=
+github.com/rocky-linux/srpmproc v0.4.1 h1:qcq7bGLplKbu+dSKQ9VBwcTao3OqPNb6rdKz58MCFLA=
+github.com/rocky-linux/srpmproc v0.4.1/go.mod h1:x8Z2wqhV2JqRnYMhYz3thOQkfsSWjJkyX8DVGDPOb48=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -981,6 +985,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

modulemd/BUILD.bazel

@@ -2,8 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "modulemd",
-    srcs = ["modulemd.go"],
+    srcs = [
+        "modulemd.go",
+        "v3.go",
+    ],
     importpath = "peridot.resf.org/modulemd",
     visibility = ["//visibility:public"],
-    deps = ["//vendor/gopkg.in/yaml.v3:yaml_v3"],
+    deps = [
+        "//vendor/github.com/go-git/go-billy/v5:go-billy",
+        "//vendor/gopkg.in/yaml.v3:yaml_v3",
+    ],
 )

modulemd/modulemd.go

@@ -32,6 +32,7 @@ package modulemd
 
 import (
 	"fmt"
+	"github.com/go-git/go-billy/v5"
 	"gopkg.in/yaml.v3"
 )
 
@@ -152,6 +153,11 @@ type ModuleMd struct {
 	Data     *Data  `yaml:"data,omitempty"`
 }
 
+type DetectVersionDocument struct {
+	Document string `yaml:"document,omitempty"`
+	Version  int    `yaml:"version,omitempty"`
+}
+
 type DefaultsData struct {
 	Module   string              `yaml:"module,omitempty"`
 	Stream   string              `yaml:"stream,omitempty"`
@@ -165,11 +171,71 @@ type Defaults struct {
 }
 
 func Parse(input []byte) (*ModuleMd, error) {
-	var ret ModuleMd
-	err := yaml.Unmarshal(input, &ret)
+	var detect DetectVersionDocument
+	err := yaml.Unmarshal(input, &detect)
 	if err != nil {
-		return nil, fmt.Errorf("error parsing modulemd: %s", err)
+		return nil, fmt.Errorf("error detecting document version: %s", err)
+	}
+
+	var ret ModuleMd
+
+	if detect.Version == 2 {
+		err = yaml.Unmarshal(input, &ret)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing modulemd: %s", err)
+		}
+	} else if detect.Version == 3 {
+		var v3 V3
+		err = yaml.Unmarshal(input, &v3)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing modulemd: %s", err)
+		}
+
+		ret = ModuleMd{
+			Document: v3.Document,
+			Version:  v3.Version,
+			Data: &Data{
+				Name:        v3.Data.Name,
+				Stream:      v3.Data.Stream,
+				Summary:     v3.Data.Summary,
+				Description: v3.Data.Description,
+				License: &License{
+					Module: v3.Data.License,
+				},
+				Xmd:        v3.Data.Xmd,
+				References: v3.Data.References,
+				Profiles:   v3.Data.Profiles,
+				Profile:    v3.Data.Profile,
+				API:        v3.Data.API,
+				Filter:     v3.Data.Filter,
+				BuildOpts: &BuildOpts{
+					Rpms:   v3.Data.Configurations[0].BuildOpts.Rpms,
+					Arches: v3.Data.Configurations[0].BuildOpts.Arches,
+				},
+				Components: v3.Data.Components,
+			},
+		}
 	}
 
 	return &ret, nil
 }
+
+func (m *ModuleMd) Marshal(fs billy.Filesystem, path string) error {
+	bts, err := yaml.Marshal(m)
+	if err != nil {
+		return err
+	}
+
+	_ = fs.Remove(path)
+	f, err := fs.Create(path)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(bts)
+	if err != nil {
+		return err
+	}
+	_ = f.Close()
+
+	return nil
+}

modulemd/v3.go

@@ -0,0 +1,62 @@
+// Copyright (c) All respective contributors to the Peridot Project. All rights reserved.
+// Copyright (c) 2021-2022 Rocky Enterprise Software Foundation, Inc. All rights reserved.
+// Copyright (c) 2021-2022 Ctrl IQ, Inc. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors
+// may be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+package modulemd
+
+type V3 struct {
+	Document string  `yaml:"document,omitempty"`
+	Version  int     `yaml:"version,omitempty"`
+	Data     *V3Data `yaml:"data,omitempty"`
+}
+
+type Configurations struct {
+	Context       string              `yaml:"context,omitempty"`
+	Platform      string              `yaml:"platform,omitempty"`
+	BuildRequires map[string][]string `yaml:"buildrequires,omitempty"`
+	Requires      map[string][]string `yaml:"requires,omitempty"`
+	BuildOpts     *BuildOpts          `yaml:"buildopts,omitempty"`
+}
+
+type V3Data struct {
+	Name           string                       `yaml:"name,omitempty"`
+	Stream         string                       `yaml:"stream,omitempty"`
+	Summary        string                       `yaml:"summary,omitempty"`
+	Description    string                       `yaml:"description,omitempty"`
+	License        []string                     `yaml:"license,omitempty"`
+	Xmd            map[string]map[string]string `yaml:"xmd,omitempty"`
+	Configurations []*Configurations            `yaml:"configurations,omitempty"`
+	References     *References                  `yaml:"references,omitempty"`
+	Profiles       map[string]*Profile          `yaml:"profiles,omitempty"`
+	Profile        map[string]*Profile          `yaml:"profile,omitempty"`
+	API            *API                         `yaml:"api,omitempty"`
+	Filter         *API                         `yaml:"filter,omitempty"`
+	Demodularized  *API                         `yaml:"demodularized,omitempty"`
+	Components     *Components                  `yaml:"components,omitempty"`
+}

peridot/builder/v1/workflow/BUILD.bazel

@@ -19,7 +19,6 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//apollo/rpmutils",
-        "//modulemd",
         "//peridot/composetools",
         "//peridot/db",
         "//peridot/db/models",
@@ -47,6 +46,7 @@ go_library(
         "//vendor/github.com/go-git/go-git/v5/storage/memory",
         "//vendor/github.com/gobwas/glob",
         "//vendor/github.com/google/uuid",
+        "//vendor/github.com/rocky-linux/srpmproc/modulemd",
         "//vendor/github.com/rocky-linux/srpmproc/pb",
         "//vendor/github.com/rocky-linux/srpmproc/pkg/data",
         "//vendor/github.com/rocky-linux/srpmproc/pkg/srpmproc",

peridot/builder/v1/workflow/arch.go

@@ -455,6 +455,10 @@ func (c *Controller) mockConfig(project *models.Project, packageVersion *models.
 	}
 
 	buildMacros := c.buildMacros(project, packageVersion)
+	if extra != nil && extra.ForceDist != "" {
+		buildMacros["%dist"] = "." + extra.ForceDist
+	}
+
 	mockConfig := `
 config_opts['root'] = '{additionalVendor}-{majorVersion}-{hostArch}'
 config_opts['target_arch'] = '{arch}'