Merge pull request #16391 from MinaProtocol/dkijania/use_new_apt_repo…

…_dev

allow to use new debian repos

buildkite/scripts/debian/promote.sh

@@ -12,6 +12,8 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   -s|--from-component) FROM_COMPONENT="$2"; shift;;
   -t|--to-component) TO_COMPONENT="$2"; shift;;
   --new-name) NEW_NAME="$2"; shift;;
+  --new-repo) NEW_REPO="$2"; shift;;
+  -r|--repo) REPO="$2"; shift;;
   --repo-key) REPO_KEY="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
@@ -29,6 +31,8 @@ function usage() {
   echo "  -s, --from-component   The source channel in which package currently resides"
   echo "  -t, --to-component     The target channel for package (unstable, alpha, beta etc.)"
   echo "  -c, --codename         The Debian codename (bullseye, focal etc.)"
+  echo "  -r, --repo             The Debian source repo"
+  echo "  --new-repo             The Debian target repo. By default equal to repo"
   echo "  --repo-key         The Debian target repo key"
   echo ""
   echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly"
@@ -42,6 +46,8 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi;
 if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi;
 if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi;
 if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi;
+if [[ -z "$REPO" ]]; then usage "Repository is not set!"; fi;
+if [[ -z "$NEW_REPO" ]]; then NEW_REPO=$REPO; fi;
 if [[ -z "$REPO_KEY" ]]; then usage "Target repository key is not set!"; fi;
 
 # check for AWS Creds
@@ -69,5 +75,7 @@ else
     --suite $FROM_COMPONENT \
     --new-suite $TO_COMPONENT \
     --new-name $NEW_NAME \
+    --repo $REPO \
+    --new-repo $NEW_REPO
     --sign $REPO_KEY
 fi

buildkite/scripts/debian/publish.sh

@@ -28,4 +28,5 @@ source scripts/debian/publish.sh \
   --release $MINA_DEB_RELEASE \
   --version $MINA_DEB_VERSION \
   --codename $MINA_DEB_CODENAME \
+  --bucket $BUCKET \
   $SIGN_ARG

buildkite/src/Command/DockerImage.dhall

@@ -113,18 +113,23 @@ let generateStep =
                 ++  " --deb-build-flags ${BuildFlags.lowerName
                                             spec.build_flags}"
 
+          let remoteRepoCmds =
+                [ Cmd.run
+                    (     exportMinaDebCmd
+                      ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
+                      ++  " && "
+                      ++  buildDockerCmd
+                      ++  " && "
+                      ++  releaseDockerCmd
+                    )
+                ]
+
           let commands =
                 merge
-                  { PackagesO1Test =
-                    [ Cmd.run
-                        (     exportMinaDebCmd
-                          ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
-                          ++  " && "
-                          ++  buildDockerCmd
-                          ++  " && "
-                          ++  releaseDockerCmd
-                        )
-                    ]
+                  { PackagesO1Test = remoteRepoCmds
+                  , Unstable = remoteRepoCmds
+                  , Nightly = remoteRepoCmds
+                  , Stable = remoteRepoCmds
                   , Local =
                     [ Cmd.run
                         (     exportMinaDebCmd

buildkite/src/Command/MinaArtifact.dhall

@@ -142,6 +142,7 @@ let publish_to_debian_repo =
                                               spec.channel}"
                       ]
                     # DebianRepo.keyIdEnvList spec.debianRepo
+                    # DebianRepo.bucketEnvList spec.debianRepo
                   )
                   "./buildkite/scripts/debian/publish.sh"
             , label =

buildkite/src/Command/PromotePackage.dhall

@@ -59,7 +59,7 @@ let PromoteDebianSpec =
           , network = Network.Type.Devnet
           , codename = DebianVersions.DebVersion.Bullseye
           , from_channel = DebianChannel.Type.Unstable
-          , to_channel = DebianChannel.Type.NightlyDevelop
+          , to_channel = DebianChannel.Type.Develop
           , profile = Profiles.Type.Standard
           , remove_profile_from_name = False
           , step_key = "promote-debian-package"

buildkite/src/Command/Promotion/PromoteDebian.dhall

@@ -12,6 +12,8 @@ let Profiles = ../../Constants/Profiles.dhall
 
 let DebianVersions = ../../Constants/DebianVersions.dhall
 
+let DebianRepo = ../../Constants/DebianRepo.dhall
+
 let Toolchain = ../../Constants/Toolchain.dhall
 
 let Command = ../Base.dhall
@@ -29,6 +31,8 @@ let PromoteDebianSpec =
           , codename : DebianVersions.DebVersion
           , from_channel : DebianChannel.Type
           , to_channel : DebianChannel.Type
+          , source_repo : DebianRepo.Type
+          , target_repo : DebianRepo.Type
           , profile : Profiles.Type
           , remove_profile_from_name : Bool
           , step_key : Text
@@ -43,7 +47,9 @@ let PromoteDebianSpec =
           , network = Network.Type.Devnet
           , codename = DebianVersions.DebVersion.Bullseye
           , from_channel = DebianChannel.Type.Unstable
-          , to_channel = DebianChannel.Type.NightlyCompatible
+          , to_channel = DebianChannel.Type.Compatible
+          , source_repo = DebianRepo.Type.Unstable
+          , target_repo = DebianRepo.Type.Nightly
           , profile = Profiles.Type.Standard
           , remove_profile_from_name = False
           , step_key = "promote-debian-package"
@@ -76,10 +82,12 @@ let promoteDebianStep =
                       , "AWS_SECRET_ACCESS_KEY"
                       , "FROM_VERSION_MANUAL"
                       ]
-                      ". ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/debian/promote.sh --package ${package_name} --version ${spec.version}  --new-version ${spec.new_version}  --architecture ${spec.architecture}  --codename ${DebianVersions.lowerName
-                                                                                                                                                                                                                                                         spec.codename}  --from-component ${DebianChannel.lowerName
-                                                                                                                                                                                                                                                                                              spec.from_channel}  --to-component ${DebianChannel.lowerName
-                                                                                                                                                                                                                                                                                                                                     spec.to_channel} ${new_name}"
+                      ". ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/debian/promote.sh --repo ${DebianRepo.bucket_or_default
+                                                                                                                        spec.source_repo} --new-repo ${DebianRepo.bucket_or_default
+                                                                                                                                                         spec.target_repo} --package ${package_name} --version ${spec.version}  --new-version ${spec.new_version}  --architecture ${spec.architecture}  --codename ${DebianVersions.lowerName
+                                                                                                                                                                                                                                                                                                                       spec.codename}  --from-component ${DebianChannel.lowerName
+                                                                                                                                                                                                                                                                                                                                                            spec.from_channel}  --to-component ${DebianChannel.lowerName
+                                                                                                                                                                                                                                                                                                                                                                                                   spec.to_channel} ${new_name}"
                 , label = "Debian: ${spec.step_key}"
                 , key = spec.step_key
                 , target = Size.Small

buildkite/src/Command/Promotion/PromotePackages.dhall

@@ -58,7 +58,7 @@ let PromotePackagesSpec =
           , network = Network.Type.Mainnet
           , codenames = [] : List DebianVersions.DebVersion
           , from_channel = DebianChannel.Type.Unstable
-          , to_channel = DebianChannel.Type.NightlyCompatible
+          , to_channel = DebianChannel.Type.Compatible
           , new_tags = [] : List Text
           , remove_profile_from_name = False
           , publish = False

buildkite/src/Command/Promotion/VerifyDebian.dhall

@@ -2,6 +2,8 @@ let Package = ../../Constants/DebianPackage.dhall
 
 let DebianChannel = ../../Constants/DebianChannel.dhall
 
+let DebianRepo = ../../Constants/DebianRepo.dhall
+
 let Profiles = ../../Constants/Profiles.dhall
 
 let DebianVersions = ../../Constants/DebianVersions.dhall
@@ -30,9 +32,10 @@ let promoteDebianVerificationStep =
                 Command.Config::{
                 , commands =
                   [ Cmd.run
-                      "source buildkite/scripts/export-git-env-vars.sh && ./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName
-                                                                                                                                                                    spec.codename}  --channel ${DebianChannel.lowerName
-                                                                                                                                                                                                  spec.to_channel}"
+                      "source buildkite/scripts/export-git-env-vars.sh && ./scripts/debian/verify.sh --bucket ${DebianRepo.bucket_or_default
+                                                                                                                  spec.target_repo}--package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName
+                                                                                                                                                                                                  spec.codename}  --channel ${DebianChannel.lowerName
+                                                                                                                                                                                                                                spec.to_channel}"
                   ]
                 , label = "Debian: ${spec.step_key}"
                 , key = spec.step_key

buildkite/src/Command/Promotion/VerifyPackages.dhall

@@ -58,7 +58,7 @@ let VerifyPackagesSpec =
           , profile = Profiles.Type.Standard
           , network = Network.Type.Mainnet
           , codenames = [] : List DebianVersions.DebVersion
-          , channel = DebianChannel.Type.NightlyCompatible
+          , channel = DebianChannel.Type.Compatible
           , new_tags = [] : List Text
           , remove_profile_from_name = False
           , published = False

buildkite/src/Constants/DebianChannel.dhall

@@ -1,8 +1,8 @@
 let Channel
     : Type
     = < Unstable
-      | NightlyDevelop
-      | NightlyCompatible
+      | Develop
+      | Compatible
       | Itn
       | Umt
       | UmtMainnet
@@ -17,8 +17,8 @@ let capitalName =
           \(channel : Channel)
       ->  merge
             { Unstable = "Unstable"
-            , NightlyDevelop = "NightlyDevelop"
-            , NightlyCompatible = "NightlyCompatible"
+            , Develop = "Develop"
+            , Compatible = "Compatible"
             , Itn = "Itn"
             , Umt = "Umt"
             , UmtMainnet = "UmtMainnet"
@@ -34,8 +34,8 @@ let lowerName =
           \(channel : Channel)
       ->  merge
             { Unstable = "unstable"
-            , NightlyDevelop = "nightly-develop"
-            , NightlyCompatible = "nightly-compatible"
+            , Develop = "develop"
+            , Compatible = "compatible"
             , Itn = "itn"
             , Umt = "umt"
             , UmtMainnet = "umt-mainnet"

buildkite/src/Constants/DebianRepo.dhall

@@ -8,20 +8,28 @@ let Optional/toList = Prelude.Optional.toList
 
 let DebianRepo
     : Type
-    = < Local | PackagesO1Test >
+    = < Local | PackagesO1Test | Unstable | Nightly | Stable >
 
 let address =
           \(repo : DebianRepo)
       ->  merge
             { Local = "http://localhost:8080"
             , PackagesO1Test = "http://packages.o1test.net"
+            , Unstable = "https://unstable.apt.packages.minaprotocol.com"
+            , Nightly = "https://nightly.apt.packages.minaprotocol.com"
+            , Stable = "https://stable.apt.packages.minaprotocol.com"
             }
             repo
 
 let bucket =
           \(repo : DebianRepo)
       ->  merge
-            { Local = None Text, PackagesO1Test = Some "packages.o1test.net" }
+            { Local = None Text
+            , PackagesO1Test = Some "packages.o1test.net"
+            , Unstable = Some "unstable.apt.packages.minaprotocol.com"
+            , Nightly = Some "nightly.apt.packages.minaprotocol.com"
+            , Stable = Some "stable.apt.packages.minaprotocol.com"
+            }
             repo
 
 let bucket_or_default =
@@ -48,11 +56,27 @@ let bucketArg =
 
 let keyId =
           \(repo : DebianRepo)
-      ->  merge { Local = None Text, PackagesO1Test = None Text } repo
+      ->  merge
+            { Local = None Text
+            , PackagesO1Test = None Text
+            , Unstable = Some "B40D16B1A4773DE415DAF9DBFE236881C07523DC"
+            , Nightly = Some "B40D16B1A4773DE415DAF9DBFE236881C07523DC"
+            , Stable = Some "B40D16B1A4773DE415DAF9DBFE236881C07523DC"
+            }
+            repo
 
 let keyAddress =
           \(repo : DebianRepo)
-      ->  merge { Local = None Text, PackagesO1Test = None Text } repo
+      ->  let keyPath = "/key.asc"
+
+          in  merge
+                { Local = None Text
+                , PackagesO1Test = None Text
+                , Unstable = Some (address repo ++ keyPath)
+                , Nightly = Some (address repo ++ keyPath)
+                , Stable = Some (address repo ++ keyPath)
+                }
+                repo
 
 let keyAddressArg =
           \(repo : DebianRepo)
@@ -87,25 +111,25 @@ let keyIdEnvList =
 
           in  Optional/toList Text maybeKey
 
-let bucketEnv =
+let bucketEnvList =
           \(repo : DebianRepo)
-      ->  let maybeKey =
+      ->  let maybeBucket =
                 Optional/map
                   Text
                   Text
                   (\(repo : Text) -> "BUCKET=" ++ repo)
                   (bucket repo)
 
-          in  Optional/default Text "" maybeKey
+          in  Optional/toList Text maybeBucket
 
 in  { Type = DebianRepo
     , keyIdEnvList = keyIdEnvList
+    , bucketEnvList = bucketEnvList
     , keyAddressArg = keyAddressArg
     , address = address
     , bucket = bucket
     , bucket_or_default = bucket_or_default
     , bucketArg = bucketArg
-    , bucketEnv = bucketEnv
     , keyId = keyId
     , keyArg = keyArg
     }

buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall

@@ -44,7 +44,7 @@ let promotePackages =
       , codenames =
         [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ]
       , from_channel = DebianChannel.Type.Unstable
-      , to_channel = DebianChannel.Type.NightlyCompatible
+      , to_channel = DebianChannel.Type.Compatible
       , new_tags =
         [ "latest-compatible-nightly"
         , "compatible-nightly-\\\$(date \"+%Y%m%d\")"
@@ -67,7 +67,7 @@ let verifyPackages =
       , network = Network.Type.Devnet
       , codenames =
         [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ]
-      , channel = DebianChannel.Type.NightlyCompatible
+      , channel = DebianChannel.Type.Compatible
       , new_tags =
         [ "latest-compatible-nightly"
         , "compatible-nightly-\\\$(date \"+%Y%m%d\")"

scripts/debian/clear-s3-lockfile.sh

@@ -2,11 +2,12 @@
 
 MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=unstable}
 MINA_DEB_RELEASE=${MINA_DEB_RELEASE:=main}
+MINA_DEB_BUCKET=${MINA_DEB_BUCKET:=packages.o1test.net}
 
-S3_LOCKFILE_DATE="$(aws s3 ls s3://packages.o1test.net/dists/${MINA_DEB_CODENAME}/${MINA_DEB_RELEASE}/binary-/lockfile | awk '{print $1 " " $2}')"
+S3_LOCKFILE_DATE="$(aws s3 ls s3://${MINA_DEB_BUCKET}/dists/${MINA_DEB_CODENAME}/${MINA_DEB_RELEASE}/binary-/lockfile | awk '{print $1 " " $2}')"
 if [[ "$S3_LOCKFILE_DATE" == "" ]]; then
     echo "Could not get the lockfile timestamp from the S3 bucket. Have you set your AWS credentials correctly?"
-    aws s3 rm s3://packages.o1test.net/dists/${MINA_DEB_CODENAME}/${MINA_DEB_RELEASE}/binary-/lockfile
+    aws s3 rm s3://${MINA_DEB_BUCKET}/dists/${MINA_DEB_CODENAME}/${MINA_DEB_RELEASE}/binary-/lockfile
     echo "Lockfile deleted anyway"
     exit 1
 fi
@@ -15,7 +16,7 @@ NOW_SECONDS=$(date +%s)
 TIME_DIFF=$(($NOW_SECONDS - $S3_LOCKFILE_SECONDS))
 if [[ $TIME_DIFF > 300 ]]; then
     echo "Lockfile has been held for > 5 mins. The deb-s3 instance is likely to have died. Deleting lockfile.."
-    aws s3 rm s3://packages.o1test.net/dists/${MINA_DEB_CODENAME}/${MINA_DEB_RELEASE}/binary-/lockfile
+    aws s3 rm s3://${MINA_DEB_BUCKET}/dists/${MINA_DEB_CODENAME}/${MINA_DEB_RELEASE}/binary-/lockfile
     echo "Lockfile deleted"
 else
     echo "Lockfile is younger than 5 mins. There may be a deb-s3 instance actively using it. Refusing to delete."

scripts/debian/publish.sh

@@ -5,12 +5,14 @@ CLEAR='\033[0m'
 RED='\033[0;31m'
 
 ARCH=amd64
+BUCKET=packages.o1test.net
 
 while [[ "$#" -gt 0 ]]; do case $1 in
   -n|--names) DEB_NAMES="$2"; shift;;
   -r|--release) DEB_RELEASE="$2"; shift;;
   -v|--version) DEB_VERSION="$2"; shift;;
   -c|--codename) DEB_CODENAME="$2"; shift;;
+  -b|--bucket) BUCKET="$2"; shift;;
   -s|--sign) SIGN="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
@@ -22,6 +24,7 @@ function usage() {
   echo "Usage: $0 -n names -r release -v version -c codename"
   echo "  -n, --names         The Debians archive names"
   echo "  -r, --release       The Debian release"
+  echo "  -b, --bucket        The Bucket which holds debian repo"
   echo "  -v, --version       The Debian version"
   echo "  -c, --codename      The Debian codename"
   echo "  -s, --sign          The Debian key id used for sign"
@@ -42,7 +45,7 @@ else
   SIGN_ARG="--sign=$SIGN"
 fi
 
-BUCKET_ARG="--bucket=packages.o1test.net"
+BUCKET_ARG="--bucket=$BUCKET"
 S3_REGION_ARG="--s3-region=us-west-2"
 # utility for publishing deb repo with commons options
 # deb-s3 https://github.com/krobertson/deb-s3
@@ -71,7 +74,7 @@ echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${D
 # If this fails, attempt to remove the lockfile and retry.
 for _ in {1..10}; do (
   ${DEBS3_UPLOAD} --component "${DEB_RELEASE}" --codename "${DEB_CODENAME}" "${GPG_OPTS[@]}" "${DEB_NAMES}"
-) && break || scripts/debian/clear-s3-lockfile.sh; done
+) && break || (MINA_DEB_BUCKET=${BUCKET} scripts/debian/clear-s3-lockfile.sh); done
 
 for deb in $DEB_NAMES
 do