feat: new repository source interface (#86)

* feat: new repository source interface

* chore: self mutation

Signed-off-by: github-actions <github-actions@github.com>

* fix: lock NPM_LICENSE_CHECKER_TOOL version to 4.3.0

* fix: python support was missing yaml in the bundle

* fix: license checker has to use the projectRoot

---------

Signed-off-by: github-actions <github-actions@github.com>
Co-authored-by: github-actions <github-actions@github.com>

.projenrc.ts

@@ -95,6 +95,7 @@ const root = new yarn.Monorepo({
     'node_modules',
     '.env',
     '.env.*',
+    '.venv',
   ],
 });
 
@@ -167,6 +168,7 @@ pipeline.tasks
 // Copy bundle dependencies to the package
 const postCompile = pipeline.tasks.tryFind('post-compile')!;
 postCompile.exec("export DEP='@cloudcomponents';cp -rf ../../../node_modules/$DEP ./node_modules/ 2>/dev/null;");
+postCompile.exec("export DEP='yaml';cp -rf ../../../node_modules/$DEP ./node_modules/ 2>/dev/null;");
 
 //============================================
 //

OSS_License_Summary.csv

@@ -14,6 +14,6 @@
 "CC-BY-4.0","1"
 "CC0-1.0","2"
 "ISC","78"
-"MIT","661"
+"MIT","660"
 "Python-2.0","1"
 "UNLICENSED","1"

Taskfile.codeartifact.yml

@@ -59,7 +59,7 @@ tasks:
     internal: true
     cmds:
       - task: publish:pipeline:js
-      # - task: publish:pipeline:python
+      - task: publish:pipeline:python
 
   publish:pipeline:js:
     desc: Publish the pipeline package locally
@@ -123,8 +123,8 @@ tasks:
     cmds:
       - test -f .npmrc && rm -rf ./.npmrc && echo "[Warning] Project level .npmrc file has been removed." || true
       - aws codeartifact {{with .AWS_PROFILE}} --profile {{.}} {{end}}login --domain {{.DOMAIN}} --repository {{.REPOSITORY}} {{with .AWS_REGION}} --region {{.}} {{end}} --namespace cdklabs --tool npm
-      # - aws codeartifact {{with .AWS_PROFILE}} --profile {{.}} {{end}}login --domain {{.DOMAIN}} --repository {{.REPOSITORY}} {{with .AWS_REGION}} --region {{.}} {{end}} --tool pip
-      # - aws codeartifact {{with .AWS_PROFILE}} --profile {{.}} {{end}}login --domain {{.DOMAIN}} --repository {{.REPOSITORY}} {{with .AWS_REGION}} --region {{.}} {{end}} --tool twine
+      - aws codeartifact {{with .AWS_PROFILE}} --profile {{.}} {{end}}login --domain {{.DOMAIN}} --repository {{.REPOSITORY}} {{with .AWS_REGION}} --region {{.}} {{end}} --tool pip
+      - aws codeartifact {{with .AWS_PROFILE}} --profile {{.}} {{end}}login --domain {{.DOMAIN}} --repository {{.REPOSITORY}} {{with .AWS_REGION}} --region {{.}} {{end}} --tool twine
       - task: set-token-secret
     silent: true
     preconditions:

devbox.json

@@ -1,27 +1,33 @@
 {
-    "$schema": "https://raw.githubusercontent.com/jetpack-io/devbox/0.10.3/.schema/devbox.schema.json",
-    "packages": [
-      "git@2.42.0",
-      "git-remote-codecommit@1.15.1",
-      "jq@1.7.1",
-      "go-task@3.35.1",
-      "python@3.12.2",
-      "pipenv@2023.2.4",
-      "nodejs@18.19.1"
+  "$schema": "https://raw.githubusercontent.com/jetpack-io/devbox/0.10.3/.schema/devbox.schema.json",
+  "packages": [
+    "git@2.42.0",
+    "git-remote-codecommit@1.15.1",
+    "jq@1.7.1",
+    "go-task@3.35.1",
+    "python@3.12.2",
+    "pipenv@2023.2.4",
+    "nodejs@18.19.1",
+    "twine@latest"
+  ],
+  "shell": {
+    "init_hook": [
+      "echo 'Welcome to CDKLabs CDK CI/CD Wrapper'",
+      "echo 'You can run the following commands:'",
+      ". $VENV_DIR/bin/activate",
+      "task -l",
+      "echo 'For more information, see Readme.md'",
+      "task init"
     ],
-    "shell": {
-      "init_hook": [
-        "echo 'Welcome to CDKLabs CDK CI/CD Wrapper'",
-        "echo 'You can run the following commands:'",
-        "task -l",
-        "echo 'For more information, see Readme.md'",
-        "task init"
-      ],
-      "scripts": {
-        "test": [
-          "echo \"Error: no test specified\" && exit 1"
-        ]
-      }
+    "scripts": {
+      "test": [
+        "echo \"Error: no test specified\" && exit 1"
+      ]
     }
+  },
+  "env": {
+    "VENV_DIR":                ".venv",
+    "PIPENV_VERBOSITY":        "-1",
+    "DEVBOX_COREPACK_ENABLED": "1"
   }
-  
+}

devbox.lock

@@ -502,6 +502,70 @@
           "store_path": "/nix/store/7yh2ax34jd7fgf17mjfd3c6niw1h2hsj-python3-3.12.2"
         }
       }
+    },
+    "twine@latest": {
+      "last_modified": "2024-08-27T08:49:48Z",
+      "resolved": "github:NixOS/nixpkgs/a6292e34000dc93d43bccf78338770c1c5ec8a99#twine",
+      "source": "devbox-search",
+      "version": "5.1.0",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/2qlcms1l0lh8cxjmryg1jbnbv0mxih4r-python3.12-twine-5.1.0",
+              "default": true
+            },
+            {
+              "name": "dist",
+              "path": "/nix/store/ya9s86ab0wjry6d6pq8izdghzbrq54w5-python3.12-twine-5.1.0-dist"
+            }
+          ],
+          "store_path": "/nix/store/2qlcms1l0lh8cxjmryg1jbnbv0mxih4r-python3.12-twine-5.1.0"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/w8k56bsfncdc67rl195aq98hbhgvjylx-python3.12-twine-5.1.0",
+              "default": true
+            },
+            {
+              "name": "dist",
+              "path": "/nix/store/33118r1fr0j9r3p6srpql7fpd1aha487-python3.12-twine-5.1.0-dist"
+            }
+          ],
+          "store_path": "/nix/store/w8k56bsfncdc67rl195aq98hbhgvjylx-python3.12-twine-5.1.0"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/aplcp7jzsxbnmxhzdldczygpv56gify8-python3.12-twine-5.1.0",
+              "default": true
+            },
+            {
+              "name": "dist",
+              "path": "/nix/store/8zq56pfskha9gdmks6dzkjv8capj4wz2-python3.12-twine-5.1.0-dist"
+            }
+          ],
+          "store_path": "/nix/store/aplcp7jzsxbnmxhzdldczygpv56gify8-python3.12-twine-5.1.0"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/7gxvyicpv5xhf62rhp9clpjk5wyxnzmx-python3.12-twine-5.1.0",
+              "default": true
+            },
+            {
+              "name": "dist",
+              "path": "/nix/store/70j2cmnk5fqfang6mx5h50n8pc5ghvl5-python3.12-twine-5.1.0-dist"
+            }
+          ],
+          "store_path": "/nix/store/7gxvyicpv5xhf62rhp9clpjk5wyxnzmx-python3.12-twine-5.1.0"
+        }
+      }
     }
   }
 }

docs/content/developer_guides/cd.md

@@ -74,15 +74,15 @@ PipelineBlueprint.builder().addStack({
         applicationName: context.blueprintProps.applicationName,
         stageName: context.stage,
     });
-}}, vp.COMMON_STAGES.DEV).addStack({
+}}, Stage.DEV).addStack({
   provide: (context) => {
     new example.S3BucketStack(context.scope, `${context.blueprintProps.applicationName}S3Stack`, {
         bucketName: 'test-bucket',
         stageName: context.stage,
         applicationQualifier: context.blueprintProps.applicationQualifier,
         encryptionKey: context.get(GlobalResources.ENCRYPTION)!.kmsKey,
     });
-}}, vp.COMMON_STAGES.INT, vp.COMMON_STAGES.PROD).synth(app);
+}}, Stage.INT, Stage.PROD).synth(app);
 ```
 
 With this configuration the `LambdaStack` will be deployed in the DEV stage only and not in the INT and PROD stages where as the `S3BucketStack` will be deployed in the INT and PROD stages.
@@ -97,10 +97,10 @@ You can define custom stages through the VanillaPipelineBuilder, so that you can
 ```typescript
 PipelineBlueprint.builder()
     .defineStages([
-        vp.COMMON_STAGES.RES,
+        Stage.RES,
         { stage: 'EXP', account: '1234567891012', region: 'eu-west-1'},
-        { stage: vp.COMMON_STAGES.DEV, account: '2345678910123', region: 'eu-west-1'},
-        { stage: vp.COMMON_STAGES.INT}
+        { stage: Stage.DEV, account: '2345678910123', region: 'eu-west-1'},
+        { stage: Stage.INT}
     ])
     .addStack({
   provide: (context) => {
@@ -116,7 +116,7 @@ PipelineBlueprint.builder()
             applicationQualifier: context.blueprintProps.applicationQualifier,
             encryptionKey: context.get(GlobalResources.ENCRYPTION)!.kmsKey,
         });
-    }}, vp.COMMON_STAGES.INT, vp.COMMON_STAGES.PROD).synth(app);
+    }}, Stage.INT, Stage.PROD).synth(app);
 ```
 
 With this example you can see the options of how the Stages can be defined.

docs/content/developer_guides/global_resource.md

@@ -9,7 +9,7 @@ The benefit of the dependency injection is that the stack dependency implementat
 Let's see an example:
 
 ```typescript
-new vp.S3BucketStack(context.scope, `${context.blueprintProps.applicationName}S3Stack`, {
+new S3BucketStack(context.scope, `${context.blueprintProps.applicationName}S3Stack`, {
     bucketName: 'test-bucket',
     stageName: context.stage,
     applicationQualifier: context.blueprintProps.applicationQualifier,

package-verification.json

@@ -1,11 +1,11 @@
 {
   "license": {
-    "package.json": "2fa6e73426cf7478f18abdad18c34aab0e7cc00a9c7be587a820772228ccb927",
+    "package.json": "27eb954737b58156e44e11e20987a453921d49bc602480b2df850cebcbdc9ead",
     "samples/cdk-ts-example/package.json": "54e2441af495ddf02e1e232fdd1ed4edc697c31fe03671776290a254e52ba2a6",
-    "packages/@cdklabs/cdk-cicd-wrapper/package.json": "5a037128d50a11a17cfddff266a40b2c939001931e30c27a2e0ad4cf8052186b",
-    "packages/@cdklabs/cdk-cicd-wrapper-cli/package.json": "56ad46900253b2fd1c44d965bdb0c40975ef5035dccff7be73c4864ccba8f57b",
-    "packages/@cdklabs/cdk-cicd-wrapper-projen/package.json": "8d45188ba2b0e57ba00df30f1cc2931032353a42c2da1171a708fb339b719b98",
+    "packages/@cdklabs/cdk-cicd-wrapper/package.json": "7cee38ba7a290df197937217245c7f333e3cc7c02cc6e56de51a314ceaa16d1b",
+    "packages/@cdklabs/cdk-cicd-wrapper-cli/package.json": "dd2779775dd746a96beee3fb8bf74b2aa02d0b9e93336ec9fdf6c3e90248ea99",
+    "packages/@cdklabs/cdk-cicd-wrapper-projen/package.json": "fa9a0a02cf185d82d98680f8a657ee7baa87542ddc35f164eb0c44acafe6358d",
     "projectList": "5790138ec76401840b75a0409a66d8e7e10445628567d18dbe5b2feedd20f463"
   },
-  "npm-lock-file": "4f8e486be973f8b92a779e2491d525290be5ffb230129f934662e0b7a4f5008c"
+  "npm-lock-file": "3b26da52d4e5d1c84e560562bcaeedbd2e88e40eb5ceb7c9cc81ae2afd5bee1c"
 }

packages/@cdklabs/cdk-cicd-wrapper-cli/src/cmds/ValidateCommand.ts

@@ -43,7 +43,7 @@ class Command implements yargs.CommandModule {
   static validateChecksum(filePath: string, expectedHash: string) {
     const hexCheckSum = CliHelpers.generateChecksum(filePath);
     if (hexCheckSum !== expectedHash) {
-      logger.info(
+      logger.warn(
         `File at ${filePath} has checksum ${hexCheckSum}, which does not match expected value ${expectedHash}`,
       );
       logger.info('This likely means dependencies have updated. You must get the changes approved before proceeding');

packages/@cdklabs/cdk-cicd-wrapper-cli/src/cmds/license/NodeLicenseCollector.ts

@@ -8,7 +8,7 @@ import { parse, stringify } from 'csv/sync';
 import { LicenseCollector, LicenseConfig, ScanningContext } from './Types';
 import { logger } from '../../utils/Logging';
 
-const NPM_LICENSE_CHECKER_TOOL = 'license-checker-rseidelsohn';
+const NPM_LICENSE_CHECKER_TOOL = 'license-checker-rseidelsohn@4.3.0';
 
 enum ProjectType {
   NPM,

packages/@cdklabs/cdk-cicd-wrapper-cli/src/cmds/license/Projects.ts

@@ -21,7 +21,7 @@ export class Projects {
   private verificationFilePath: string;
 
   constructor(readonly config: LicenseConfig, readonly context: ScanningContext) {
-    this.verificationFilePath = path.resolve(this.context.workingDir, this.config.verificationFile);
+    this.verificationFilePath = path.resolve(this.context.projectRoot, this.config.verificationFile);
   }
 
   public getProjects(type: string) {
@@ -72,6 +72,8 @@ export class Projects {
       this.newLicensesSection[projectRelativePath] = currentHashCode;
       if (verifiedHashCode !== currentHashCode) {
         logger.info(`File ${projectFile} has changed since last scan.`);
+        logger.debug(`Old hash: ${verifiedHashCode}`);
+        logger.debug(`New hash: ${currentHashCode}`);
         result = true;
       }
 
@@ -85,6 +87,8 @@ export class Projects {
     this.newLicensesSection[LICENSE_FILES_SUMMARY_HASH] = currentHashCodeOfProjectFiles;
     if (verifiedHashCode !== currentHashCodeOfProjectFiles) {
       logger.info('Source of licenses has changed since last scan.');
+      logger.debug(`Old hash: ${verifiedHashCode}`);
+      logger.debug(`New hash: ${currentHashCodeOfProjectFiles}`);
       result = true;
     }
 

packages/@cdklabs/cdk-cicd-wrapper-cli/src/utils/Logging.ts

@@ -3,4 +3,8 @@
 
 import { Logger } from 'tslog';
 
-export const logger = new Logger({ name: 'CDK CI/CD CLI', minLevel: 4 });
+export const logger = new Logger({
+  name: 'CDK CI/CD CLI',
+  minLevel: 3,
+  prettyLogTemplate: '{{yyyy}}.{{mm}}.{{dd}} {{hh}}:{{MM}}:{{ss}}:{{ms}}\t{{logLevelName}}\t[{{name}}]\t',
+});

packages/@cdklabs/cdk-cicd-wrapper/src/common/types/Types.ts

@@ -5,6 +5,7 @@ import * as codebuild from 'aws-cdk-lib/aws-codebuild';
 import { Step } from 'aws-cdk-lib/pipelines';
 import { Construct } from 'constructs';
 import { PipelineOptions } from '../../code-pipeline';
+import { RepositorySource } from '../../resource-providers';
 import { IVpcConfig } from '../../resource-providers/VPCProvider';
 import { ResourceContext } from '../spi';
 
@@ -179,6 +180,11 @@ export interface IPipelineConfig {
    * Additional pipelineOptions.
    */
   pipelineOptions?: PipelineOptions;
+
+  /**
+   * The repository source for the pipeline.
+   */
+  repositorySource?: RepositorySource;
 }
 
 /**
@@ -369,6 +375,11 @@ export interface RepositoryConfig {
    * The description of the repository (optional).
    */
   readonly description?: string;
+
+  /**
+   * Enforce full clone for the repository.
+   */
+  readonly codeBuildCloneOutput?: boolean;
 }
 
 /**