build: add alternative build process to enable faster developer builds (

#22506) Closes MetaMask/MetaMask-planning#1477 and MetaMask/MetaMask-planning#1903 This commit adds an alternative build process that is much faster than the gulp build we have now, which is quite slow and doesn't make use of modern build system improvements. The speed up is possible by making use of the SWC compiler, and more modern build system. The build system is also a bit simpler and hopefully more maintainable. This build doesn't yet support: * HMR/chromereload (requires we get rid of our circular dependencies) * lavamoat (neither running the build system _in_ lavamoat, or adding lavamoat protections) * production builds (because of not supporting lavamoat) * MV3 (requires writing a webpack plugin) Co-authored-by: Howard Braham <howrad@gmail.com>
MetaMask · Aug 2, 2024 · 090cb0b · 090cb0b
1 parent e5651cf
commit 090cb0b
Show file tree

Hide file tree

Showing 80 changed files with 7,974 additions and 1,722 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -163,6 +163,9 @@ workflows:
       - prep-build-test-mv2:
           requires:
             - prep-deps
+      - prep-build-test-webpack:
+          requires:
+            - prep-deps
       - prep-build-test-flask:
           requires:
             - prep-deps
@@ -193,6 +196,10 @@ workflows:
       - test-lint-changelog:
           requires:
             - prep-deps
+      - test-e2e-chrome-webpack:
+          requires:
+            - prep-build-test-webpack
+            - get-changed-files-with-git-diff
       - test-e2e-chrome:
           requires:
             - prep-build-test
@@ -293,6 +300,7 @@ workflows:
             - test-e2e-chrome-mmi
             - test-e2e-chrome-rpc-mmi
             - test-e2e-chrome-vault-decryption
+            - test-e2e-chrome-webpack
             - test-storybook
       - benchmark:
           requires:
@@ -904,6 +912,26 @@ jobs:
             - dist-test-mv2
             - builds-test-mv2
 
+  prep-build-test-webpack:
+    executor: node-linux-medium
+    steps:
+      - run: *shallow-git-clone
+      - attach_workspace:
+          at: .
+      - run:
+          name: Activate yarn
+          command: corepack enable
+      - run:
+          name: Build extension for testing
+          command: yarn build:test:webpack
+      - run:
+          name: Move test build to 'dist-test-webpack' to avoid conflict with production build
+          command: mv ./dist ./dist-test-webpack
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist-test-webpack
+
   prep-build-storybook:
     executor: node-linux-medium
     steps:
@@ -1045,6 +1073,27 @@ jobs:
           name: depcheck
           command: yarn depcheck
 
+  test-e2e-chrome-webpack:
+    executor: node-browsers-medium-plus
+    parallelism: 20
+    steps:
+      - run: *shallow-git-clone
+      - run: sudo corepack enable
+      - attach_workspace:
+          at: .
+      - run:
+          name: Move test build to dist
+          command: mv ./dist-test-webpack ./dist
+      - run:
+          name: test:e2e:chrome:webpack
+          command: timeout 20m yarn test:e2e:chrome:webpack --retries 1
+          no_output_timeout: 5m
+      - store_artifacts:
+          path: test-artifacts
+          destination: test-artifacts
+      - store_test_results:
+          path: test/test-results/e2e
+
   test-api-specs:
     executor: node-browsers-medium-plus
     steps:

diff --git a/.depcheckrc.yml b/.depcheckrc.yml
@@ -58,6 +58,22 @@ ignores:
   - 'resolve-url-loader'
   # jest environments
   - 'jest-environment-jsdom'
+  # webpack
+  - '@pmmmwh/react-refresh-webpack-plugin' # dev tool
+  - 'webpack-dev-server' # dev tool
+  - 'html-bundler-webpack-plugin' # build tool
+  - 'postcss-loader' # build tool
+  - '@swc/helpers' # build tool
+  - browserslist # build tool
+  - 'buffer' # polyfill
+  - 'crypto-browserify' # polyfill
+  - 'process' # polyfill
+  - 'stream-http' # polyfill
+  - 'rimraf' # misc: install helper
+  - 'json-schema-to-ts' # misc: typescript helper
+  - 'https-browserify' # polyfill
+  - 'path-browserify' # polyfill
+  - 'nyc' # coverage
   # babel
   - '@babel/plugin-transform-logical-assignment-operators'
   # trezor

diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml
@@ -30,6 +30,26 @@ jobs:
           name: coverage-${{matrix.shard}}
           path: coverage/coverage-${{matrix.shard}}.json
 
+  test-webpack:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup environment
+        uses: metamask/github-tools/.github/actions/setup-environment@main
+
+      - name: test:unit:webpack:coverage
+        run: yarn test:unit:webpack:coverage
+
+      - name: Rename coverage
+        run: mv coverage/coverage-final.json coverage/coverage-webpack.json
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-webpack
+          path: coverage/coverage-webpack.json
+
   report-coverage:
     runs-on: ubuntu-latest
     needs:

diff --git a/.vscode/package.json-schema.json b/.vscode/package.json-schema.json
@@ -5,9 +5,22 @@
       "required": ["lavamoat"],
       "properties": {
         "scripts": {
+          "required": ["webpack", "webpack:clearcache", "postinstall"],
           "properties": {
             "start:dev": {
               "description": "Runs `yarn start` with the addition of react/redux backend devtool servers enabled."
+            },
+            "webpack": {
+              "type": "string",
+              "description": "Builds the extension in \"dev\" mode. Run `yarn webpack --help` for usage information."
+            },
+            "webpack:clearcache": {
+              "type": "string",
+              "description": "Deletes webpack's build cache. Useful to force a rebuild (webpack not detecting changes, node_modules have changed, etc)."
+            },
+            "postinstall": {
+              "type": "string",
+              "description": "Runs automatically after running `yarn` (`yarn install`) in order to prime the webpack dev build."
             }
           }
         },
@@ -16,6 +29,62 @@
           "properties": {
             "autoprefixer": {
               "description": "Used by our build systems to automatically add prefixes to CSS rules based on our browserslist."
+            },
+            "@types/chrome": {
+              "type": "string",
+              "description": "Provides type definitions for the Chrome extension manifest.json files."
+            },
+            "buffer": {
+              "type": "string",
+              "description": "Provides a global Buffer object for use in the browser (webpack)"
+            },
+            "crypto-browserify": {
+              "type": "string",
+              "description": "Polyfill's node's crypto API for use in the browser (webpack)"
+            },
+            "dotenv": {
+              "type": "string",
+              "description": "Loads environment variables from a .metamaskrc file (webpack)"
+            },
+            "fflate": {
+              "type": "string",
+              "description": "Provides zip capabilities for bundling (webpack)"
+            },
+            "postcss-loader": {
+              "type": "string",
+              "description": "Loads postcss plugins (webpack)"
+            },
+            "process": {
+              "type": "string",
+              "description": "Provides a global process object for use in the browser (webpack)"
+            },
+            "schema-utils": {
+              "type": "string",
+              "description": "Provides utilities for validating options objects (webpack)"
+            },
+            "stream-http": {
+              "type": "string",
+              "description": "Polyfill's node's stream API for use in the browser (webpack)"
+            },
+            "@swc/core": {
+              "type": "string",
+              "description": "Transpiles javascript and typescript (webpack)"
+            },
+            "tsx": {
+              "type": "string",
+              "description": "Provides blazing fast typescript compilation (no type checking) (webpack)"
+            },
+            "rimraf": {
+              "type": "string",
+              "description": "Provides a cross-platform way of deleting files from the command line (webpack)"
+            },
+            "json-schema-to-ts": {
+              "type": "string",
+              "description": "Generates typescript types from json schemas (webpack)"
+            },
+            "@pmmmwh/react-refresh-webpack-plugin": {
+              "type": "string",
+              "description": "Provides hot reloading for react components (webpack)"
             }
           }
         },

diff --git a/.yarn/patches/@metamask-keyring-controller-npm-17.1.1-098cb41930.patch b/.yarn/patches/@metamask-keyring-controller-npm-17.1.1-098cb41930.patch
@@ -0,0 +1,13 @@
+diff --git a/package.json b/package.json
+index 5a6217eaed16fdfe7f1ad693871f85320bd6b421..69bdf1a9155497e37fc58db7bbc74597fd543535 100644
+--- a/package.json
++++ b/package.json
+@@ -18,7 +18,7 @@
+   "sideEffects": false,
+   "exports": {
+     ".": {
+-      "import": "./dist/index.mjs",
++      "import": "./dist/index.js",
+       "require": "./dist/index.js",
+       "types": "./dist/types/index.d.ts"
+     },
diff --git a/.yarn/patches/@metamask-snaps-controllers-npm-8.4.0-574cd5a8a9.patch b/.yarn/patches/@metamask-snaps-controllers-npm-8.4.0-574cd5a8a9.patch
@@ -0,0 +1,13 @@
+diff --git a/package.json b/package.json
+index 6dedde043d1bd5fc195e72b3e06ec37cf6532476..3986b5b0c1f3bf7ff49e023c934bed26f44735ae 100644
+--- a/package.json
++++ b/package.json
+@@ -9,7 +9,7 @@
+   "sideEffects": false,
+   "exports": {
+     ".": {
+-      "import": "./dist/index.mjs",
++      "import": "./dist/index.js",
+       "require": "./dist/index.js",
+       "types": "./dist/types/index.d.ts"
+     },
diff --git a/.yarn/patches/@reduxjs-toolkit-npm-1.9.7-b14925495c.patch b/.yarn/patches/@reduxjs-toolkit-npm-1.9.7-b14925495c.patch
@@ -92,3 +92,29 @@ index bb433432ec76331e12d6b62e200f06530055cb16..9caf4051aa96bd14ee2890ef6c79bf5b
      return EnhancerArray;
  }(Array));
  function freezeDraftable(val) {
+diff --git a/dist/redux-toolkit.esm.js b/dist/redux-toolkit.esm.js
+index f26a1669405b4dd92dfecd791dc536078a7e2e12..591e7495fcaf3233d26cfb9c4eae09fd7ae3eb98 100644
+--- a/dist/redux-toolkit.esm.js
++++ b/dist/redux-toolkit.esm.js
+@@ -192,9 +192,6 @@ function getMessage(type) {
+ }
+ function createActionCreatorInvariantMiddleware(options) {
+     if (options === void 0) { options = {}; }
+-    if (process.env.NODE_ENV === "production") {
+-        return function () { return function (next) { return function (action) { return next(action); }; }; };
+-    }
+     var _c = options.isActionCreator, isActionCreator2 = _c === void 0 ? isActionCreator : _c;
+     return function () { return function (next) { return function (action) {
+         if (isActionCreator2(action)) {
+diff --git a/package.json b/package.json
+index 684ea845ee663f719bff6c140001baebdaa69344..568d6215514a8625bfb3be5e49b6cbfe11231e6a 100644
+--- a/package.json
++++ b/package.json
+@@ -23,7 +23,6 @@
+     "access": "public"
+   },
+   "main": "dist/index.js",
+-  "module": "dist/redux-toolkit.esm.js",
+   "unpkg": "dist/redux-toolkit.umd.min.js",
+   "types": "dist/index.d.ts",
+   "devDependencies": {
diff --git a/README.md b/README.md
@@ -85,6 +85,7 @@ To start a development build (e.g. with logging and file watching) run `yarn sta
 
 Alternatively, one can skip wallet onboarding and preload the vault state with a specific SRP by adding `TEST_SRP='<insert SRP here>'` and `PASSWORD='<insert wallet password here>'` to the `.metamaskrc` file and running `yarn start:skip-onboarding`.
 
+You can also start a development build using the `yarn webpack` command, or `yarn webpack --watch`. This uses an alternative build system that is much faster, but not yet production ready. See the [Webpack README](./development/webpack/README.md) for more information.
 
 #### React and Redux DevTools
 
@@ -138,7 +139,7 @@ Note: The `yarn start:test` command (which initiates the testDev build type) has
 Once you have your test build ready, choose the browser for your e2e tests:
 
 - For Firefox, run `yarn test:e2e:firefox`.
-  - Note:  If you are running Firefox as a snap package on Linux, ensure you enable the appropriate environment variable: `FIREFOX_SNAP=true yarn test:e2e:firefox`
+  - Note: If you are running Firefox as a snap package on Linux, ensure you enable the appropriate environment variable: `FIREFOX_SNAP=true yarn test:e2e:firefox`
 - For Chrome, run `yarn test:e2e:chrome`.
 
 These scripts support additional options for debugging. Use `--help`to see all available options.

diff --git a/app/background.html b/app/background.html
@@ -4,7 +4,6 @@
     <meta charset="utf-8">
   </head>
   <body>
-    <script src="./load-background.js"></script>
-    <script src="./chromereload.js"></script>
+    <script src="./scripts/load/background.ts" defer></script>
   </body>
 </html>
diff --git a/app/home.html b/app/home.html
@@ -8,14 +8,14 @@
     <% } else { %>
     <title>MetaMask</title>
     <% } %>
-    <link rel="stylesheet" href="./index.css">
+    <link rel="stylesheet" href="../ui/css/index.scss">
   </head>
   <body>
     <div id="app-content">
       <img class="loading-logo" src="./images/logo/metamask-fox.svg" alt="" loading="lazy" />
       <img class="loading-spinner" src="./images/spinner.gif" alt="" loading="lazy" />
     </div>
     <div id="popover-content"></div>
-    <script src="./load-app.js" async></script>
+    <script src="./scripts/load/ui.ts" async></script>
   </body>
 </html>
diff --git a/app/loading.html b/app/loading.html
@@ -2,13 +2,13 @@
 <html lang="en" dir="ltr">
 <head>
   <meta charset="UTF-8">
-  <% if (it.shouldIncludeSnow) { %>
-  <script src="./scripts/snow.js"></script>
-  <script src="./scripts/use-snow.js"></script>
-  <% } %>
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="X-UA-Compatible" content="ie=edge">
   <title>MetaMask Loading</title>
+  <% if (it.shouldIncludeSnow) { %>
+  <script src="@lavamoat/snow/snow.prod.js"></script>
+  <script src="./scripts/use-snow.js"></script>
+  <% } %>
   <style>
     #div-logo {
       position: absolute;

diff --git a/app/manifest/v2/brave.json b/app/manifest/v2/brave.json
@@ -1 +1,7 @@
-{}
+{
+  "content_security_policy": "frame-ancestors 'none'; script-src 'self' 'wasm-unsafe-eval'; object-src 'none'",
+  "externally_connectable": {
+    "matches": ["https://metamask.io/*"],
+    "ids": ["*"]
+  }
+}
diff --git a/app/notification.html b/app/notification.html
@@ -32,7 +32,7 @@
         margin-top: 1rem;
       }
     </style>
-    <link rel="stylesheet" href="./index.css" />
+    <link rel="stylesheet" href="../ui/css/index.scss" />
   </head>
   <body class="notification">
     <div id="app-content">
@@ -50,6 +50,6 @@
       />
     </div>
     <div id="popover-content"></div>
-    <script src="./load-app.js" async></script>
+    <script src="./scripts/load/ui.ts" async></script>
   </body>
 </html>
diff --git a/app/popup.html b/app/popup.html
@@ -4,14 +4,14 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1 user-scalable=no">
     <title>MetaMask</title>
-    <link rel="stylesheet" href="./index.css">
+    <link rel="stylesheet" href="../ui/css/index.scss" />
   </head>
   <body style="width:357px; height:600px;">
     <div id="app-content">
       <img class="loading-logo" src="./images/logo/metamask-fox.svg" alt="" loading="lazy" />
       <img class="loading-spinner" src="./images/spinner.gif" alt="" loading="lazy" />
     </div>
     <div id="popover-content"></div>
-    <script src="./load-app.js" async></script>
+    <script src="./scripts/load/ui.ts" async></script>
   </body>
 </html>