From ded5d2b8d3c3ba86f459f97f1c46a4fcc8262050 Mon Sep 17 00:00:00 2001 From: Pascal Wilbrink Date: Thu, 22 Feb 2024 22:42:18 +0100 Subject: [PATCH] Feat: Added download script --- scripts/.gitignore | 1 + scripts/package-lock.json | 406 ++++++++++++++++++++++++++++++++++++++ scripts/package.json | 20 ++ scripts/src/downloader.ts | 138 +++++++++++++ scripts/src/index.ts | 32 +++ scripts/tsconfig.json | 14 ++ 6 files changed, 611 insertions(+) create mode 100644 scripts/.gitignore create mode 100644 scripts/package-lock.json create mode 100644 scripts/package.json create mode 100644 scripts/src/downloader.ts create mode 100644 scripts/src/index.ts create mode 100644 scripts/tsconfig.json diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..40b878d --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +node_modules/ \ No newline at end of file diff --git a/scripts/package-lock.json b/scripts/package-lock.json new file mode 100644 index 0000000..7fc17b3 --- /dev/null +++ b/scripts/package-lock.json @@ -0,0 +1,406 @@ +{ + "name": "@openscd/scripts", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@openscd/scripts", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@octokit/rest": "^20.0.2", + "atob": "^2.1.2" + }, + "devDependencies": { + "@types/atob": "^2.1.4", + "@types/node": "^20.11.20", + "ts-node": "^10.9.2", + "typescript": "^5.3.3" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@octokit/auth-token": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz", + "integrity": "sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g==", + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.0.0", + "@octokit/request": "^8.0.2", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^12.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/endpoint": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.4.tgz", + "integrity": "sha512-DWPLtr1Kz3tv8L0UvXTDP1fNwM0S+z6EJpRcvH66orY6Eld4XBMCSYsaWp4xIm61jTWxK68BrR7ibO+vSDnZqw==", + "dependencies": { + "@octokit/types": "^12.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz", + "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==", + "dependencies": { + "@octokit/request": "^8.0.1", + "@octokit/types": "^12.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "19.1.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.1.0.tgz", + "integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw==" + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "9.1.5", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.1.5.tgz", + "integrity": "sha512-WKTQXxK+bu49qzwv4qKbMMRXej1DU2gq017euWyKVudA6MldaSSQuxtz+vGbhxV4CjxpUxjZu6rM2wfc1FiWVg==", + "dependencies": { + "@octokit/types": "^12.4.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/plugin-request-log": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-4.0.0.tgz", + "integrity": "sha512-2uJI1COtYCq8Z4yNSnM231TgH50bRkheQ9+aH8TnZanB6QilOnx8RMD2qsnamSOXtDj0ilxvevf5fGsBhBBzKA==", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.3.0.tgz", + "integrity": "sha512-c/fjpoHispRvBZuRoTVt/uALg7pXa9RQbXWJiDMk6NDkGNomuAZG7YuYYpZoxeoXv+kVRjIDTsO0e1z0pei+PQ==", + "dependencies": { + "@octokit/types": "^12.4.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/request": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.2.0.tgz", + "integrity": "sha512-exPif6x5uwLqv1N1irkLG1zZNJkOtj8bZxuVHd71U5Ftuxf2wGNvAJyNBcPbPC+EBzwYEbBDdSFb8EPcjpYxPQ==", + "dependencies": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^12.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.1.tgz", + "integrity": "sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==", + "dependencies": { + "@octokit/types": "^12.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/rest": { + "version": "20.0.2", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-20.0.2.tgz", + "integrity": "sha512-Ux8NDgEraQ/DMAU1PlAohyfBBXDwhnX2j33Z1nJNziqAfHi70PuxkFYIcIt8aIAxtRE7KVuKp8lSR8pA0J5iOQ==", + "dependencies": { + "@octokit/core": "^5.0.0", + "@octokit/plugin-paginate-rest": "^9.0.0", + "@octokit/plugin-request-log": "^4.0.0", + "@octokit/plugin-rest-endpoint-methods": "^10.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/types": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.5.0.tgz", + "integrity": "sha512-YJEKcb0KkJlIUNU/zjnZwHEP8AoVh/OoIcP/1IyR4UHxExz7fzpe/a8IG4wBtQi7QDEqiomVLX88S6FpxxAJtg==", + "dependencies": { + "@octokit/openapi-types": "^19.1.0" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true + }, + "node_modules/@types/atob": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/atob/-/atob-2.1.4.tgz", + "integrity": "sha512-FisOhG87cCFqzCgq6FUtSYsTMOHCB/p28zJbSN1QBo4ZGJfg9PEhMjdIV++NDeOnloUUe0Gz6jwBV+L1Ac00Mw==", + "dev": true + }, + "node_modules/@types/node": { + "version": "20.11.20", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.20.tgz", + "integrity": "sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/acorn": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", + "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz", + "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, + "node_modules/atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "bin": { + "atob": "bin/atob.js" + }, + "engines": { + "node": ">= 4.5.0" + } + }, + "node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/typescript": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, + "node_modules/universal-user-agent": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "engines": { + "node": ">=6" + } + } + } +} diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 0000000..a34d754 --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,20 @@ +{ + "name": "@openscd/scripts", + "version": "1.0.0", + "description": "", + "scripts": { + "download": "ts-node src/index.ts" + }, + "author": "OpenSCD", + "license": "ISC", + "devDependencies": { + "@types/atob": "^2.1.4", + "@types/node": "^20.11.20", + "ts-node": "^10.9.2", + "typescript": "^5.3.3" + }, + "dependencies": { + "@octokit/rest": "^20.0.2", + "atob": "^2.1.2" + } +} diff --git a/scripts/src/downloader.ts b/scripts/src/downloader.ts new file mode 100644 index 0000000..c7e37d9 --- /dev/null +++ b/scripts/src/downloader.ts @@ -0,0 +1,138 @@ +import { Octokit, RestEndpointMethodTypes } from "@octokit/rest"; + +import fs from "fs"; + +import { extname } from "path"; +import { promisify } from "util"; +import { dirname, join } from "path"; + +const mkdir = promisify(fs.mkdir); +const writeFile = promisify(fs.writeFile); + +export interface Repository { + organization: string; + repository: string; + sha?: string; +} + +export interface Filter { + extensions?: string[]; + files?: string[]; +} + +export interface GitHubOptions { + GH_TOKEN: string; + repository: Repository; + input_path: string; + filter?: Filter; + output_path?: string; +} + +export class GitHubDownloader { + private octokit: Octokit; + + constructor(private options: GitHubOptions) { + this.octokit = new Octokit({ + auth: options.GH_TOKEN, + }); + } + + downloadFolder(): Promise { + return new Promise((resolve, reject) => { + this.getFileStructure( + this.options.input_path, + this.options.repository + ).then((files) => { + Promise.all( + files + .filter( + (node) => + node.path.startsWith(this.options.input_path) && + node.type === "file" + ) + .filter((node) => { + return this.options.filter + ? ( + this.options.filter.extensions || [extname(node.path)] + ).includes(extname(node.path)) && + (this.options.filter.files || [node.path]).includes( + node.path + ) + : true; + }) + .map(async (node) => + this.getFileContent(this.options.repository, node) + ) + ).then((files) => { + Promise.all( + files.map((file) => { + return mkdir( + dirname(join(this.options.output_path || "", file.path)), + { + recursive: true, + } + ) + .then(() => { + return writeFile( + join(this.options.output_path || "", file.path), + file.contents + ); + }) + .catch((err) => reject(err)); + }) + ) + .then(() => resolve()) + .catch((err) => reject(err)); + }); + }); + }); + } + + protected async getFileContent( + repository: Repository, + node: { sha: string; path: string } + ): Promise<{ path: string; contents: Buffer }> { + const { data } = await this.octokit.git.getBlob({ + owner: repository.organization, + repo: repository.repository, + file_sha: node.sha, + }); + + return { + path: node.path, + contents: Buffer.from(data.content, data.encoding as BufferEncoding), + }; + } + + protected getFileStructure( + path: string, + repository: Repository + ): Promise<{ path: string; type: string; sha: string }[]> { + return new Promise((resolve, reject) => { + const req: RestEndpointMethodTypes["repos"]["getContent"]["parameters"] = + { + owner: repository.organization, + repo: repository.repository, + path: path, + }; + this.octokit.repos + .getContent(req) + .then((res) => { + const dirs = (res.data as unknown[]).map((node: any) => { + if (node.type === "dir") { + return this.getFileStructure(node.path, repository); + } + return { + path: node.path, + type: node.type, + sha: node.sha, + }; + }); + Promise.all(dirs) + .then((res) => resolve(res.flat())) + .catch((err) => reject(err)); + }) + .catch((err) => reject(err)); + }); + } +} diff --git a/scripts/src/index.ts b/scripts/src/index.ts new file mode 100644 index 0000000..f74de01 --- /dev/null +++ b/scripts/src/index.ts @@ -0,0 +1,32 @@ +import { join } from "path"; +import { env } from "process"; +import { GitHubDownloader, GitHubOptions } from "./downloader"; + +const GH_TOKEN: string = process.env!.GH_TOKEN! || ""; + +const repositories: GitHubOptions[] = [ + { + repository: { + organization: "openscd", + repository: "open-scd", + }, + input_path: "docs", + GH_TOKEN: GH_TOKEN, + filter: { + extensions: [".md", ".png"], + }, + output_path: join("../", "pages", "decisions"), + }, +]; + +Promise.all( + repositories + .map((repo) => new GitHubDownloader(repo)) + .map((repo) => repo.downloadFolder()) +) + .then(() => { + console.log("Done"); + }) + .catch((err) => { + console.error(`Error: ${err}`); + }); diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json new file mode 100644 index 0000000..b13029d --- /dev/null +++ b/scripts/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "CommonJS", + "esModuleInterop": true, + "strict": true, + "lib": [ + "ESNext" + ], + "skipLibCheck": true, + "rootDir": "./src", + "outDir": "./dist" + } +} \ No newline at end of file