diff --git a/.env.example b/.env.example index d749205..3b65301 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,8 @@ ROMS_DIR_FULLPATH=./games AUTH_USERS=admin:123,other:421 UNAUTHORIZED_MSG='No tricks and treats for you!!' WELCOME_MSG='The Server Works!!' +NX_PORTS=5000 # device ftp port +# NX_IPS=192.168.0.103 # device ip port / detected when tinfoil list games +SAVE_SYNC_INTERVAL=50000 # device sync cycle interval +# NX_USER=ftpd # device ftpd user +# NX_PASSWORD= # device ftpd password diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..d98559a --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,55 @@ +name: Multi-Arch Docker + +on: + push: + branches: [main, master] + tags: + - "*.*.*" + workflow_call: + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + # Get the repository's code + - name: Checkout + uses: actions/checkout@v3 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: vinicioslc/tinfoil-hat + flavor: | + latest=auto + tags: | + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index dfcfdba..08e4eb4 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -1,32 +1,34 @@ name: Playwright Tests on: push: - branches: [ main, master ] + branches: [main, master] pull_request: - branches: [ main, master ] + branches: [main, master] + workflow_call: + jobs: test: timeout-minutes: 60 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 18 - - name: Install dependencies - run: npm ci - - name: Copy environment files - working-directory: . - run: | - cp -f ./.env.example ./.env - cp -f ./.env.example ./test/project/.env - - name: Install Playwright Browsers - run: npx playwright install --with-deps - - name: Run Playwright tests - run: npx playwright test - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: playwright-report/ - retention-days: 30 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - name: Install dependencies + run: npm ci + - name: Copy environment files + working-directory: . + run: | + cp -f ./.env.example ./.env + cp -f ./.env.example ./test/project/.env + - name: Install Playwright Browsers + run: npx playwright install --with-deps + - name: Run Playwright tests + run: npx playwright test + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 diff --git a/README.md b/README.md index 96cbcce..62b7877 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # 📂 Tinfoil-Hat Server [![Playwright Tests](https://github.com/vinicioslc/tinfoil-hat/actions/workflows/playwright.yml/badge.svg)](https://github.com/vinicioslc/tinfoil-hat/actions/workflows/playwright.yml) +[![Docker Publish](https://github.com/vinicioslc/tinfoil-hat/actions/workflows/docker.yml/badge.svg)](https://github.com/vinicioslc/tinfoil-hat/actions/workflows/docker.yml) > A Docker based Tinfoil Server - you could download code and `npm run dev` as well... @@ -14,6 +15,7 @@ With this server Tinfoil users can serve all .NSP .XCI files in local network wi - Multi user Authentication through user:pass,user2:pass2 ENV supplied as docker env - Customize Hello message and not logged in message throught ENVs - 96% less RAM consumption, compared to the NUT solution !!! (in some cases see below!) +

@@ -96,6 +98,33 @@ services: - 9008:8080 ``` +## Automatic Saves Backup using FTP + +> How it works : + +1. First you need backup your saves using Tinfoil or JKSV the folders path supported are `/JKSV` and `/switch/tinfoil/saves` + + > Both folders will be downloaded to `//Saves/JKSV` and `//Saves/Tinfoil` this will ensure these folders will be found by tinfoil app. + +2. You need to serve your Switch files at network using ftpd + + ##### Setup sys-ftpd on the Switch + + - Install sys-ftpd - available as sys ftpd light in the Homebrew Menu + - Install ovl-sysmodule from Homebrew Menu - optional but recommended + + Follow the sys-ftpd configuration to set up the user, password and port used for the FTP connection. Note these for Ownfoil configuration, as well as the IP of your Switch. If you installed ovl-sysmodule you can toggle on/off the FTP server using the Tesla overlay. + + It is recommended to test the FTP connection at least once with a regular FTP Client to make sure everything is working as expected on the switch. + +3. To start sync saves inside Tinfoil app on switch connect and list the server homebrews to the server recongnize the switch IP and start sync + + - This will allow the server "capture" the switch device IP and start to syncing it + +4. After first sync the server will fetch periodically using sync interval. + +![Save Sync Diagram](/.diagrams/save%20sync.drawio.png) + ## We want - [ ] Organize and rename app listing by GAMEID @@ -104,7 +133,7 @@ services: ## Images with Tinfoil -#### setup connection +#### Setup Connection ![image](https://user-images.githubusercontent.com/10997022/214877049-8d369eb5-7440-4b22-9763-96da1c277f41.png) @@ -117,11 +146,3 @@ services: - `express` | Serve Dynamically shop(.json|.tfl) with updated content at every refresh and serve files statically - `serve-index` | To serve a rich listing of files (in case only shop.json shop.tfl for tinfoil) - `json5` | To parse custom shop_template.jsonc (you can define on it custom content like a welcome message !!!) - -## Saves FTPD sync - -The server will comunicate with the NSW that have connected throught tinfoil for list games in some time on past. - -> How sync workflow works : - -![Save Sync Diagram](/.diagrams/save%20sync.drawio.png) diff --git a/package-lock.json b/package-lock.json index 90db838..e2d38af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "local-devices": "^4.0.0", "local-ip-address": "^1.0.0", "lodash": "^4.17.21", + "mkdirp": "^3.0.1", "public-ip": "^6.0.1", "serve-index": "^1.9.1", "supports-color": "^9.3.1", @@ -1173,6 +1174,20 @@ "node": "*" } }, + "node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2695,6 +2710,11 @@ "brace-expansion": "^1.1.7" } }, + "mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", diff --git a/package.json b/package.json index 8cbc7af..bdae43f 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "local-devices": "^4.0.0", "local-ip-address": "^1.0.0", "lodash": "^4.17.21", + "mkdirp": "^3.0.1", "public-ip": "^6.0.1", "serve-index": "^1.9.1", "supports-color": "^9.3.1", diff --git a/src/debug.js b/src/debug.js index c2804e8..b404322 100644 --- a/src/debug.js +++ b/src/debug.js @@ -3,11 +3,13 @@ import debug from "debug"; var log = debug("tinfoil-hat"); var http = debug("tinfoil-hat:request"); var file = debug("tinfoil-hat:file"); -var error = debug("tinfoil-hat:error"); +var ftp = debug("tinfoil-hat:ftp"); +var error = debug("tinfoil-hat:err"); export default { http, file, log, - error: console.error, + ftp, + error, }; diff --git a/src/index.js b/src/index.js index 2a72999..880f1a8 100644 --- a/src/index.js +++ b/src/index.js @@ -6,8 +6,9 @@ import shopFileBuilder from "./shop-file-builder.js"; import { romsDirPath, appPort, unauthorizedMessage } from "./helpers/envs.js"; import { afterStartFunction } from "./afterStartFunction.js"; import { getUsersFromEnv } from "./authUsersParser.js"; -import FTPClient from "./modules/ftp-client.js"; +import SaveSyncManager from "./modules/ftp-client.js"; +const saveSyncManager = new SaveSyncManager(); const expressApp = express(); // Serve static files and interface const BasicAuthUsers = getUsersFromEnv(); @@ -21,13 +22,14 @@ if (BasicAuthUsers) { ); } -expressApp.use(shopFileBuilder); +expressApp.use(shopFileBuilder(saveSyncManager)); expressApp.use(express.static(path.join(romsDirPath))); expressApp.use( serveIndex(romsDirPath, { icons: true, - hidden: true, + // will ignore . starting files like .hidden, .env .... + hidden: false, // should ignore games folders only showing index files // to avoid bug with not listing when have many games filter: (filename, index, files) => { @@ -36,6 +38,7 @@ expressApp.use( }) ); const server = expressApp.listen(appPort, afterStartFunction(appPort)); -const sync = new FTPClient(); -await sync.start(); + +await saveSyncManager.start(); + export default server; diff --git a/src/modules/ftp-client.js b/src/modules/ftp-client.js index 791f15d..f108a35 100644 --- a/src/modules/ftp-client.js +++ b/src/modules/ftp-client.js @@ -5,39 +5,88 @@ import { savesDirPath } from "../helpers/envs.js"; import ftp from "basic-ftp"; import debug from "../debug.js"; -export default class FTPClient { +export default class SaveSyncManager { retryTimeout = null; + checkInterval = null; + DEVICES_IP_LIST = []; + constructor() { + this.intervalMS = 5000; + if (process?.env?.SAVE_SYNC_INTERVAL) { + this.intervalMS = process.env.SAVE_SYNC_INTERVAL; + } + if (process?.env?.NX_IPS && process.env.NX_IPS.includes(";")) { + this.DEVICES_IP_LIST = process.env.NX_IPS.split(";")[0]; + } else if (process?.env?.NX_IPS) { + this.DEVICES_IP_LIST.push(process.env.NX_IPS); + } + debug.log("Save sync manager initialized", this.DEVICES_IP_LIST); + } + async addRecentDevice(ip) { + if (!this.DEVICES_IP_LIST.includes(ip)) { + debug.ftp("ADD TO DEVICES", ip); + this.DEVICES_IP_LIST.push(ip); + } + } + async start(config) { const { retry = false } = config ?? {}; if (retry && this.retryTimeout) { clearTimeout(this.retryTimeout); this.retryTimeout.unref(); } - try { - let switchIP = process.env.NX_IPS; - const switchPORT = process.env.NX_PORTS ?? 5000; - if (!process.env.NX_IPS) { - const valuesFound = await ipSearch.searchLanDevices(5000); - console.log("DEVICES FOUND", valuesFound[0].ip); - switchIP = valuesFound[0].ip; + this.ensurePauseInterval(); + debug.ftp("RUNING FTP CHECK ON DEVICES:", this.DEVICES_IP_LIST.length); + for (const switchIP of this.DEVICES_IP_LIST) { + try { + await this.makeDeviceSync(switchIP); + } catch (error) { + if (!retry) { + debug.ftp( + "Fail during ftp sync, ip:", + switchIP, + "will retry in next sync..." + ); + } } + } + this.ensureIntervalExists(); + } + ensurePauseInterval() { + if (this.checkInterval) { + clearInterval(this.checkInterval); + this.checkInterval = null; + } + } - const client = new ftp.Client(0); - client.trackProgress((info) => { - process.stdout.write("."); - }); - - console.log("CONNECTED TO", switchIP); - await client.access({ - host: switchIP, - port: switchPORT, - }); - await downloadEverthing(client); - } catch (error) { - debug.error("Fail during ftp sync, retrying in 5 seconds...", error); - this.retryTimeout = setTimeout(this.start({ retry: true }), 5000); + ensureIntervalExists() { + if (!this.checkInterval && this.intervalMS >= 5000) { + this.checkInterval = setInterval(() => this.start(), this.intervalMS); + debug.log( + "SCHEDULED NEXT EXECUTION IN", + this.intervalMS / 1000, + "seconds" + ); } } + + async makeDeviceSync(switchIP) { + const switchPORT = process.env.NX_PORTS ?? 5000; + const switchPass = process.env.NX_PASSWORD ?? undefined; + const switchUser = process.env.NX_USER ?? undefined; + const client = new ftp.Client(0); + client.trackProgress((info) => { + process.stdout.write("."); + }); + + await client.access({ + host: switchIP, + port: switchPORT, + password: switchPass, + user: switchUser, + }); + debug.ftp("CONNECTED TO DEVICE:", switchIP); + await downloadEverthing(client); + } } /** * Description @@ -46,17 +95,24 @@ export default class FTPClient { */ async function downloadEverthing(ftp) { const supportedBackups = await checkSwitchBackups(ftp); - debug.log("SUPPORTED BACKUPS", supportedBackups); + debug.ftp( + "SUPPORTED BACKUPS", + supportedBackups.map((val) => val.name) + ); if (supportedBackups.length) { for (const backupFolder of supportedBackups) { const localBackupFolder = path.resolve(savesDirPath, backupFolder.local); - debug.log('DOWNLOADING REMOTE SWITCH "', backupFolder.remote, '"'); - debug.log('TO LOCAL PATH "', localBackupFolder, '"'); + debug.ftp( + "START SYNC SWITCH FOLDER:", + backupFolder.remote, + "TO:", + localBackupFolder + ); await ftp.downloadToDir(localBackupFolder, backupFolder.remote); - debug.log("DOWNLOAD TO BACKUP FINISHED"); + debug.ftp("END SYNC SWITCH FOLDER"); } } else { - console.warn("NO BACKUPS SUPPORTED"); + debug.ftp("NO BACKUPS SUPPORTED BY"); } } @@ -71,16 +127,16 @@ async function checkSwitchBackups(ftp) { console.warn("NOT FOUND JKSV FOLDER", "/JKSV"); } try { - const tinfoilFiles = await ftp.list("/switch/tinfoil/saves/common"); + const tinfoilFiles = await ftp.list("/switch/tinfoil/saves"); if (tinfoilFiles.length) { supportedBackups.push({ name: "TINFOIL", local: "Tinfoil", - remote: "/switch/tinfoil/saves/common", + remote: "/switch/tinfoil/saves", }); } } catch (error) { - console.warn("NOT FOUND TINFOIL FOLDER", "/switch/tinfoil/saves/common"); + debug.ftp("NOT FOUND TINFOIL FOLDER", "/switch/tinfoil/saves"); } return supportedBackups; } diff --git a/src/shop-file-builder.js b/src/shop-file-builder.js index a945b76..656a4dd 100644 --- a/src/shop-file-builder.js +++ b/src/shop-file-builder.js @@ -6,29 +6,43 @@ import debug from "./debug.js"; import generateIndex from "./create-index-content.js"; - +import SaveSyncManager from "./modules/ftp-client.js"; +function formatIPAddress(ip) { + return ip.split(":")[ip.split(":").length - 1]; +} /** - * @param {import("express").Request} req - * @param {import("express").Response} res - * @param {import("express").NextFunction} next + * Description + * @param {SaveSyncManager} saveSyncManager + * @returns {any} */ -export default async (req, res, next) => { - if (req.path === "/shop.json") { - debug.http("IN-> %o", req.path); - res.header("Content-Type", "application/json"); - res.status(200).send(await generateIndex()); - debug.http("OUT-< %o", req.path); - return; - } else if (req.path === "/shop.tfl") { - debug.http("IN-> %o", req.path); - res.header("Content-Type", "application/octet-stream"); - res.status(200).send(await generateIndex()); - debug.http("OUT-< %o", req.path); - return; - } else { - debug.http("IN-> %o", req.path); - next(); - debug.http("OUT-< %o", req.path); - return; - } -}; +export default function (saveSyncManager) { + /** + * @param {import("express").Request} req + * @param {import("express").Response} res + * @param {import("express").NextFunction} next + */ + return async (req, res, next) => { + if (req.path === "/shop.json") { + debug.http("IN-> %o", req.path, "IP:", req.ip); + res.header("Content-Type", "application/json"); + res.status(200).send(await generateIndex()); + debug.http("OUT-< %o", req.path); + return; + } else if (req.path === "/shop.tfl") { + /** When occurs some tinfoil listing the server will store the IP on memory */ + if (req.ip.split(":").length) { + await saveSyncManager.addRecentDevice(formatIPAddress(req.ip)); + } + debug.http("IN-> %o", req.path); + res.header("Content-Type", "application/octet-stream"); + res.status(200).send(await generateIndex()); + debug.http("OUT-< %o", req.path); + return; + } else { + debug.http("IN-> %o", req.path); + next(); + debug.http("OUT-< %o", req.path); + return; + } + }; +}