From 36b85909fc1769474834b8d5fdc1f36cb788871b Mon Sep 17 00:00:00 2001 From: Jonathan Gamble Date: Fri, 6 Dec 2024 12:02:17 -0600 Subject: [PATCH 01/11] server support for inline script injection via manifest --- app/views/base/page.scala | 2 +- app/views/relay.scala | 2 +- modules/web/src/main/AssetManifest.scala | 12 +++++++----- modules/web/src/main/ui/layout.scala | 21 +++++++-------------- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/app/views/base/page.scala b/app/views/base/page.scala index d2035242a6373..ca9610d18bd5e 100644 --- a/app/views/base/page.scala +++ b/app/views/base/page.scala @@ -155,7 +155,7 @@ object page: ) )(p.transform(p.body)), bottomHtml, - ctx.nonce.map(inlineJs.apply), + ctx.nonce.map(inlineJs(_, allModules)), modulesInit(allModules, ctx.nonce), p.jsFrag.fold(emptyFrag)(_(ctx.nonce)), p.pageModule.map { mod => frag(jsonScript(mod.data)) } diff --git a/app/views/relay.scala b/app/views/relay.scala index 842351654fac9..f688ea6da1310 100644 --- a/app/views/relay.scala +++ b/app/views/relay.scala @@ -52,5 +52,5 @@ def embed( div(id := "main-wrap", cls := "is2d"): ui.showPreload(rt, data)(cls := "relay-embed") , - views.base.page.ui.inlineJs(ctx.nonce) + views.base.page.ui.inlineJs(ctx.nonce, List(Esm("site").some)) ) diff --git a/modules/web/src/main/AssetManifest.scala b/modules/web/src/main/AssetManifest.scala index bc2c493c2853b..989e4aef23f0a 100644 --- a/modules/web/src/main/AssetManifest.scala +++ b/modules/web/src/main/AssetManifest.scala @@ -9,7 +9,7 @@ import java.nio.file.Files import lila.core.config.NetConfig -case class SplitAsset(name: String, imports: List[String]): +case class SplitAsset(name: String, imports: List[String], inlineJs: Option[String]): def all = name :: imports case class AssetMaps( js: Map[String, SplitAsset], @@ -31,7 +31,8 @@ final class AssetManifest(environment: Environment, net: NetConfig)(using ws: St def jsAndDeps(keys: List[String]): List[String] = keys.flatMap { key => maps.js.get(key).so(_.all) }.distinct - def lastUpdate: Instant = maps.modified + def inlineJs(key: String): Option[String] = maps.js.get(key).flatMap(_.inlineJs) + def lastUpdate: Instant = maps.modified def update(): Unit = if environment.mode.isProd || net.externalManifest then @@ -73,9 +74,10 @@ final class AssetManifest(environment: Environment, net: NetConfig)(using ws: St .as[JsObject] .value .map { (k, value) => - val name = (value \ "hash").asOpt[String].fold(s"$k.js")(h => s"$k.$h.js") - val imports = (value \ "imports").asOpt[List[String]].getOrElse(Nil) - (k, SplitAsset(name, imports)) + val name = (value \ "hash").asOpt[String].fold(s"$k.js")(h => s"$k.$h.js") + val imports = (value \ "imports").asOpt[List[String]].getOrElse(Nil) + val inlineJs = (value \ "inline").asOpt[String] + (k, SplitAsset(name, imports, inlineJs)) } .toMap diff --git a/modules/web/src/main/ui/layout.scala b/modules/web/src/main/ui/layout.scala index 9b8c4f6b22d0f..f062695790559 100644 --- a/modules/web/src/main/ui/layout.scala +++ b/modules/web/src/main/ui/layout.scala @@ -160,6 +160,13 @@ final class layout(helpers: Helpers, assetHelper: lila.web.ui.AssetFullHelper)( def modulesInit(modules: EsmList, nonce: Optionce) = modules.flatMap(_.map(_.init(nonce))) // in body + def inlineJs(nonce: Nonce, modules: EsmList = Nil): Frag = + val code = + (Esm("site").some :: modules) + .flatMap(_.flatMap(m => assetHelper.manifest.inlineJs(m.key).map(js => s"(()=>{${js}})()"))) + .mkString(";") + embedJsUnsafe(code)(nonce.some) + private def hrefLang(langStr: String, path: String) = s"""""" @@ -331,17 +338,3 @@ final class layout(helpers: Helpers, assetHelper: lila.web.ui.AssetFullHelper)( .getOrElse { (!error).option(anonDasher) } ) ) - - object inlineJs: - def apply(nonce: Nonce)(using Translate): Frag = embedJsUnsafe(jsCode)(nonce.some) - - private val cache = new java.util.concurrent.ConcurrentHashMap[Lang, String] - lila.common.Bus.subscribeFun("i18n.load"): - case lang: Lang => cache.remove(lang) - - private def jsCode(using t: Translate) = - cache.computeIfAbsent( - t.lang, - _ => """window.site={load:new Promise(r=>document.addEventListener("DOMContentLoaded",r))};""" - ) - end inlineJs From 8e056a25a279bfb2f24e5ea4d2e4b23e096032f4 Mon Sep 17 00:00:00 2001 From: Jonathan Gamble Date: Fri, 6 Dec 2024 12:05:14 -0600 Subject: [PATCH 02/11] refactor for maintainability and support script injection via manifest.json --- ui/.build/src/build.ts | 103 ++++++----- ui/.build/src/clean.ts | 2 +- ui/.build/src/console.ts | 2 +- ui/.build/src/env.ts | 201 ++++++++++++++++++++ ui/.build/src/esbuild.ts | 152 +++++++++++---- ui/.build/src/i18n.ts | 67 ++++--- ui/.build/src/main.ts | 318 +++++--------------------------- ui/.build/src/manifest.ts | 185 +++---------------- ui/.build/src/monitor.ts | 59 ------ ui/.build/src/parse.ts | 74 +++++--- ui/.build/src/sass.ts | 236 +++++++++++++----------- ui/.build/src/sync.ts | 72 ++++++-- ui/.build/src/tsc.ts | 29 +-- ui/.build/{readme => usage.txt} | 25 ++- ui/build | 5 +- 15 files changed, 769 insertions(+), 761 deletions(-) create mode 100644 ui/.build/src/env.ts delete mode 100644 ui/.build/src/monitor.ts rename ui/.build/{readme => usage.txt} (62%) diff --git a/ui/.build/src/build.ts b/ui/.build/src/build.ts index 13c066acb997f..1cfe28c4b1871 100644 --- a/ui/.build/src/build.ts +++ b/ui/.build/src/build.ts @@ -1,29 +1,31 @@ import fs from 'node:fs'; import cps from 'node:child_process'; import ps from 'node:process'; -import { parsePackages } from './parse.ts'; -import { tsc, stopTsc } from './tsc.ts'; +import path from 'node:path'; +import { parsePackages, globArray } from './parse.ts'; +import { tsc, stopTscWatch } from './tsc.ts'; import { sass, stopSass } from './sass.ts'; -import { esbuild, stopEsbuild } from './esbuild.ts'; +import { esbuild, stopEsbuildWatch } from './esbuild.ts'; import { sync, stopSync } from './sync.ts'; -import { monitor, stopMonitor } from './monitor.ts'; -import { writeManifest } from './manifest.ts'; -import { type Package, env, errorMark, colors as c } from './main.ts'; -import { i18n, stopI18n } from './i18n.ts'; +import { stopManifest } from './manifest.ts'; +import { env, errorMark, colors as c } from './env.ts'; +import { i18n, stopI18nWatch } from './i18n.ts'; +import { unique } from './algo.ts'; +import { clean } from './clean.ts'; export async function build(pkgs: string[]): Promise { - if (env.install) - cps.execSync('pnpm install --prefer-frozen-lockfile', { cwd: env.rootDir, stdio: 'inherit' }); + if (env.install) cps.execSync('pnpm install', { cwd: env.rootDir, stdio: 'inherit' }); if (!pkgs.length) env.log(`Parsing packages in '${c.cyan(env.uiDir)}'`); ps.chdir(env.uiDir); - [env.packages, env.deps] = await parsePackages(); + await parsePackages(); pkgs .filter(x => !env.packages.has(x)) .forEach(x => env.exit(`${errorMark} - unknown package '${c.magenta(x)}'`)); - env.building = pkgs.length === 0 ? [...env.packages.values()] : depsMany(pkgs); + env.building = + pkgs.length === 0 ? [...env.packages.values()] : unique(pkgs.flatMap(p => env.transitiveDeps(p))); if (pkgs.length) env.log(`Building ${c.grey(env.building.map(x => x.name).join(', '))}`); @@ -36,43 +38,60 @@ export async function build(pkgs: string[]): Promise { ]); await Promise.all([sass(), sync(), i18n()]); - await tsc().then(esbuild); - monitor(pkgs); + await Promise.all([tsc(), esbuild()]); + if (env.watch) monitor(pkgs); } -export async function stopBuild(): Promise { - stopMonitor(); +export async function stopBuildWatch(): Promise { + for (const w of watchers) w.close(); + watchers.length = 0; + clearTimeout(tscTimeout); + clearTimeout(packageTimeout); + tscTimeout = packageTimeout = undefined; stopSass(); - stopTsc(); stopSync(); - stopI18n(); - await stopEsbuild(); + stopI18nWatch(); + stopManifest(); + await Promise.allSettled([stopTscWatch(), stopEsbuildWatch()]); } -export function postBuild(): void { - writeManifest(); - for (const pkg of env.building) { - pkg.post.forEach((args: string[]) => { - env.log(`[${c.grey(pkg.name)}] exec - ${c.cyanBold(args.join(' '))}`); - const stdout = cps.execSync(`${args.join(' ')}`, { cwd: pkg.root }); - if (stdout) env.log(stdout, { ctx: pkg.name }); - }); - } -} - -export function prePackage(pkg: Package | undefined): void { - pkg?.pre.forEach((args: string[]) => { - env.log(`[${c.grey(pkg.name)}] exec - ${c.cyanBold(args.join(' '))}`); - const stdout = cps.execSync(`${args.join(' ')}`, { cwd: pkg.root }); - if (stdout) env.log(stdout, { ctx: pkg.name }); - }); -} +const watchers: fs.FSWatcher[] = []; -function depsOne(pkgName: string): Package[] { - const collect = (dep: string): string[] => [...(env.deps.get(dep) || []).flatMap(d => collect(d)), dep]; - return unique(collect(pkgName).map(name => env.packages.get(name))); -} +let packageTimeout: NodeJS.Timeout | undefined; +let tscTimeout: NodeJS.Timeout | undefined; -const depsMany = (pkgNames: string[]): Package[] => unique(pkgNames.flatMap(depsOne)); +async function monitor(pkgs: string[]): Promise { + const [typePkgs, typings] = await Promise.all([ + globArray('*/package.json', { cwd: env.typesDir }), + globArray('*/*.d.ts', { cwd: env.typesDir }), + ]); + const tscChange = async () => { + if (packageTimeout) return; + stopManifest(); + await Promise.allSettled([stopTscWatch(), stopEsbuildWatch()]); + clearTimeout(tscTimeout); + tscTimeout = setTimeout(() => { + if (packageTimeout) return; + tsc().then(esbuild); + }, 2000); + }; + const packageChange = async () => { + if (env.watch && env.install) { + clearTimeout(tscTimeout); + clearTimeout(packageTimeout); + await stopBuildWatch(); + packageTimeout = setTimeout(() => clean().then(() => build(pkgs)), 2000); + return; + } + env.warn('Exiting due to package.json change'); + ps.exit(0); + }; -const unique = (pkgs: (T | undefined)[]): T[] => [...new Set(pkgs.filter(x => x))] as T[]; + watchers.push(fs.watch(path.join(env.rootDir, 'package.json'), packageChange)); + for (const p of typePkgs) watchers.push(fs.watch(p, packageChange)); + for (const t of typings) watchers.push(fs.watch(t, tscChange)); + for (const pkg of env.building) { + watchers.push(fs.watch(path.join(pkg.root, 'package.json'), packageChange)); + watchers.push(fs.watch(path.join(pkg.root, 'tsconfig.json'), tscChange)); + } +} diff --git a/ui/.build/src/clean.ts b/ui/.build/src/clean.ts index ce2f02c2dd625..a582b265ca675 100644 --- a/ui/.build/src/clean.ts +++ b/ui/.build/src/clean.ts @@ -1,6 +1,6 @@ import { promises as fs } from 'fs'; import fg from 'fast-glob'; -import { env, colors as c } from './main.ts'; +import { env, colors as c } from './env.ts'; const globOpts: fg.Options = { absolute: true, diff --git a/ui/.build/src/console.ts b/ui/.build/src/console.ts index b3b1826a5dcca..14cc931dfbbe2 100644 --- a/ui/.build/src/console.ts +++ b/ui/.build/src/console.ts @@ -1,5 +1,5 @@ import { createServer, IncomingMessage, ServerResponse } from 'node:http'; -import { env, errorMark, warnMark, colors as c } from './main.ts'; +import { env, errorMark, warnMark, colors as c } from './env.ts'; export async function startConsole() { if (!env.remoteLog || !env.watch) return; diff --git a/ui/.build/src/env.ts b/ui/.build/src/env.ts new file mode 100644 index 0000000000000..7c5cb22a26489 --- /dev/null +++ b/ui/.build/src/env.ts @@ -0,0 +1,201 @@ +import path from 'node:path'; +import type { Package } from './parse.ts'; +import { unique, isEquivalent } from './algo.ts'; +import { updateManifest } from './manifest.ts'; + +// state, logging, and exit code logic + +type Builder = 'sass' | 'tsc' | 'esbuild'; + +export const env = new (class { + readonly rootDir = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..'); + readonly uiDir = path.join(this.rootDir, 'ui'); + readonly outDir = path.join(this.rootDir, 'public'); + readonly cssOutDir = path.join(this.outDir, 'css'); + readonly jsOutDir = path.join(this.outDir, 'compiled'); + readonly hashOutDir = path.join(this.outDir, 'hashed'); + readonly themeDir = path.join(this.uiDir, 'common', 'css', 'theme'); + readonly themeGenDir = path.join(this.themeDir, 'gen'); + readonly buildDir = path.join(this.uiDir, '.build'); + readonly cssTempDir = path.join(this.buildDir, 'build', 'css'); + readonly buildSrcDir = path.join(this.uiDir, '.build', 'src'); + readonly buildTempDir = path.join(this.buildDir, 'build'); + readonly typesDir = path.join(this.uiDir, '@types'); + readonly i18nSrcDir = path.join(this.rootDir, 'translation', 'source'); + readonly i18nDestDir = path.join(this.rootDir, 'translation', 'dest'); + readonly i18nJsDir = path.join(this.rootDir, 'translation', 'js'); + + packages: Map = new Map(); + workspaceDeps: Map = new Map(); + building: Package[] = []; + + watch = false; + clean = false; + prod = false; + debug = false; + remoteLog: string | boolean = false; + rgb = false; + install = true; + sync = true; + i18n = true; + test = false; + exitCode: Map = new Map(); + startTime: number | undefined = Date.now(); + logTime = true; + logContext = true; + color: any = { + build: 'green', + sass: 'magenta', + tsc: 'yellow', + esbuild: 'blue', + }; + + get sass(): boolean { + return this.exitCode.get('sass') !== false; + } + + get tsc(): boolean { + return this.exitCode.get('tsc') !== false; + } + + get esbuild(): boolean { + return this.exitCode.get('esbuild') !== false; + } + + get manifestOk(): boolean { + return ( + isEquivalent(this.building, [...this.packages.values()]) && + this.sync && + this.i18n && + (['tsc', 'esbuild', 'sass'] as const).every(x => this.exitCode.get(x) === 0) + ); + } + + get manifestFile(): string { + return path.join(this.jsOutDir, `manifest.${this.prod ? 'prod' : 'dev'}.json`); + } + + transitiveDeps(pkgName: string): Package[] { + const depList = (dep: string): string[] => [ + ...(this.workspaceDeps.get(dep) ?? []).flatMap(d => depList(d)), + dep, + ]; + return unique(depList(pkgName).map(name => this.packages.get(name))); + } + + warn(d: any, ctx = 'build'): void { + this.log(d, { ctx: ctx, warn: true }); + } + + error(d: any, ctx = 'build'): void { + this.log(d, { ctx: ctx, error: true }); + } + + exit(d: any, ctx = 'build'): void { + this.log(d, { ctx: ctx, error: true }); + process.exit(1); + } + + good(ctx = 'build'): void { + this.log(colors.good('No errors') + this.watch ? ` - ${colors.grey('Watching')}...` : '', { ctx: ctx }); + } + + log(d: any, { ctx = 'build', error = false, warn = false }: any = {}): void { + let text: string = + !d || typeof d === 'string' || d instanceof Buffer + ? String(d) + : Array.isArray(d) + ? d.join('\n') + : JSON.stringify(d); + + const esc = this.color ? escape : (text: string, _: any) => text; + + if (!this.color) text = stripColorEscapes(text); + + const prefix = ( + (this.logTime === false ? '' : prettyTime()) + + (!ctx || !this.logContext ? '' : `[${esc(ctx, colorForCtx(ctx, this.color))}] `) + ).trim(); + + lines(text).forEach(line => + console.log( + `${prefix ? prefix + ' - ' : ''}${ + error ? esc(line, codes.error) : warn ? esc(line, codes.warn) : line + }`, + ), + ); + } + + done(code: number, ctx: Builder): void { + this.exitCode.set(ctx, code); + const err = [...this.exitCode.values()].find(x => x); + const allDone = this.exitCode.size === 3; + if (ctx !== 'tsc' || code === 0) + this.log( + `${code === 0 ? 'Done' : colors.red('Failed')}` + + (this.watch ? ` - ${colors.grey('Watching')}...` : ''), + { ctx }, + ); + if (allDone) { + if (this.startTime && !err) + this.log(`Done in ${colors.green((Date.now() - this.startTime) / 1000 + '')}s`); + this.startTime = undefined; // it's pointless to time subsequent builds, they are too fast + } + if (!this.watch && err) process.exitCode = err; + if (!err) updateManifest(); + } +})(); + +export const lines = (s: string): string[] => s.split(/[\n\r\f]+/).filter(x => x.trim()); + +const escape = (text: string, code: string): string => `\x1b[${code}m${stripColorEscapes(text)}\x1b[0m`; + +const colorLines = (text: string, code: string) => + lines(text) + .map(t => (env.color ? escape(t, code) : t)) + .join('\n'); + +const codes: Record = { + black: '30', + red: '31', + green: '32', + yellow: '33', + blue: '34', + magenta: '35', + cyan: '36', + grey: '90', + error: '31', + warn: '33', +}; + +export const colors: Record string> = { + red: (text: string): string => colorLines(text, codes.red), + green: (text: string): string => colorLines(text, codes.green), + yellow: (text: string): string => colorLines(text, codes.yellow), + blue: (text: string): string => colorLines(text, codes.blue), + magenta: (text: string): string => colorLines(text, codes.magenta), + cyan: (text: string): string => colorLines(text, codes.cyan), + grey: (text: string): string => colorLines(text, codes.grey), + black: (text: string): string => colorLines(text, codes.black), + error: (text: string): string => colorLines(text, codes.error), + warn: (text: string): string => colorLines(text, codes.warn), + good: (text: string): string => colorLines(text, codes.green + ';1'), + cyanBold: (text: string): string => colorLines(text, codes.cyan + ';1'), +}; + +export const errorMark: string = colors.red('✘ ') + colors.error('[ERROR]'); +export const warnMark: string = colors.yellow('⚠ ') + colors.warn('[WARNING]'); + +const colorForCtx = (ctx: string, color: any): string => + color && ctx in color && color[ctx] in codes ? codes[color[ctx]] : codes.grey; + +const pad2 = (n: number) => (n < 10 ? `0${n}` : `${n}`); + +function stripColorEscapes(text: string) { + return text.replace(/\x1b\[[0-9;]*m/, ''); +} + +function prettyTime() { + const now = new Date(); + return `${pad2(now.getHours())}:${pad2(now.getMinutes())}:${pad2(now.getSeconds())} `; +} diff --git a/ui/.build/src/esbuild.ts b/ui/.build/src/esbuild.ts index 624a05b37b843..c2b0215cd35eb 100644 --- a/ui/.build/src/esbuild.ts +++ b/ui/.build/src/esbuild.ts @@ -1,27 +1,21 @@ import path from 'node:path'; import es from 'esbuild'; -import { prePackage } from './build.ts'; -import { env, errorMark, colors as c } from './main.ts'; -import { jsManifest } from './manifest.ts'; +import fs from 'node:fs'; +import { env, errorMark, colors as c } from './env.ts'; +import { type Manifest, updateManifest } from './manifest.ts'; +import { readable } from './parse.ts'; -const bundles = new Map(); const esbuildCtx: es.BuildContext[] = []; - -export async function stopEsbuild(): Promise { - const proof = Promise.allSettled(esbuildCtx.map(x => x.dispose())); - esbuildCtx.length = 0; - bundles.clear(); - await proof; -} +const inlineWatch: fs.FSWatcher[] = []; +let inlineTimer: NodeJS.Timeout; export async function esbuild(): Promise { if (!env.esbuild) return; const entryPoints = []; for (const pkg of env.building) { - prePackage(pkg); - for (const bundle of pkg.bundles ?? []) { - entryPoints.push(path.join(pkg.root, bundle)); + for (const { module } of pkg.bundle) { + if (module) entryPoints.push(path.join(pkg.root, module)); } } entryPoints.sort(); @@ -39,7 +33,7 @@ export async function esbuild(): Promise { outdir: env.jsOutDir, entryNames: '[name].[hash]', chunkNames: 'common.[hash]', - plugins: [onEndPlugin], + plugins, }); if (env.watch) { ctx.watch(); @@ -50,26 +44,114 @@ export async function esbuild(): Promise { } } -const onEndPlugin = { - name: 'onEnd', - setup(build: es.PluginBuild) { - build.onEnd(async (result: es.BuildResult) => { - for (const err of result.errors) esbuildMessage(err, true); - for (const warn of result.warnings) esbuildMessage(warn); - if (result.errors.length === 0) jsManifest(result.metafile!); - env.done(result.errors.length, 'esbuild'); - }); +export async function stopEsbuildWatch(): Promise { + const proof = Promise.allSettled(esbuildCtx.map(x => x.dispose())); + for (const w of inlineWatch) w.close(); + inlineWatch.length = 0; + esbuildCtx.length = 0; + await proof; +} + +const plugins = [ + { + name: 'onBundleDone', + setup(build: es.PluginBuild) { + build.onEnd(async (result: es.BuildResult) => { + esbuildLog(result.errors, true); + esbuildLog(result.warnings); + env.done(result.errors.length, 'esbuild'); + if (result.errors.length === 0) jsManifest(result.metafile!); + }); + }, }, -}; +]; -function esbuildMessage(msg: es.Message, error = false) { - const file = msg.location?.file.replace(/^[./]*/, '') ?? ''; - const line = msg.location?.line - ? `:${msg.location.line}` - : '' + (msg.location?.column ? `:${msg.location.column}` : ''); - const srcText = msg.location?.lineText; - env.log(`${error ? errorMark : c.warn('WARNING')} - '${c.cyan(file + line)}' - ${msg.text}`, { - ctx: 'esbuild', - }); - if (srcText) env.log(' ' + c.magenta(srcText), { ctx: 'esbuild' }); +function esbuildLog(msgs: es.Message[], error = false): void { + for (const msg of msgs) { + const file = msg.location?.file.replace(/^[./]*/, '') ?? ''; + const line = msg.location?.line + ? `:${msg.location.line}` + : '' + (msg.location?.column ? `:${msg.location.column}` : ''); + const srcText = msg.location?.lineText; + env.log(`${error ? errorMark : c.warn('WARNING')} - '${c.cyan(file + line)}' - ${msg.text}`, { + ctx: 'esbuild', + }); + if (srcText) env.log(' ' + c.magenta(srcText), { ctx: 'esbuild' }); + } +} + +async function jsManifest(meta: es.Metafile = { inputs: {}, outputs: {} }) { + for (const w of inlineWatch) w.close(); + inlineWatch.length = 0; + clearTimeout(inlineTimer); + + const newJsManifest: Manifest = {}; + for (const [filename, info] of Object.entries(meta.outputs)) { + const out = parsePath(filename); + if (!out) continue; + if (out.name === 'common') { + out.name = `common.${out.hash}`; + newJsManifest[out.name] = {}; + } else newJsManifest[out.name] = { hash: out.hash }; + const imports: string[] = []; + for (const imp of info.imports) { + if (imp.kind === 'import-statement') { + const path = parsePath(imp.path); + if (path) imports.push(`${path.name}.${path.hash}.js`); + } + } + newJsManifest[out.name].imports = imports; + } + await inlineManifest(newJsManifest); +} + +async function inlineManifest(js: Manifest) { + const makeWatchers = env.watch && inlineWatch.length === 0; + let success = true; + for (const pkg of env.building) { + for (const bundle of pkg.bundle ?? []) { + if (!bundle.inline) continue; + + const inlineSrc = path.join(pkg.root, bundle.inline); + const moduleName = bundle.module + ? path.basename(bundle.module, '.ts') + : path.basename(bundle.inline, '.inline.ts'); + const packageError = `${errorMark} - Package error ${c.blue(JSON.stringify(bundle))}`; + + if (!(await readable(inlineSrc))) { + env.log(packageError); + for (const w of inlineWatch) w.close(); + inlineWatch.length = 0; + if (!env.watch) env.exit('Failed'); // all inline sources must exist + } + + try { + const res = await es.transform(await fs.promises.readFile(inlineSrc), { + minify: true, + loader: 'ts', + }); + esbuildLog(res.warnings); + js[moduleName] ??= {}; + js[moduleName].inline = res.code; + } catch (e) { + if (e && typeof e === 'object' && 'errors' in e) esbuildLog((e as es.TransformFailure).errors, true); + else env.log(`${packageError} - ${JSON.stringify(e)}`); + if (env.watch) success = false; + else env.exit('Failed'); + } + if (makeWatchers) + inlineWatch.push( + fs.watch(inlineSrc, () => { + clearTimeout(inlineTimer); + inlineTimer = setTimeout(() => inlineManifest(js), 200); + }), + ); + } + } + if (success) updateManifest({ js }); +} + +function parsePath(path: string) { + const match = path.match(/\/public\/compiled\/(.*)\.([A-Z0-9]+)\.js$/); + return match ? { name: match[1], hash: match[2] } : undefined; } diff --git a/ui/.build/src/i18n.ts b/ui/.build/src/i18n.ts index 0da4a106a2f76..f029a3d1a5154 100644 --- a/ui/.build/src/i18n.ts +++ b/ui/.build/src/i18n.ts @@ -1,9 +1,10 @@ import path from 'node:path'; +import crypto from 'node:crypto'; import fs from 'node:fs'; import { XMLParser } from 'fast-xml-parser'; -import { env, colors as c } from './main.ts'; -import { globArray } from './parse.ts'; -import { i18nManifest } from './manifest.ts'; +import { env, colors as c } from './env.ts'; +import { globArray, readable } from './parse.ts'; +import { type Manifest, updateManifest } from './manifest.ts'; import { quantize, zip } from './algo.ts'; import { transform } from 'esbuild'; @@ -14,17 +15,11 @@ let dicts: Map = new Map(); let locales: string[], cats: string[]; let watchTimeout: NodeJS.Timeout | undefined; const i18nWatch: fs.FSWatcher[] = []; -const isFormat = /%(?:[\d]\$)?s/; +const formatStringRe = /%(?:[\d]\$)?s/; -export function stopI18n(): void { - clearTimeout(watchTimeout); - watchTimeout = undefined; - for (const watcher of i18nWatch) watcher.close(); - i18nWatch.length = 0; -} - -export async function i18n(isBoot = true): Promise { +export async function i18n(): Promise { if (!env.i18n) return; + if (!watchTimeout) env.log(`Building ${c.grey('i18n')}`); [locales, cats] = ( await Promise.all([ @@ -34,13 +29,13 @@ export async function i18n(isBoot = true): Promise { ).map(list => list.map(x => x.split('.')[0])); await compileTypings(); - compileJavascripts(isBoot); // no await + compileJavascripts(!watchTimeout); // !watchTimeout means first time run - if (!isBoot || !env.watch) return; + if (i18nWatch.length || !env.watch) return; const onChange = () => { clearTimeout(watchTimeout); - watchTimeout = setTimeout(() => i18n(false), 2000); + watchTimeout = setTimeout(() => i18n(), 2000); }; i18nWatch.push(fs.watch(env.i18nSrcDir, onChange)); for (const d of cats) { @@ -49,6 +44,12 @@ export async function i18n(isBoot = true): Promise { } } +export function stopI18nWatch(): void { + clearTimeout(watchTimeout); + for (const watcher of i18nWatch) watcher.close(); + i18nWatch.length = 0; +} + async function compileTypings(): Promise { const typingsPathname = path.join(env.typesDir, 'lichess', `i18n.d.ts`); const [tstat] = await Promise.all([ @@ -58,7 +59,6 @@ async function compileTypings(): Promise { const catStats = await Promise.all(cats.map(d => updated(d))); if (!tstat || catStats.some(x => x)) { - env.log(`Building ${c.grey('i18n')}`); dicts = new Map( zip( cats, @@ -78,7 +78,7 @@ async function compileTypings(): Promise { .map(([k, v]) => { if (!/^[A-Za-z_]\w*$/.test(k)) k = `'${k}'`; const tpe = - typeof v !== 'string' ? 'I18nPlural' : isFormat.test(v) ? 'I18nFormat' : 'string'; + typeof v !== 'string' ? 'I18nPlural' : formatStringRe.test(v) ? 'I18nFormat' : 'string'; const comment = typeof v === 'string' ? v.split('\n')[0] : v['other']?.split('\n')[0]; return ` /** ${comment} */\n ${k}: ${tpe};`; }) @@ -96,22 +96,28 @@ async function compileTypings(): Promise { } } -async function compileJavascripts(dirty: boolean = true): Promise { +async function compileJavascripts(withManifest: boolean = false): Promise { + const reportOnce = () => { + if (!withManifest) env.log(`Building ${c.grey('i18n')}`); + withManifest = true; + }; for (const cat of cats) { const u = await updated(cat); - if (u) await writeJavascript(cat, undefined, u); + if (u) { + reportOnce(); + await writeJavascript(cat, undefined, u); + } await Promise.all( locales.map(locale => updated(cat, locale).then(xstat => { if (!u && !xstat) return; - if (!dirty) env.log(`Building ${c.grey('i18n')}`); - dirty = true; + reportOnce(); return writeJavascript(cat, locale, xstat); }), ), ); } - if (dirty) i18nManifest(); + if (withManifest) i18nManifest(); } async function writeJavascript(cat: string, locale?: string, xstat: fs.Stats | false = false) { @@ -148,7 +154,7 @@ async function writeJavascript(cat: string, locale?: string, xstat: fs.Stats | f `i['${k}']=` + (typeof v !== 'string' ? `p(${JSON.stringify(v)})` - : isFormat.test(v) + : formatStringRe.test(v) ? `s(${JSON.stringify(v)})` : JSON.stringify(v)), ) @@ -196,6 +202,21 @@ async function min(js: string): Promise { return (await transform(js, { minify: true, loader: 'js' })).code; } +export async function i18nManifest(): Promise { + const i18nManifest: Manifest = {}; + fs.mkdirSync(path.join(env.jsOutDir, 'i18n'), { recursive: true }); + const scripts = await globArray('*.js', { cwd: env.i18nJsDir }); + for (const file of scripts) { + const name = `i18n/${path.basename(file, '.js')}`; + const content = await fs.promises.readFile(file, 'utf-8'); + const hash = crypto.createHash('md5').update(content).digest('hex').slice(0, 12); + const destPath = path.join(env.jsOutDir, `${name}.${hash}.js`); + i18nManifest[name] = { hash }; + if (!(await readable(destPath))) await fs.promises.writeFile(destPath, content); + } + updateManifest({ i18n: i18nManifest }); +} + const tsPrelude = `// Generated interface I18nFormat { (...args: (string | number)[]): string; // formatted diff --git a/ui/.build/src/main.ts b/ui/.build/src/main.ts index 57823f9b2074d..c99309adbb318 100644 --- a/ui/.build/src/main.ts +++ b/ui/.build/src/main.ts @@ -2,8 +2,12 @@ import ps from 'node:process'; import path from 'node:path'; import fs from 'node:fs'; import { deepClean } from './clean.ts'; -import { build, postBuild } from './build.ts'; +import { build } from './build.ts'; import { startConsole } from './console.ts'; +import { env } from './env.ts'; + +// process arguments and kick off the build +// it more or less fits on one page so no need for an args library // readme should be up to date but this is the definitive list of flags const args: Record = { @@ -22,285 +26,55 @@ const args: Record = { '--test': 't', '--clean-exit': '', '--clean': 'c', - '--update': '', '--no-install': 'n', '--log': 'l', }; +const argv = ps.argv.slice(2); +const oneDashRe = /^-([a-z]+)(?:=[a-zA-Z0-9-_:./]+)?$/; -type Builder = 'sass' | 'tsc' | 'esbuild'; - -export async function main(): Promise { - const argv = ps.argv.slice(2); - const oneDashRe = /^-([a-z]+)(?:=[a-zA-Z0-9-_:./]+)?$/; - const stringArg = (arg: string): string | boolean => { - const it = argv.find( - x => x.startsWith(arg) || (args[arg] && oneDashRe.exec(x)?.[1]?.includes(args[arg])), - ); - return it?.split('=')[1] ?? (it ? true : false); - }; - const oneDashArgs = argv - .flatMap(x => oneDashRe.exec(x)?.[1] ?? '') - .join('') - .split(''); - oneDashArgs - .filter(x => !Object.values(args).includes(x)) - .forEach(arg => env.exit(`Unknown flag '-${arg}'`)); - argv - .filter(x => x.startsWith('--') && !Object.keys(args).includes(x.split('=')[0])) - .forEach(arg => env.exit(`Unknown argument '${arg}'`)); - - if (['--tsc', '--sass', '--esbuild', '--sync', '--i18n'].filter(x => argv.includes(x)).length) { - // including one or more of these disables the others - if (!argv.includes('--sass')) env.exitCode.set('sass', false); - if (!argv.includes('--tsc')) env.exitCode.set('tsc', false); - if (!argv.includes('--esbuild')) env.exitCode.set('esbuild', false); - env.i18n = argv.includes('--i18n'); - env.sync = argv.includes('--sync'); - } - if (argv.includes('--no-color')) env.color = undefined; - - env.logTime = !argv.includes('--no-time'); - env.logContext = !argv.includes('--no-context'); - env.watch = argv.includes('--watch') || oneDashArgs.includes('w'); - env.prod = argv.includes('--prod') || oneDashArgs.includes('p'); - env.debug = argv.includes('--debug') || oneDashArgs.includes('d'); - env.remoteLog = stringArg('--log'); - env.clean = argv.some(x => x.startsWith('--clean')) || oneDashArgs.includes('c'); - env.install = !argv.includes('--no-install') && !oneDashArgs.includes('n'); - env.rgb = argv.includes('--rgb'); - env.test = argv.includes('--test') || oneDashArgs.includes('t'); - - if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { - console.log(fs.readFileSync(path.resolve(env.buildDir, 'readme'), 'utf8')); - return; - } else if (env.clean) { - await deepClean(); - if (argv.includes('--clean-exit')) return; - } - startConsole(); - build(argv.filter(x => !x.startsWith('-'))); -} - -export interface Package { - root: string; // absolute path to package.json parentdir (package root) - name: string; // dirname of package root - pkg: any; // the entire package.json object - pre: string[][]; // pre-bundle build steps from package.json scripts - post: string[][]; // post-bundle build steps from package.json scripts - bundles?: string[]; - hashGlobs?: string[]; - sync?: Sync[]; // pre-bundle filesystem copies from package json -} - -export interface Sync { - // src must be a file or a glob expression, use /** to sync entire directories - src: string; - dest: string; - pkg: Package; -} - -export const lines = (s: string): string[] => s.split(/[\n\r\f]+/).filter(x => x.trim()); - -const colorLines = (text: string, code: string) => - lines(text) - .map(t => (env.color ? escape(t, code) : t)) - .join('\n'); - -export const colors: Record string> = { - red: (text: string): string => colorLines(text, codes.red), - green: (text: string): string => colorLines(text, codes.green), - yellow: (text: string): string => colorLines(text, codes.yellow), - blue: (text: string): string => colorLines(text, codes.blue), - magenta: (text: string): string => colorLines(text, codes.magenta), - cyan: (text: string): string => colorLines(text, codes.cyan), - grey: (text: string): string => colorLines(text, codes.grey), - black: (text: string): string => colorLines(text, codes.black), - error: (text: string): string => colorLines(text, codes.error), - warn: (text: string): string => colorLines(text, codes.warn), - good: (text: string): string => colorLines(text, codes.green + ';1'), - cyanBold: (text: string): string => colorLines(text, codes.cyan + ';1'), -}; - -class Env { - rootDir: string = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..'); // absolute path to lila project root - - deps: Map = new Map(); - packages: Map = new Map(); - building: Package[] = []; - - watch = false; - clean = false; - prod = false; - debug = false; - remoteLog: string | boolean = false; - rgb = false; - install = true; - sync = true; - i18n = true; - test = false; - exitCode: Map = new Map(); - startTime: number | undefined = Date.now(); - logTime = true; - logContext = true; - color: any = { - build: 'green', - sass: 'magenta', - tsc: 'yellow', - esbuild: 'blue', - }; - - get sass(): boolean { - return this.exitCode.get('sass') !== false; - } - get tsc(): boolean { - return this.exitCode.get('tsc') !== false; - } - get esbuild(): boolean { - return this.exitCode.get('esbuild') !== false; - } - get manifestOk(): boolean { - return (['tsc', 'esbuild', 'sass'] as const).every( - x => this.exitCode.get(x) === 0 || this.exitCode.get(x) === false, - ); - } - get uiDir(): string { - return path.join(this.rootDir, 'ui'); - } - get outDir(): string { - return path.join(this.rootDir, 'public'); - } - get cssOutDir(): string { - return path.join(this.outDir, 'css'); - } - get jsOutDir(): string { - return path.join(this.outDir, 'compiled'); - } - get hashOutDir(): string { - return path.join(this.outDir, 'hashed'); - } - get themeDir(): string { - return path.join(this.uiDir, 'common', 'css', 'theme'); - } - get themeGenDir(): string { - return path.join(this.themeDir, 'gen'); - } - get buildDir(): string { - return path.join(this.uiDir, '.build'); - } - get cssTempDir(): string { - return path.join(this.buildDir, 'build', 'css'); - } - get buildSrcDir(): string { - return path.join(this.uiDir, '.build', 'src'); - } - get buildTempDir(): string { - return path.join(this.buildDir, 'build'); - } - get typesDir(): string { - return path.join(this.uiDir, '@types'); - } - get i18nSrcDir(): string { - return path.join(this.rootDir, 'translation', 'source'); - } - get i18nDestDir(): string { - return path.join(this.rootDir, 'translation', 'dest'); - } - get i18nJsDir(): string { - return path.join(this.rootDir, 'translation', 'js'); - } - get manifestFile(): string { - return path.join(this.jsOutDir, `manifest.${this.prod ? 'prod' : 'dev'}.json`); - } - warn(d: any, ctx = 'build'): void { - this.log(d, { ctx: ctx, warn: true }); - } - error(d: any, ctx = 'build'): void { - this.log(d, { ctx: ctx, error: true }); - } - exit(d: any, ctx = 'build'): void { - this.log(d, { ctx: ctx, error: true }); - process.exit(1); - } - good(ctx = 'build'): void { - this.log(colors.good('No errors') + env.watch ? ` - ${colors.grey('Watching')}...` : '', { ctx: ctx }); - } - log(d: any, { ctx = 'build', error = false, warn = false }: any = {}): void { - let text: string = - !d || typeof d === 'string' || d instanceof Buffer - ? String(d) - : Array.isArray(d) - ? d.join('\n') - : JSON.stringify(d); - - const esc = this.color ? escape : (text: string, _: any) => text; - - if (!this.color) text = stripColorEscapes(text); - - const prefix = ( - (this.logTime === false ? '' : prettyTime()) + - (!ctx || !this.logContext ? '' : `[${esc(ctx, colorForCtx(ctx, this.color))}] `) - ).trim(); - - lines(text).forEach(line => - console.log( - `${prefix ? prefix + ' - ' : ''}${ - error ? esc(line, codes.error) : warn ? esc(line, codes.warn) : line - }`, - ), - ); - } - done(code: number, ctx: Builder): void { - this.exitCode.set(ctx, code); - const err = [...this.exitCode.values()].find(x => x); - const allDone = this.exitCode.size === 3; - if (ctx !== 'tsc' || code === 0) - this.log( - `${code === 0 ? 'Done' : colors.red('Failed')}` + - (this.watch ? ` - ${colors.grey('Watching')}...` : ''), - { ctx }, - ); - if (allDone) { - if (!err) postBuild(); - if (this.startTime && !err) - this.log(`Done in ${colors.green((Date.now() - this.startTime) / 1000 + '')}s`); - this.startTime = undefined; // it's pointless to time subsequent builds, they are too fast - } - if (!env.watch && err) process.exitCode = err; - } -} - -export const env: Env = new Env(); - -export const codes: Record = { - black: '30', - red: '31', - green: '32', - yellow: '33', - blue: '34', - magenta: '35', - cyan: '36', - grey: '90', - error: '31', - warn: '33', +const stringArg = (arg: string): string | boolean => { + const it = argv.find(x => x.startsWith(arg) || (args[arg] && oneDashRe.exec(x)?.[1]?.includes(args[arg]))); + return it?.split('=')[1] ?? (it ? true : false); }; -const colorForCtx = (ctx: string, color: any): string => - color && ctx in color && color[ctx] in codes ? codes[color[ctx]] : codes.grey; +const oneDashArgs = argv + .flatMap(x => oneDashRe.exec(x)?.[1] ?? '') + .join('') + .split(''); -const escape = (text: string, code: string): string => `\x1b[${code}m${stripColorEscapes(text)}\x1b[0m`; +oneDashArgs.filter(x => !Object.values(args).includes(x)).forEach(arg => env.exit(`Unknown flag '-${arg}'`)); -const pad2 = (n: number) => (n < 10 ? `0${n}` : `${n}`); +argv + .filter(x => x.startsWith('--') && !Object.keys(args).includes(x.split('=')[0])) + .forEach(arg => env.exit(`Unknown argument '${arg}'`)); -function stripColorEscapes(text: string) { - return text.replace(/\x1b\[[0-9;]*m/, ''); +if (['--tsc', '--sass', '--esbuild', '--sync', '--i18n'].filter(x => argv.includes(x)).length) { + // including one or more of these disables the others + if (!argv.includes('--sass')) env.exitCode.set('sass', false); + if (!argv.includes('--tsc')) env.exitCode.set('tsc', false); + if (!argv.includes('--esbuild')) env.exitCode.set('esbuild', false); + env.i18n = argv.includes('--i18n'); + env.sync = argv.includes('--sync'); } - -export const errorMark: string = colors.red('✘ ') + colors.error('[ERROR]'); -export const warnMark: string = colors.yellow('⚠ ') + colors.warn('[WARNING]'); - -function prettyTime() { - const now = new Date(); - return `${pad2(now.getHours())}:${pad2(now.getMinutes())}:${pad2(now.getSeconds())} `; +if (argv.includes('--no-color')) env.color = undefined; + +env.logTime = !argv.includes('--no-time'); +env.logContext = !argv.includes('--no-context'); +env.watch = argv.includes('--watch') || oneDashArgs.includes('w'); +env.prod = argv.includes('--prod') || oneDashArgs.includes('p'); +env.debug = argv.includes('--debug') || oneDashArgs.includes('d'); +env.remoteLog = stringArg('--log'); +env.clean = argv.some(x => x.startsWith('--clean')) || oneDashArgs.includes('c'); +env.install = !argv.includes('--no-install') && !oneDashArgs.includes('n'); +env.rgb = argv.includes('--rgb'); +env.test = argv.includes('--test') || oneDashArgs.includes('t'); + +if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { + console.log(fs.readFileSync(path.resolve(env.buildDir, 'usage.txt'), 'utf8')); + process.exit(0); +} else if (env.clean) { + await deepClean(); + if (argv.includes('--clean-exit')) process.exit(0); } - -main(); +startConsole(); +build(argv.filter(x => !x.startsWith('-'))); diff --git a/ui/.build/src/manifest.ts b/ui/.build/src/manifest.ts index d09f4ff62d2cb..9f2160cdb073e 100644 --- a/ui/.build/src/manifest.ts +++ b/ui/.build/src/manifest.ts @@ -2,15 +2,17 @@ import cps from 'node:child_process'; import path from 'node:path'; import fs from 'node:fs'; import crypto from 'node:crypto'; -import es from 'esbuild'; -import { env, colors as c, warnMark } from './main.ts'; -import { globArray, globArrays } from './parse.ts'; -import { allSources } from './sass.ts'; +import { env, colors as c, warnMark } from './env.ts'; +import { allSources as allCssSources } from './sass.ts'; import { jsLogger } from './console.ts'; +import { shallowSort, isEquivalent } from './algo.ts'; +let writeTimer: NodeJS.Timeout; -export type Manifest = { [key: string]: { hash?: string; imports?: string[]; mtime?: number } }; +export type Manifest = { + [key: string]: { hash?: string; imports?: string[]; inline?: string; mtime?: number }; +}; -const current: { js: Manifest; i18n: Manifest; css: Manifest; hashed: Manifest; dirty: boolean } = { +export const current: { js: Manifest; i18n: Manifest; css: Manifest; hashed: Manifest; dirty: boolean } = { i18n: {}, js: {}, css: {}, @@ -18,97 +20,25 @@ const current: { js: Manifest; i18n: Manifest; css: Manifest; hashed: Manifest; dirty: false, }; -let writeTimer: NodeJS.Timeout; - -export function writeManifest(): void { - if (!current.dirty) return; +export function stopManifest(): void { clearTimeout(writeTimer); - writeTimer = setTimeout(write, 500); -} - -export function jsManifest(meta: es.Metafile): void { - const newJsManifest: Manifest = {}; - for (const [filename, info] of Object.entries(meta.outputs)) { - const out = parsePath(filename); - if (!out) continue; - if (out.name === 'common') { - out.name = `common.${out.hash}`; - newJsManifest[out.name] = {}; - } else newJsManifest[out.name] = { hash: out.hash }; - const imports: string[] = []; - for (const imp of info.imports) { - if (imp.kind === 'import-statement') { - const path = parsePath(imp.path); - if (path) imports.push(`${path.name}.${path.hash}.js`); - } - } - newJsManifest[out.name].imports = imports; - } - if (isEquivalent(newJsManifest, current.js) && fs.existsSync(env.manifestFile)) return; - current.js = shallowSort({ ...current.js, ...newJsManifest }); - current.dirty = true; -} - -export async function cssManifest(): Promise { - const files = await globArray(path.join(env.cssTempDir, '*.css')); - const css: { name: string; hash: string }[] = await Promise.all(files.map(hashMoveCss)); - const newCssManifest: Manifest = {}; - for (const { name, hash } of css) newCssManifest[name] = { hash }; - if (isEquivalent(newCssManifest, current.css)) return; - current.css = shallowSort({ ...current.css, ...newCssManifest }); - current.dirty = true; - writeManifest(); -} - -export async function hashedManifest(): Promise { - const newHashLinks = new Map(); - const alreadyHashed = new Map(); - const sources: string[] = await globArrays( - env.building.flatMap(x => x.hashGlobs ?? []), - { cwd: env.outDir }, - ); - const sourceStats = await Promise.all(sources.map(file => fs.promises.stat(file))); - - for (const [i, stat] of sourceStats.entries()) { - const name = sources[i].slice(env.outDir.length + 1); - - if (stat.mtimeMs === current.hashed[name]?.mtime) alreadyHashed.set(name, current.hashed[name].hash!); - else newHashLinks.set(name, stat.mtimeMs); - } - await Promise.allSettled([...alreadyHashed].map(([name, hash]) => link(name, hash))); - - for (const { name, hash } of await Promise.all([...newHashLinks.keys()].map(hashLink))) { - current.hashed[name] = Object.defineProperty({ hash }, 'mtime', { value: newHashLinks.get(name) }); - } - - if (newHashLinks.size === 0 && alreadyHashed.size === Object.keys(current.hashed).length) return; - - for (const name of Object.keys(current.hashed)) { - if (!sources.some(x => x.endsWith(name))) delete current.hashed[name]; - } - current.dirty = true; - writeManifest(); } -export async function i18nManifest(): Promise { - const i18nManifest: Manifest = {}; - fs.mkdirSync(path.join(env.jsOutDir, 'i18n'), { recursive: true }); - const scripts = await globArray('*.js', { cwd: env.i18nJsDir }); - for (const file of scripts) { - const name = `i18n/${path.basename(file, '.js')}`; - const content = await fs.promises.readFile(file, 'utf-8'); - const hash = crypto.createHash('md5').update(content).digest('hex').slice(0, 12); - const destPath = path.join(env.jsOutDir, `${name}.${hash}.js`); - i18nManifest[name] = { hash }; - if (!fs.existsSync(destPath)) await fs.promises.writeFile(destPath, content); +export function updateManifest(update: Partial = {}): void { + if (update?.dirty) current.dirty = true; + for (const key of Object.keys(update ?? {}) as (keyof typeof current)[]) { + if (key === 'dirty' || isEquivalent(current[key], update?.[key])) continue; + current[key] = shallowSort({ ...current[key], ...update?.[key] }); + current.dirty = true; } - current.i18n = shallowSort(i18nManifest); - current.dirty = true; - writeManifest(); + if (!current.dirty) return; + clearTimeout(writeTimer); + writeTimer = setTimeout(writeManifest, 500); } -async function write() { +async function writeManifest() { if (!env.manifestOk || !(await isComplete())) return; + const commitMessage = cps .execSync('git log -1 --pretty=%s', { encoding: 'utf-8' }) .trim() @@ -155,83 +85,28 @@ async function write() { ), ]); current.dirty = false; - env.log(`Client manifest '${c.cyan(`public/compiled/manifest.${hash}.js`)}'`); - env.log(`Server manifest '${c.cyan(`public/compiled/manifest.${env.prod ? 'prod' : 'dev'}.json`)}'`); -} - -async function hashMoveCss(src: string) { - const content = await fs.promises.readFile(src, 'utf-8'); - const hash = crypto.createHash('sha256').update(content).digest('hex').slice(0, 8); - const basename = path.basename(src, '.css'); - await Promise.allSettled([ - env.prod ? undefined : fs.promises.rename(`${src}.map`, path.join(env.cssOutDir, `${basename}.css.map`)), - fs.promises.rename(src, path.join(env.cssOutDir, `${basename}.${hash}.css`)), - ]); - return { name: path.basename(src, '.css'), hash }; -} - -async function hashLink(name: string) { - const src = path.join(env.outDir, name); - const hash = crypto - .createHash('sha256') - .update(await fs.promises.readFile(src)) - .digest('hex') - .slice(0, 8); - link(name, hash); - return { name, hash }; + env.log( + `Manifest '${c.cyan(`public/compiled/manifest.${env.prod ? 'prod' : 'dev'}.json`)}' -> '${c.cyan( + `public/compiled/manifest.${hash}.js`, + )}'`, + ); } async function isComplete() { - for (const bundle of [...env.packages.values()].map(x => x.bundles ?? []).flat()) { - const name = path.basename(bundle, '.ts'); + for (const bundle of [...env.packages.values()].map(x => x.bundle ?? []).flat()) { + if (!bundle.module) continue; + const name = path.basename(bundle.module, '.ts'); if (!current.js[name]) { env.log(`${warnMark} - No manifest without building '${c.cyan(name + '.ts')}'`); return false; } } - for (const css of await allSources()) { + for (const css of await allCssSources()) { const name = path.basename(css, '.scss'); if (!current.css[name]) { env.log(`${warnMark} - No manifest without building '${c.cyan(name + '.scss')}'`); return false; } } - return Object.keys(current.i18n).length; -} - -function shallowSort(obj: { [key: string]: any }): { [key: string]: any } { - // es6 string properties are insertion order, we need more determinism - const sorted: { [key: string]: any } = {}; - for (const key of Object.keys(obj).sort()) sorted[key] = obj[key]; - return sorted; -} - -function parsePath(path: string) { - const match = path.match(/\/public\/compiled\/(.*)\.([A-Z0-9]+)\.js$/); - return match ? { name: match[1], hash: match[2] } : undefined; -} - -function isEquivalent(a: any, b: any): boolean { - if (a === b) return true; - if (typeof a !== typeof b) return false; - if (Array.isArray(a)) - return Array.isArray(b) && a.length === b.length && a.every(x => b.find((y: any) => isEquivalent(x, y))); - if (typeof a !== 'object') return false; - const [aKeys, bKeys] = [Object.keys(a), Object.keys(b)]; - if (aKeys.length !== bKeys.length) return false; - for (const key of aKeys) { - if (!bKeys.includes(key) || !isEquivalent(a[key], b[key])) return false; - } - return true; -} - -function asHashed(path: string, hash: string) { - const name = path.slice(path.lastIndexOf('/') + 1); - const extPos = name.indexOf('.'); - return extPos < 0 ? `${name}.${hash}` : `${name.slice(0, extPos)}.${hash}${name.slice(extPos)}`; -} - -function link(name: string, hash: string) { - const link = path.join(env.hashOutDir, asHashed(name, hash)); - fs.promises.symlink(path.join('..', name), link).catch(() => {}); + return Object.keys(current.i18n).length > 0; } diff --git a/ui/.build/src/monitor.ts b/ui/.build/src/monitor.ts deleted file mode 100644 index fcccde5f11625..0000000000000 --- a/ui/.build/src/monitor.ts +++ /dev/null @@ -1,59 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import ps from 'node:process'; -import { build, stopBuild } from './build.ts'; -import { env } from './main.ts'; -import { clean } from './clean.ts'; -import { globArray } from './parse.ts'; -import { stopTsc, tsc } from './tsc.ts'; -import { stopEsbuild, esbuild } from './esbuild.ts'; - -const watchers: fs.FSWatcher[] = []; - -let packageTimeout: NodeJS.Timeout | undefined; -let tscTimeout: NodeJS.Timeout | undefined; - -export function stopMonitor(): void { - for (const w of watchers) w.close(); - watchers.length = 0; - clearTimeout(tscTimeout); - clearTimeout(packageTimeout); - tscTimeout = packageTimeout = undefined; -} - -export async function monitor(pkgs: string[]): Promise { - if (!env.watch) return; - const [typePkgs, typings] = await Promise.all([ - globArray('*/package.json', { cwd: env.typesDir }), - globArray('*/*.d.ts', { cwd: env.typesDir }), - ]); - const tscChange = async () => { - if (packageTimeout) return; - await stopTsc(); - await stopEsbuild(); - clearTimeout(tscTimeout); - tscTimeout = setTimeout(() => { - if (packageTimeout) return; - tsc().then(esbuild); - }, 2000); - }; - const packageChange = async () => { - if (env.watch && env.install) { - clearTimeout(tscTimeout); - clearTimeout(packageTimeout); - await stopBuild(); - packageTimeout = setTimeout(() => clean().then(() => build(pkgs)), 2000); - return; - } - env.warn('Exiting due to package.json change'); - ps.exit(0); - }; - - watchers.push(fs.watch(path.join(env.rootDir, 'package.json'), packageChange)); - for (const p of typePkgs) watchers.push(fs.watch(p, packageChange)); - for (const t of typings) watchers.push(fs.watch(t, tscChange)); - for (const pkg of env.building) { - watchers.push(fs.watch(path.join(pkg.root, 'package.json'), packageChange)); - watchers.push(fs.watch(path.join(pkg.root, 'tsconfig.json'), tscChange)); - } -} diff --git a/ui/.build/src/parse.ts b/ui/.build/src/parse.ts index 6a920bc00adfa..60d47562d6c27 100644 --- a/ui/.build/src/parse.ts +++ b/ui/.build/src/parse.ts @@ -1,25 +1,38 @@ import fs from 'node:fs'; import path from 'node:path'; import fg from 'fast-glob'; -import { type Package, env } from './main.ts'; +import { env } from './env.ts'; -export async function parsePackages(): Promise<[Map, Map]> { - const packages = new Map(); - const packageDeps = new Map(); +export type Bundle = { module?: string; inline?: string }; +export interface Package { + root: string; // absolute path to package.json parentdir (package root) + name: string; // dirname of package root + pkg: any; // the entire package.json object + bundle: { module?: string; inline?: string }[]; // TODO doc + hashGlobs: string[]; // TODO doc + sync: Sync[]; // pre-bundle filesystem copies from package json +} + +export interface Sync { + src: string; // src must be a file or a glob expression, use /** to sync entire directories + dest: string; // TODO doc + pkg: Package; +} + +export async function parsePackages(): Promise { for (const dir of (await globArray('[^@.]*/package.json')).map(pkg => path.dirname(pkg))) { const pkgInfo = await parsePackage(dir); - packages.set(pkgInfo.name, pkgInfo); + env.packages.set(pkgInfo.name, pkgInfo); } - for (const pkgInfo of packages.values()) { + for (const pkgInfo of env.packages.values()) { const deplist: string[] = []; for (const dep in pkgInfo.pkg.dependencies) { - if (packages.has(dep)) deplist.push(dep); + if (env.packages.has(dep)) deplist.push(dep); } - packageDeps.set(pkgInfo.name, deplist); + env.workspaceDeps.set(pkgInfo.name, deplist); } - return [packages, packageDeps]; } export async function globArray(glob: string, opts: fg.Options = {}): Promise { @@ -45,32 +58,49 @@ export async function folderSize(folder: string): Promise { else if (file.isFile()) totalSize += (await fs.promises.stat(path.join(dir, file.name))).size; } } - await getSize(folder); - return totalSize; } +export async function readable(file: string): Promise { + return fs.promises + .access(file, fs.constants.R_OK) + .then(() => true) + .catch(() => false); +} + async function parsePackage(packageDir: string): Promise { - const pkg = JSON.parse(await fs.promises.readFile(path.join(packageDir, 'package.json'), 'utf8')); const pkgInfo: Package = { - pkg, + pkg: JSON.parse(await fs.promises.readFile(path.join(packageDir, 'package.json'), 'utf8')), name: path.basename(packageDir), root: packageDir, - pre: [], - post: [], + bundle: [], + sync: [], + hashGlobs: [], }; + if (!('build' in pkgInfo.pkg)) return pkgInfo; + const build = pkgInfo.pkg.build; - if ('lichess' in pkg && 'hashed' in pkg.lichess) pkgInfo.hashGlobs = pkg.lichess.hashed as string[]; + if ('hash' in build) pkgInfo.hashGlobs = [].concat(build.hash); - if ('lichess' in pkg && 'bundles' in pkg.lichess) { - if (typeof pkg.lichess.bundles === 'string') pkgInfo.bundles = [pkg.lichess.bundles]; - else pkgInfo.bundles = pkg.lichess.bundles as string[]; + if ('bundle' in build) { + for (const one of [].concat(build.bundle).map(b => (typeof b === 'string' ? { module: b } : b))) { + if (!one.module) continue; + + if (await readable(path.join(pkgInfo.root, one.module))) pkgInfo.bundle.push(one); + else + pkgInfo.bundle.push( + ...(await globArray(one.module, { cwd: pkgInfo.root, absolute: false })).map(module => ({ + ...one, + module, + })), + ); + } } - if ('lichess' in pkg && 'sync' in pkg.lichess) { - pkgInfo.sync = Object.entries(pkg.lichess.sync).map(x => ({ + if ('sync' in build) { + pkgInfo.sync = Object.entries(build.sync).map(x => ({ src: x[0], - dest: x[1] as string, + dest: x[1], pkg: pkgInfo, })); } diff --git a/ui/.build/src/sass.ts b/ui/.build/src/sass.ts index 5fdd376ee3331..79fde199b2d9c 100644 --- a/ui/.build/src/sass.ts +++ b/ui/.build/src/sass.ts @@ -2,10 +2,12 @@ import cps from 'node:child_process'; import fs from 'node:fs'; import ps from 'node:process'; import path from 'node:path'; +import crypto from 'node:crypto'; import clr from 'tinycolor2'; -import { env, colors as c, lines, errorMark } from './main.ts'; -import { globArray } from './parse.ts'; -import { cssManifest } from './manifest.ts'; +import { env, colors as c, lines, errorMark } from './env.ts'; +import { globArray, readable } from './parse.ts'; +import { updateManifest } from './manifest.ts'; +import { clamp } from './algo.ts'; const colorMixMap = new Map(); const themeColorMap = new Map>(); @@ -49,11 +51,104 @@ export async function allSources(): Promise { } async function unbuiltSources(): Promise { - return (await allSources()).filter( - src => !fs.existsSync(path.join(env.cssTempDir, `${path.basename(src, '.scss')}.css`)), + return Promise.all( + (await allSources()).filter( + src => !readable(path.join(env.cssTempDir, `${path.basename(src, '.scss')}.css`)), + ), ); } +class SassWatch { + dependencies = new Set(); + touched = new Set(); + timeout: NodeJS.Timeout | undefined; + watchers: fs.FSWatcher[] = []; + watchDirs = new Set(); + constructor() { + this.watch(); + } + + async watch() { + if (!env.watch) return; + const watchDirs = new Set([...importMap.keys()].map(path.dirname)); + (await allSources()).forEach(s => watchDirs.add(path.dirname(s))); + if (this.watchDirs.size === watchDirs.size || [...watchDirs].every(d => this.watchDirs.has(d))) return; + if (this.watchDirs.size) env.log('Rebuilding watchers...', { ctx: 'sass' }); + for (const x of this.watchers) x.close(); + this.watchers.length = 0; + this.watchDirs = watchDirs; + for (const dir of this.watchDirs) { + const fsWatcher = fs.watch(dir); + fsWatcher.on('change', (event: string, srcFile: string) => this.onChange(dir, event, srcFile)); + fsWatcher.on('error', (err: Error) => env.error(err, 'sass')); + this.watchers.push(fsWatcher); + } + } + + destroy() { + this.clear(); + for (const x of this.watchers) x.close(); + this.watchers.length = 0; + } + + clear() { + clearTimeout(this.timeout); + this.timeout = undefined; + this.dependencies.clear(); + this.touched.clear(); + } + + add(files: string[]): boolean { + clearTimeout(this.timeout); + this.timeout = setTimeout(() => this.fire(), 200); + if (files.every(f => this.touched.has(f))) return false; + files.forEach(src => { + this.touched.add(src); + if (!/[^_].*\.scss/.test(path.basename(src))) { + this.dependencies.add(src); + } else importersOf(src).forEach(dest => this.dependencies.add(dest)); + }); + return true; + } + + onChange(dir: string, event: string, srcFile: string) { + if (event === 'change') { + if (this.add([path.join(dir, srcFile)])) env.log(`File '${c.cyanBold(srcFile)}' changed`); + } else if (event === 'rename') { + globArray('*.scss', { cwd: dir, absolute: false }).then(files => { + if (this.add(files.map(f => path.join(dir, f)))) { + env.log(`Cross your fingers - directory '${c.cyanBold(dir)}' changed`, { ctx: 'sass' }); + } + }); + } + } + + async fire() { + const sources = [...this.dependencies].filter(src => /\/[^_][^/]+\.scss$/.test(src)); + const touched = [...this.touched]; + this.clear(); + this.watch(); // experimental + let rebuildColors = false; + + for (const src of touched) { + processed.delete(src); + if (src.includes('common/css/theme/_')) { + rebuildColors = true; + await parseThemeColorDefs(); + } + } + const oldMixSet = new Set([...colorMixMap.keys()]); + for (const src of touched) await parseScss(src); + const newMixSet = new Set([...colorMixMap.keys()]); + if ([...newMixSet].some(mix => !oldMixSet.has(mix))) rebuildColors = true; + if (rebuildColors) { + buildColorMixes() + .then(buildColorWrap) + .then(() => compile(sources)); + } else compile(sources); + } +} + // compile an array of concrete scss files to ui/.build/dist/css/*.css (css temp dir prior to hashMove) function compile(sources: string[], logAll = true) { if (!sources.length) return sources.length; @@ -119,7 +214,7 @@ async function parseScss(src: string) { for (const match of text.matchAll(/^@(?:import|use)\s+['"](.*)['"]/gm)) { if (match.length !== 2) continue; - const absDep = fs.existsSync(path.resolve(path.dirname(src), match[1]) + '.scss') + const absDep = (await readable(path.resolve(path.dirname(src), match[1]) + '.scss')) ? path.resolve(path.dirname(src), match[1] + '.scss') : path.resolve(path.dirname(src), resolvePartial(match[1])); @@ -171,16 +266,18 @@ async function buildColorMixes() { for (const [colorMix, mix] of colorMixMap) { const c1 = colorMap.get(mix.c1)?.clone() ?? new clr(mix.c1); const c2 = (mix.c2 ? colorMap.get(mix.c2) : undefined) ?? new clr(mix.c2); - const mixed = - mix.op === 'mix' - ? clr.mix(c2!, c1, clamp(mix.val, { min: 0, max: 100 })) - : mix.op === 'lighten' - ? c1.lighten(clamp(mix.val, { min: 0, max: 100 })) - : mix.op === 'alpha' - ? c1.setAlpha(clamp(mix.val / 100, { min: 0, max: 1 })) - : mix.op === 'fade' - ? c1.setAlpha(c1.getAlpha() * (1 - clamp(mix.val / 100, { min: 0, max: 1 }))) - : undefined; + const mixed = (() => { + switch (mix.op) { + case 'mix': + return clr.mix(c2!, c1, clamp(mix.val, { min: 0, max: 100 })); + case 'lighten': + return c1.lighten(clamp(mix.val, { min: 0, max: 100 })); + case 'alpha': + return c1.setAlpha(clamp(mix.val / 100, { min: 0, max: 1 })); + case 'fade': + return c1.setAlpha(c1.getAlpha() * (1 - clamp(mix.val / 100, { min: 0, max: 1 }))); + } + })(); if (mixed) colors.push(` --m-${colorMix}: ${env.rgb ? mixed.toRgbString() : mixed.toHslString()};`); else env.log(`${errorMark} - invalid mix op: '${c.magenta(colorMix)}'`, { ctx: 'sass' }); } @@ -211,7 +308,7 @@ async function buildColorWrap() { const wrapFile = path.join(env.themeDir, 'gen', '_wrap.scss'); await fs.promises.mkdir(path.dirname(wrapFile), { recursive: true }); - if (fs.existsSync(wrapFile)) { + if (await readable(wrapFile)) { if ((await fs.promises.readFile(wrapFile, 'utf8')) === scssWrap) return; // don't touch wrap if no changes } await fs.promises.writeFile(wrapFile, scssWrap); @@ -246,10 +343,6 @@ function sassError(error: string) { } } -function clamp(val: number, { min, max }: { min: number; max: number }) { - return Math.min(max, Math.max(min, val)); -} - function importersOf(srcFile: string, bset = new Set()): Set { if (bset.has(srcFile)) return bset; bset.add(srcFile); @@ -257,93 +350,18 @@ function importersOf(srcFile: string, bset = new Set()): Set { return bset; } -class SassWatch { - dependencies = new Set(); - touched = new Set(); - timeout: NodeJS.Timeout | undefined; - watchers: fs.FSWatcher[] = []; - watchDirs = new Set(); - constructor() { - this.watch(); - } - - async watch() { - if (!env.watch) return; - const watchDirs = new Set([...importMap.keys()].map(path.dirname)); - (await allSources()).forEach(s => watchDirs.add(path.dirname(s))); - if (this.watchDirs.size === watchDirs.size || [...watchDirs].every(d => this.watchDirs.has(d))) return; - if (this.watchDirs.size) env.log('Rebuilding watchers...', { ctx: 'sass' }); - for (const x of this.watchers) x.close(); - this.watchers.length = 0; - this.watchDirs = watchDirs; - for (const dir of this.watchDirs) { - const fsWatcher = fs.watch(dir); - fsWatcher.on('change', (event: string, srcFile: string) => this.onChange(dir, event, srcFile)); - fsWatcher.on('error', (err: Error) => env.error(err, 'sass')); - this.watchers.push(fsWatcher); - } - } - - destroy() { - this.clear(); - for (const x of this.watchers) x.close(); - this.watchers.length = 0; - } - - clear() { - clearTimeout(this.timeout); - this.timeout = undefined; - this.dependencies.clear(); - this.touched.clear(); - } - - add(files: string[]): boolean { - clearTimeout(this.timeout); - this.timeout = setTimeout(() => this.fire(), 200); - if (files.every(f => this.touched.has(f))) return false; - files.forEach(src => { - this.touched.add(src); - if (!/[^_].*\.scss/.test(path.basename(src))) { - this.dependencies.add(src); - } else importersOf(src).forEach(dest => this.dependencies.add(dest)); - }); - return true; - } - - onChange(dir: string, event: string, srcFile: string) { - if (event === 'change') { - if (this.add([path.join(dir, srcFile)])) env.log(`File '${c.cyanBold(srcFile)}' changed`); - } else if (event === 'rename') { - globArray('*.scss', { cwd: dir, absolute: false }).then(files => { - if (this.add(files.map(f => path.join(dir, f)))) { - env.log(`Cross your fingers - directory '${c.cyanBold(dir)}' changed`, { ctx: 'sass' }); - } - }); - } - } - - async fire() { - const sources = [...this.dependencies].filter(src => /\/[^_][^/]+\.scss$/.test(src)); - const touched = [...this.touched]; - this.clear(); - this.watch(); // experimental - let rebuildColors = false; +export async function cssManifest(): Promise { + const files = await globArray(path.join(env.cssTempDir, '*.css')); + updateManifest({ css: Object.fromEntries(await Promise.all(files.map(hashMoveCss))) }); +} - for (const src of touched) { - processed.delete(src); - if (src.includes('common/css/theme/_')) { - rebuildColors = true; - await parseThemeColorDefs(); - } - } - const oldMixSet = new Set([...colorMixMap.keys()]); - for (const src of touched) await parseScss(src); - const newMixSet = new Set([...colorMixMap.keys()]); - if ([...newMixSet].some(mix => !oldMixSet.has(mix))) rebuildColors = true; - if (rebuildColors) { - buildColorMixes() - .then(buildColorWrap) - .then(() => compile(sources)); - } else compile(sources); - } +async function hashMoveCss(src: string) { + const content = await fs.promises.readFile(src, 'utf-8'); + const hash = crypto.createHash('sha256').update(content).digest('hex').slice(0, 8); + const basename = path.basename(src, '.css'); + await Promise.allSettled([ + env.prod ? undefined : fs.promises.rename(`${src}.map`, path.join(env.cssOutDir, `${basename}.css.map`)), + fs.promises.rename(src, path.join(env.cssOutDir, `${basename}.${hash}.css`)), + ]); + return [path.basename(src, '.css'), { hash }]; } diff --git a/ui/.build/src/sync.ts b/ui/.build/src/sync.ts index aeebb1de5d180..eb29af033139e 100644 --- a/ui/.build/src/sync.ts +++ b/ui/.build/src/sync.ts @@ -1,8 +1,9 @@ import fs from 'node:fs'; import path from 'node:path'; -import { globArray, globArrays } from './parse.ts'; -import { hashedManifest, writeManifest } from './manifest.ts'; -import { type Sync, env, errorMark, colors as c } from './main.ts'; +import crypto from 'node:crypto'; +import { type Sync, globArray, globArrays } from './parse.ts'; +import { updateManifest, current } from './manifest.ts'; +import { env, errorMark, colors as c } from './env.ts'; import { quantize } from './algo.ts'; const syncWatch: fs.FSWatcher[] = []; @@ -10,7 +11,6 @@ let watchTimeout: NodeJS.Timeout | undefined; export function stopSync(): void { clearTimeout(watchTimeout); - watchTimeout = undefined; for (const watcher of syncWatch) watcher.close(); syncWatch.length = 0; } @@ -21,7 +21,7 @@ export async function sync(): Promise { const updated = new Set(); for (const pkg of env.building) { - for (const cp of pkg.sync ?? []) { + for (const cp of pkg.sync) { for (const src of await globSync(cp)) { if (env.watch) watched.set(src, [...(watched.get(src) ?? []), cp]); } @@ -33,6 +33,7 @@ export async function sync(): Promise { if (!watched.has(path.dirname(src))) watched.set(path.dirname(src), []); } } + hashedManifest(); if (env.watch) for (const dir of watched.keys()) { const watcher = fs.watch(dir); @@ -40,20 +41,16 @@ export async function sync(): Promise { watcher.on('change', () => { updated.add(dir); clearTimeout(watchTimeout); - watchTimeout = setTimeout(() => { - Promise.all([...updated].flatMap(d => (watched.get(d) ?? []).map(x => globSync(x)))).then(() => { - hashedManifest(); - writeManifest(); - }); + Promise.all([...updated].flatMap(d => (watched.get(d) ?? []).map(x => globSync(x)))).then( + hashedManifest, + ); updated.clear(); - watchTimeout = undefined; }, 2000); }); watcher.on('error', (err: Error) => env.error(err)); syncWatch.push(watcher); } - hashedManifest(); } export function isUnmanagedAsset(absfile: string): boolean { @@ -106,3 +103,54 @@ async function syncOne(absSrc: string, absDest: string, pkgName: string) { env.log(`[${c.grey(pkgName)}] - ${errorMark} - failed sync '${c.cyan(absSrc)}' to '${c.cyan(absDest)}'`); } } + +async function hashedManifest(): Promise { + const newHashLinks = new Map(); + const alreadyHashed = new Map(); + const sources: string[] = await globArrays( + env.building.flatMap(x => x.hashGlobs), + { cwd: env.outDir }, + ); + const sourceStats = await Promise.all(sources.map(file => fs.promises.stat(file))); + + for (const [i, stat] of sourceStats.entries()) { + const name = sources[i].slice(env.outDir.length + 1); + + if (stat.mtimeMs === current.hashed[name]?.mtime) alreadyHashed.set(name, current.hashed[name].hash!); + else newHashLinks.set(name, stat.mtimeMs); + } + await Promise.allSettled([...alreadyHashed].map(([name, hash]) => link(name, hash))); + + for (const { name, hash } of await Promise.all([...newHashLinks.keys()].map(hashLink))) { + current.hashed[name] = Object.defineProperty({ hash }, 'mtime', { value: newHashLinks.get(name) }); + } + + if (newHashLinks.size === 0 && alreadyHashed.size === Object.keys(current.hashed).length) return; + + for (const name of Object.keys(current.hashed)) { + if (!sources.some(x => x.endsWith(name))) delete current.hashed[name]; + } + updateManifest({ dirty: true }); +} + +async function hashLink(name: string) { + const src = path.join(env.outDir, name); + const hash = crypto + .createHash('sha256') + .update(await fs.promises.readFile(src)) + .digest('hex') + .slice(0, 8); + link(name, hash); + return { name, hash }; +} + +function link(name: string, hash: string) { + const link = path.join(env.hashOutDir, asHashed(name, hash)); + fs.promises.symlink(path.join('..', name), link).catch(() => {}); +} + +function asHashed(path: string, hash: string) { + const name = path.slice(path.lastIndexOf('/') + 1); + const extPos = name.indexOf('.'); + return extPos < 0 ? `${name}.${hash}` : `${name.slice(0, extPos)}.${hash}${name.slice(extPos)}`; +} diff --git a/ui/.build/src/tsc.ts b/ui/.build/src/tsc.ts index 759bbd8c7de18..366d26d6f656e 100644 --- a/ui/.build/src/tsc.ts +++ b/ui/.build/src/tsc.ts @@ -1,20 +1,15 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import ts from 'typescript'; import { Worker } from 'node:worker_threads'; -import { env, colors as c, errorMark } from './main.ts'; -import { globArray, folderSize } from './parse.ts'; +import { env, colors as c, errorMark } from './env.ts'; +import { globArray, folderSize, readable } from './parse.ts'; import { clamp } from './algo.ts'; import type { WorkerData, Message } from './tscWorker.ts'; -import ts from 'typescript'; const workers: Worker[] = []; -export async function stopTsc(): Promise { - await Promise.allSettled(workers.map(w => w.terminate())); - workers.length = 0; -} - export async function tsc(): Promise { if (!env.tsc) return; await Promise.allSettled([ @@ -42,13 +37,18 @@ export async function tsc(): Promise { ) .push(cfg); - tscLog(`Transpiling ${c.grey('noCheck')} (${workBuckets.noCheck.length} workers)`); + tscLog(`Typing ${c.grey('noCheck')} (${workBuckets.noCheck.length} workers)`); await assignWork(workBuckets.noCheck, 'noCheck'); tscLog(`Typechecking ${c.grey('noEmit')} (${workBuckets.noEmit.length} workers)`); await assignWork(workBuckets.noEmit, 'noEmit'); } +export async function stopTscWatch(): Promise { + await Promise.allSettled(workers.map(w => w.terminate())); + workers.length = 0; +} + function assignWork(buckets: SplitConfig[][], key: 'noCheck' | 'noEmit') { let resolve: (() => void) | undefined = undefined; const status: ('ok' | 'busy' | 'error')[] = []; @@ -83,9 +83,9 @@ function assignWork(buckets: SplitConfig[][], key: 'noCheck' | 'noEmit') { } // the splitConfig transform generates noCheck and noEmit tsconfigs within the 'build' temp folder. -// each package that emits gets a --noCheck pass for fast transpilations and declarations. -// then we do a --noEmit pass on EVERY package to verify things, regardless of its original emit. -// the declarations from the --noCheck pass allow efficient parallel type checking. +// each workspace depndency package gets a --noCheck pass for fast declaration generation. +// then we do a --noEmit pass on EVERY package to verify things. the declarations from the --noCheck +// pass allow more efficient parallel type checking on most architectures interface SplitConfig { type: 'noEmit' | 'noCheck'; @@ -121,7 +121,7 @@ async function splitConfig(cfgPath: string): Promise { ?.filter((glob: string) => !env.test || !glob.includes('tests')) .map((glob: string) => path.resolve(root, glob.replace('${configDir}', '.'))); config.extends = undefined; - config.references = env.deps + config.references = env.workspaceDeps .get(pkgName) ?.map(ref => ({ path: path.join(env.buildTempDir, 'noCheck', `${ref}.tsconfig.json`) })); @@ -133,7 +133,7 @@ async function splitConfig(cfgPath: string): Promise { 'noEmit', `${pkgName}.tsbuildinfo`, ); - if (env.test && fs.existsSync(path.join(root, 'tests'))) { + if (env.test && (await readable(path.join(root, 'tests')))) { noEmitData.include.push(path.join(root, 'tests')); noEmitData.compilerOptions.rootDir = root; noEmitData.compilerOptions.skipLibCheck = true; @@ -147,6 +147,7 @@ async function splitConfig(cfgPath: string): Promise { const noCheckData = structuredClone(config); const noCheck = path.join(env.buildTempDir, 'noCheck', `${pkgName}.tsconfig.json`); noCheckData.compilerOptions.noCheck = true; + noCheckData.compilerOptions.emitDeclarationOnly = true; noCheckData.compilerOptions.tsBuildInfoFile = path.join( env.buildTempDir, 'noCheck', diff --git a/ui/.build/readme b/ui/.build/usage.txt similarity index 62% rename from ui/.build/readme rename to ui/.build/usage.txt index 71881450d4fa7..8facbb93d6c97 100644 --- a/ui/.build/readme +++ b/ui/.build/usage.txt @@ -1,10 +1,5 @@ Usage: - ui/build # are top level directories in ui - # if no packages are specified, all will be processed - # multiple short options can be preceded by a single dash - -Recommended: - ui/build -cdw # clean, build debug, and watch for changes with clean rebuilds + ui/build # multiple short options can be preceded by a single dash Options: -h, --help show this help and exit @@ -22,7 +17,7 @@ Options: --no-time don't log the time --no-context don't log the context -Exclusive Options: (any of these will disable other functions) +Exclusive Options: (any of these will disable other functions) --clean-exit clean all build artifacts and exit --tsc run tsc on {package}/tsconfig.json and dependencies --sass run sass on {package}/css/build/*.scss and dependencies @@ -30,10 +25,12 @@ Exclusive Options: (any of these will disable other functions) --sync run sync copies (given in {package}/package.json/lichess/sync objects) --i18n build @types/lichess/i18n.d.ts and translation/js files -Examples: - ./build -np # no pnpm install, build minified - ./build -w dasher chart # watch mode for dasher and chart packages - ./build --tsc -w # watch mode but type checking only - ./build -w -l=/path # build, watch. patch console methods in emitted js to also POST messages - # to ${location.origin}/path. ui/build watch process displays received - # messages as 'web' in build logs +Recommended: + ui/build -cdw # clean, build debug, and watch for changes with clean rebuilds + +Other Examples: + ./build -np # no pnpm install, build minified + ./build --tsc -w # watch mode but type checking only + ./build -dwl=/x # build debug, watch. patch console methods in emitted js to POST log statements + to ${location.origin}/x. ui/build watch process displays messages received + via http/s on this endpoint as 'web' in build logs diff --git a/ui/build b/ui/build index 5381e7c2684dc..7b3f2a3eecbc3 100755 --- a/ui/build +++ b/ui/build @@ -5,10 +5,11 @@ set -e cd "$(dirname "${BASH_SOURCE:-$0}")/.build" if ! command -v pnpm &> /dev/null; then - echo "The 'pnpm' tool is required. See https://github.com/lichess-org/lila/wiki/Lichess-Development-Onboarding#tools-and-dependency-managers" + echo "The 'pnpm' tool is required. cd to the repo directory and type 'corepack enable'" + echo "https://github.com/lichess-org/lila/wiki/Lichess-Development-Onboarding" exit 1 fi -pnpm install --silent --ignore-workspace --prefer-frozen-lockfile +pnpm install --silent --ignore-workspace node --experimental-strip-types --no-warnings src/main.ts "$@" From 35031eddaea61a05a310e900df32972309c6c7a5 Mon Sep 17 00:00:00 2001 From: Jonathan Gamble Date: Fri, 6 Dec 2024 12:13:33 -0600 Subject: [PATCH 03/11] move tiny board package to common sources --- ui/analyse/css/_analyse.abstract.scss | 1 + ui/board/package.json | 28 ------------------- ui/board/tsconfig.json | 4 --- .../css/component/_board-menu.scss} | 0 .../src/menu.ts => common/src/boardMenu.ts} | 14 +++++----- 5 files changed, 8 insertions(+), 39 deletions(-) delete mode 100644 ui/board/package.json delete mode 100644 ui/board/tsconfig.json rename ui/{board/css/_menu.scss => common/css/component/_board-menu.scss} (100%) rename ui/{board/src/menu.ts => common/src/boardMenu.ts} (88%) diff --git a/ui/analyse/css/_analyse.abstract.scss b/ui/analyse/css/_analyse.abstract.scss index 652c09589db9d..649245d95a4e3 100644 --- a/ui/analyse/css/_analyse.abstract.scss +++ b/ui/analyse/css/_analyse.abstract.scss @@ -1,6 +1,7 @@ @import '../../common/css/plugin'; @import '../../common/css/theme/board/coords'; @import '../../common/css/layout/uniboard'; +@import '../../common/css/component/board-menu'; @import '../../common/css/component/board-resize'; @import '../../common/css/component/continue-with'; @import '../../common/css/component/color-icon'; diff --git a/ui/board/package.json b/ui/board/package.json deleted file mode 100644 index 5a6645069720a..0000000000000 --- a/ui/board/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "board", - "version": "2.0.0", - "private": true, - "description": "lichess.org chess utils", - "type": "module", - "module": "menu.js", - "types": "menu.d.ts", - "exports": { - "./*": "./dist/*.js" - }, - "typesVersions": { - "*": { - "*": [ - "dist/*" - ] - } - }, - "keywords": [ - "chess", - "lichess" - ], - "author": "Thibault Duplessis", - "license": "AGPL-3.0-or-later", - "dependencies": { - "common": "workspace:*" - } -} diff --git a/ui/board/tsconfig.json b/ui/board/tsconfig.json deleted file mode 100644 index 503e46569fd59..0000000000000 --- a/ui/board/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../tsconfig.base.json", - "references": [{ "path": "../common/tsconfig.json" }] -} diff --git a/ui/board/css/_menu.scss b/ui/common/css/component/_board-menu.scss similarity index 100% rename from ui/board/css/_menu.scss rename to ui/common/css/component/_board-menu.scss diff --git a/ui/board/src/menu.ts b/ui/common/src/boardMenu.ts similarity index 88% rename from ui/board/src/menu.ts rename to ui/common/src/boardMenu.ts index c2e2f715d3e60..b96c4c9fde8ee 100644 --- a/ui/board/src/menu.ts +++ b/ui/common/src/boardMenu.ts @@ -1,10 +1,10 @@ import { h, type VNode } from 'snabbdom'; -import { type Toggle, myUserId, onClickAway } from 'common/common'; -import { bindMobileMousedown } from 'common/device'; -import * as licon from 'common/licon'; -import { type MaybeVNode, type MaybeVNodes, dataIcon, onInsert } from 'common/snabbdom'; -import { type ToggleSettings, toggle } from 'common/controls'; -import { pubsub } from 'common/pubsub'; +import { type Toggle, myUserId, onClickAway } from './common'; +import { bindMobileMousedown } from './device'; +import * as licon from './licon'; +import { type MaybeVNode, type MaybeVNodes, dataIcon, onInsert } from './snabbdom'; +import { type ToggleSettings, toggle } from './controls'; +import { pubsub } from './pubsub'; export const toggleButton = (toggle: Toggle, title: string): VNode => h('button.fbt.board-menu-toggle', { @@ -13,7 +13,7 @@ export const toggleButton = (toggle: Toggle, title: string): VNode => hook: onInsert(bindMobileMousedown(toggle.toggle)), }); -export const menu = ( +export const boardMenu = ( redraw: Redraw, toggle: Toggle, content: (menu: BoardMenu) => MaybeVNodes, From 604fb380ac6319aedca903d3fd94491b1f0f0aa7 Mon Sep 17 00:00:00 2001 From: Jonathan Gamble Date: Fri, 6 Dec 2024 12:15:22 -0600 Subject: [PATCH 04/11] express inline