From 17233dfda821c4f18895b3d324a75301a2f57c16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 12:42:47 +0000 Subject: [PATCH 1/4] chore(deps-dev): update mkdocs-material requirement from ~=8.1.3 to ~=8.1.4 in /gh-pages (#3313) Updates the requirements on [mkdocs-material](https://github.com/squidfunk/mkdocs-material) to permit the latest version.
Release notes

Sourced from mkdocs-material's releases.

mkdocs-material-8.1.4

Changelog

Sourced from mkdocs-material's changelog.

mkdocs-material-8.1.4+insiders-4.5.1 (2022-01-02)

mkdocs-material-8.1.4 (2022-01-02)

mkdocs-material-8.1.3 (2021-12-19)

mkdocs-material-8.1.2+insiders-4.5.0 (2021-12-16)

mkdocs-material-8.1.2 (2021-12-15)

mkdocs-material-8.1.1 (2021-12-13)

mkdocs-material-8.1.0+insiders-4.4.0 (2021-12-10)

mkdocs-material-8.1.0 (2021-12-10)

... (truncated)

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- gh-pages/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh-pages/requirements-dev.txt b/gh-pages/requirements-dev.txt index 48f1257071..c82f2e4ced 100644 --- a/gh-pages/requirements-dev.txt +++ b/gh-pages/requirements-dev.txt @@ -1,4 +1,4 @@ mkdocs~=1.2.3 mkdocs-awesome-pages-plugin~=2.6.0 -mkdocs-material~=8.1.3 +mkdocs-material~=8.1.4 mkdocs-git-revision-date-plugin~=0.3.1 From e37272d9118fb1cc6e3e83a0cd8907e3fef88150 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 13:51:44 +0000 Subject: [PATCH 2/4] chore(deps): update cattrs requirement from <1.10,>=1.8 to >=1.8,<1.11 in /packages/@jsii/python-runtime (#3315) Updates the requirements on [cattrs](https://github.com/python-attrs/cattrs) to permit the latest version.
Changelog

Sourced from cattrs's changelog.

1.10.0 (2022-01-04)

1.9.0 (2021-12-06)

1.8.0 (2021-08-13)

1.7.1 (2021-05-28)

1.7.0 (2021-05-26)

... (truncated)

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- packages/@jsii/python-runtime/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@jsii/python-runtime/setup.py b/packages/@jsii/python-runtime/setup.py index ac5a36e3e8..440c5e9305 100644 --- a/packages/@jsii/python-runtime/setup.py +++ b/packages/@jsii/python-runtime/setup.py @@ -32,7 +32,7 @@ install_requires=[ "attrs~=21.2", "cattrs~=1.0.0 ; python_version < '3.7'", - "cattrs>=1.8,<1.10 ; python_version >= '3.7'", + "cattrs>=1.8,<1.11 ; python_version >= '3.7'", "importlib_resources ; python_version < '3.7'", "python-dateutil", "typing_extensions>=3.7,<5.0", From 7aa69a7c51b31ab1adb37b9d5cab682b5da6c715 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 7 Jan 2022 12:29:14 +0100 Subject: [PATCH 3/4] fix(rosetta): `transliterate` tries to recompile samples from tablets (#3324) In #3262, the performance of `transliterate` was improved by running an `extract` before translating, because `extract` has the facilities to do parallel translate, and can read from a cache. However, another thing `extract` does is *discard translations from the cache* if they are considered "dirty" for some reason. This reason can be: the current source is different than the cached source, the translation mechanism has changed, compilation didn't succeed last time, or the types referenced in the example have changed. This is actually not desirable for `transliterate`, which wants to do a last-ditch effort to translate whatever is necessary to translate, but should really take what it can from the cache if there's a cached translation available. Add a flag to allow reading dirty translations from the cache. Also adds the `rosetta coverage` command, which I used to find out why the Construct Hub performance on the newest `aws-cdk-lib` version is still poor. --- By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license]. [Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0 --- packages/jsii-rosetta/bin/jsii-rosetta.ts | 16 +++ .../jsii-rosetta/lib/commands/coverage.ts | 35 +++++ packages/jsii-rosetta/lib/commands/extract.ts | 8 ++ .../lib/commands/transliterate.ts | 1 + .../jsii-rosetta/lib/rosetta-translator.ts | 136 ++++++++++++++---- .../test/commands/transliterate.test.ts | 50 ++++++- 6 files changed, 219 insertions(+), 27 deletions(-) create mode 100644 packages/jsii-rosetta/lib/commands/coverage.ts diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts index 8f05aed94c..39df072af2 100644 --- a/packages/jsii-rosetta/bin/jsii-rosetta.ts +++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts @@ -6,6 +6,7 @@ import * as yargs from 'yargs'; import { TranslateResult, translateTypeScript, RosettaDiagnostic } from '../lib'; import { translateMarkdown } from '../lib/commands/convert'; +import { checkCoverage } from '../lib/commands/coverage'; import { extractAndInfuse, extractSnippets, ExtractOptions } from '../lib/commands/extract'; import { infuse, DEFAULT_INFUSION_RESULTS_NAME } from '../lib/commands/infuse'; import { readTablet } from '../lib/commands/read'; @@ -348,6 +349,21 @@ function main() { }); }), ) + .command( + 'coverage [ASSEMBLY..]', + 'Check the translation coverage of implicit tablets for the given assemblies', + (command) => + command.positional('ASSEMBLY', { + type: 'string', + string: true, + default: ['.'], + describe: 'Assembly or directory to search', + }), + wrapHandler(async (args) => { + const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map((x) => path.resolve(x)); + await checkCoverage(absAssemblies); + }), + ) .command( 'read [KEY] [LANGUAGE]', 'Display snippets in a language tablet file', diff --git a/packages/jsii-rosetta/lib/commands/coverage.ts b/packages/jsii-rosetta/lib/commands/coverage.ts new file mode 100644 index 0000000000..d7fb57b0d9 --- /dev/null +++ b/packages/jsii-rosetta/lib/commands/coverage.ts @@ -0,0 +1,35 @@ +import { loadAssemblies, allTypeScriptSnippets, loadAllDefaultTablets } from '../jsii/assemblies'; +import * as logging from '../logging'; +import { RosettaTranslator } from '../rosetta-translator'; +import { formatLocation } from '../snippet'; + +export async function checkCoverage(assemblyLocations: readonly string[]): Promise { + logging.info(`Loading ${assemblyLocations.length} assemblies`); + const assemblies = await loadAssemblies(assemblyLocations, false); + + const snippets = Array.from(allTypeScriptSnippets(assemblies, true)); + + const translator = new RosettaTranslator({ + assemblies: assemblies.map((a) => a.assembly), + allowDirtyTranslations: true, + }); + translator.addTabletsToCache(...Object.values(await loadAllDefaultTablets(assemblies))); + + process.stdout.write(`- ${snippets.length} total snippets.\n`); + process.stdout.write(`- ${translator.cache.count} translations in cache.\n`); + process.stdout.write('\n'); + + const results = translator.readFromCache(snippets, true, true); + process.stdout.write(`- ${results.translations.length - results.dirtyCount} successful cache hits.\n`); + process.stdout.write(` ${results.infusedCount} infused.\n`); + process.stdout.write(`- ${results.dirtyCount} translations in cache but dirty (ok for pacmak, transliterate)\n`); + process.stdout.write(` ${results.dirtySourceCount} sources have changed.\n`); + process.stdout.write(` ${results.dirtyTranslatorCount} translator has changed.\n`); + process.stdout.write(` ${results.dirtyTypesCount} types have changed.\n`); + process.stdout.write(` ${results.dirtyDidntCompile} did not successfully compile.\n`); + process.stdout.write(`- ${results.remaining.length} snippets untranslated.\n`); + + for (const remaining of results.remaining) { + process.stdout.write(` ${formatLocation(remaining.location)}\n`); + } +} diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts index d878e83a85..2d0105af59 100644 --- a/packages/jsii-rosetta/lib/commands/extract.ts +++ b/packages/jsii-rosetta/lib/commands/extract.ts @@ -58,6 +58,13 @@ export interface ExtractOptions { * @default false */ readonly loose?: boolean; + + /** + * Accept dirty translations from the cache + * + * @default false + */ + readonly allowDirtyTranslations?: boolean; } export async function extractAndInfuse(assemblyLocations: string[], options: ExtractOptions): Promise { @@ -96,6 +103,7 @@ export async function extractSnippets( const translatorOptions: RosettaTranslatorOptions = { includeCompilerDiagnostics: options.includeCompilerDiagnostics ?? false, assemblies: assemblies.map((a) => a.assembly), + allowDirtyTranslations: options.allowDirtyTranslations, }; const translator = options.translatorFactory diff --git a/packages/jsii-rosetta/lib/commands/transliterate.ts b/packages/jsii-rosetta/lib/commands/transliterate.ts index 86c2f334af..ea6904a683 100644 --- a/packages/jsii-rosetta/lib/commands/transliterate.ts +++ b/packages/jsii-rosetta/lib/commands/transliterate.ts @@ -60,6 +60,7 @@ export async function transliterateAssembly( loose: options.loose, cacheFromFile: options.tablet, writeToImplicitTablets: false, + allowDirtyTranslations: true, }); // Now do a regular "tablet reader" cycle, expecting everything to be translated already, diff --git a/packages/jsii-rosetta/lib/rosetta-translator.ts b/packages/jsii-rosetta/lib/rosetta-translator.ts index 1a00e463c6..047db93de0 100644 --- a/packages/jsii-rosetta/lib/rosetta-translator.ts +++ b/packages/jsii-rosetta/lib/rosetta-translator.ts @@ -33,6 +33,13 @@ export interface RosettaTranslatorOptions { * @default false */ readonly includeCompilerDiagnostics?: boolean; + + /** + * Allow reading dirty translations from cache + * + * @default false + */ + readonly allowDirtyTranslations?: boolean; } /** @@ -49,13 +56,16 @@ export class RosettaTranslator { */ public readonly tablet = new LanguageTablet(); + public readonly cache = new LanguageTablet(); + private readonly fingerprinter: TypeFingerprinter; - private readonly cache = new LanguageTablet(); private readonly includeCompilerDiagnostics: boolean; + private readonly allowDirtyTranslations: boolean; public constructor(options: RosettaTranslatorOptions = {}) { this.fingerprinter = new TypeFingerprinter(options?.assemblies ?? []); this.includeCompilerDiagnostics = options.includeCompilerDiagnostics ?? false; + this.allowDirtyTranslations = options.allowDirtyTranslations ?? false; } /** @@ -90,25 +100,58 @@ export class RosettaTranslator { * Will remove the cached snippets from the input array. */ public readFromCache(snippets: TypeScriptSnippet[], addToTablet = true, compiledOnly = false): ReadFromCacheResults { - const remaining = [...snippets]; const translations = new Array(); + const remaining = new Array(); + + let infusedCount = 0; + let dirtyCount = 0; + let dirtySourceCount = 0; + let dirtyTypesCount = 0; + let dirtyTranslatorCount = 0; + let dirtyDidntCompile = 0; - let i = 0; - while (i < remaining.length) { - const fromCache = tryReadFromCache(remaining[i], this.cache, this.fingerprinter); - // If compiledOnly is set, do not consider cached snippets that do not compile - if (fromCache && (!compiledOnly || fromCache.snippet.didCompile)) { - if (addToTablet) { - this.tablet.addSnippet(fromCache); - } - remaining.splice(i, 1); - translations.push(fromCache); - } else { - i += 1; + for (const snippet of snippets) { + const fromCache = tryReadFromCache(snippet, this.cache, this.fingerprinter, compiledOnly); + switch (fromCache.type) { + case 'hit': + if (addToTablet) { + this.tablet.addSnippet(fromCache.snippet); + } + translations.push(fromCache.snippet); + + infusedCount += fromCache.infused ? 1 : 0; + break; + + case 'dirty': + dirtyCount += 1; + dirtySourceCount += fromCache.dirtySource ? 1 : 0; + dirtyTranslatorCount += fromCache.dirtyTranslator ? 1 : 0; + dirtyTypesCount += fromCache.dirtyTypes ? 1 : 0; + dirtyDidntCompile += fromCache.dirtyDidntCompile ? 1 : 0; + + if (this.allowDirtyTranslations) { + translations.push(fromCache.translation); + } else { + remaining.push(snippet); + } + break; + + case 'miss': + remaining.push(snippet); + break; } } - return { translations, remaining }; + return { + translations, + remaining, + infusedCount, + dirtyCount, + dirtySourceCount, + dirtyTranslatorCount, + dirtyTypesCount, + dirtyDidntCompile, + }; } public async translateAll(snippets: TypeScriptSnippet[], addToTablet = true): Promise { @@ -146,32 +189,75 @@ export class RosettaTranslator { * doesn't really make a lot of difference. So, for simplification's sake, * we'll regen all translations if there's at least one that's outdated. */ -function tryReadFromCache(sourceSnippet: TypeScriptSnippet, cache: LanguageTablet, fingerprinter: TypeFingerprinter) { +function tryReadFromCache( + sourceSnippet: TypeScriptSnippet, + cache: LanguageTablet, + fingerprinter: TypeFingerprinter, + compiledOnly: boolean, +): CacheHit { const fromCache = cache.tryGetSnippet(snippetKey(sourceSnippet)); + if (!fromCache) { + return { type: 'miss' }; + } + // infused snippets won't pass the full source check or the fingerprinter // but there is no reason to try to recompile it, so return cached snippet // if there exists one. if (isInfused(sourceSnippet)) { - return fromCache; + return { type: 'hit', snippet: fromCache, infused: true }; } - const cacheable = - fromCache && - completeSource(sourceSnippet) === fromCache.snippet.fullSource && - Object.entries(TARGET_LANGUAGES).every( - ([lang, translator]) => fromCache.snippet.translations?.[lang]?.version === translator.version, - ) && - fingerprinter.fingerprintAll(fromCache.fqnsReferenced()) === fromCache.snippet.fqnsFingerprint; + const dirtySource = completeSource(sourceSnippet) !== fromCache.snippet.fullSource; + const dirtyTranslator = !Object.entries(TARGET_LANGUAGES).every( + ([lang, translator]) => fromCache.snippet.translations?.[lang]?.version === translator.version, + ); + const dirtyTypes = fingerprinter.fingerprintAll(fromCache.fqnsReferenced()) !== fromCache.snippet.fqnsFingerprint; + const dirtyDidntCompile = compiledOnly && !fromCache.snippet.didCompile; - return cacheable ? fromCache : undefined; + if (dirtySource || dirtyTranslator || dirtyTypes || dirtyDidntCompile) { + return { type: 'dirty', translation: fromCache, dirtySource, dirtyTranslator, dirtyTypes, dirtyDidntCompile }; + } + return { type: 'hit', snippet: fromCache, infused: false }; } +export type CacheHit = + | { readonly type: 'miss' } + | { readonly type: 'hit'; readonly snippet: TranslatedSnippet; readonly infused: boolean } + | { + readonly type: 'dirty'; + readonly translation: TranslatedSnippet; + readonly dirtySource: boolean; + readonly dirtyTranslator: boolean; + readonly dirtyTypes: boolean; + readonly dirtyDidntCompile: boolean; + }; + function isInfused(snippet: TypeScriptSnippet) { return snippet.parameters?.infused !== undefined; } export interface ReadFromCacheResults { + /** + * Successful translations + */ readonly translations: TranslatedSnippet[]; + + /** + * Successful but dirty hits + */ readonly remaining: TypeScriptSnippet[]; + + /** + * How many successfully hit translations were infused + */ + readonly infusedCount: number; + + readonly dirtyCount: number; + + // Counts for dirtiness (a single snippet may be dirty for more than one reason) + readonly dirtySourceCount: number; + readonly dirtyTranslatorCount: number; + readonly dirtyTypesCount: number; + readonly dirtyDidntCompile: number; } diff --git a/packages/jsii-rosetta/test/commands/transliterate.test.ts b/packages/jsii-rosetta/test/commands/transliterate.test.ts index 9a8f36e663..c5693523f5 100644 --- a/packages/jsii-rosetta/test/commands/transliterate.test.ts +++ b/packages/jsii-rosetta/test/commands/transliterate.test.ts @@ -1,11 +1,13 @@ -import { SPEC_FILE_NAME } from '@jsii/spec'; +import { Assembly, SPEC_FILE_NAME } from '@jsii/spec'; import * as fs from 'fs-extra'; import * as jsii from 'jsii'; import * as path from 'path'; +import { extractSnippets } from '../../lib/commands/extract'; import { transliterateAssembly } from '../../lib/commands/transliterate'; import { TargetLanguage } from '../../lib/languages/target-language'; -import { withTemporaryDirectory } from '../testutil'; +import { TabletSchema } from '../../lib/tablets/schema'; +import { withTemporaryDirectory, TestJsiiModule, DUMMY_JSII_CONFIG } from '../testutil'; jest.setTimeout(60_000); @@ -1345,3 +1347,47 @@ export class ClassName implements IInterface { }), ).resolves.not.toThrow(); })); + +test('will read translations from cache even if they are dirty', async () => { + const infusedAssembly = await TestJsiiModule.fromSource( + { + 'index.ts': ` + /** + * ClassA + * + * @example x + */ + export class ClassA { + public someMethod() { + } + } + `, + }, + { + name: 'my_assembly', + jsii: DUMMY_JSII_CONFIG, + }, + ); + try { + // Run an extract + await extractSnippets([infusedAssembly.moduleDirectory]); + + // Mess up the extracted source file + const schema: TabletSchema = await fs.readJson(path.join(infusedAssembly.moduleDirectory, '.jsii.tabl.json')); + for (const snippet of Object.values(schema.snippets)) { + snippet.translations[TargetLanguage.PYTHON] = { + source: 'oops', + version: '999', + }; + } + await fs.writeJson(path.join(infusedAssembly.moduleDirectory, '.jsii.tabl.json'), schema); + + // Run a transliterate, should have used the translation from the cache even though the version is wrong + await transliterateAssembly([infusedAssembly.moduleDirectory], [TargetLanguage.PYTHON]); + + const translated: Assembly = await fs.readJson(path.join(infusedAssembly.moduleDirectory, '.jsii.python')); + expect(translated.types?.['my_assembly.ClassA'].docs?.example).toEqual('oops'); + } finally { + await infusedAssembly.cleanup(); + } +}); From c887b8281cd81975743815ce401bd3304c2c8e93 Mon Sep 17 00:00:00 2001 From: AWS CDK Team Date: Fri, 7 Jan 2022 12:46:12 +0000 Subject: [PATCH 4/4] chore(release): 1.52.0 --- CHANGELOG.md | 7 +++++++ lerna.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9928613b39..ebae26171e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.52.0](https://github.com/aws/jsii/compare/v1.51.0...v1.52.0) (2022-01-07) + + +### Bug Fixes + +* **rosetta:** `transliterate` tries to recompile samples from tablets ([#3324](https://github.com/aws/jsii/issues/3324)) ([7aa69a7](https://github.com/aws/jsii/commit/7aa69a7c51b31ab1adb37b9d5cab682b5da6c715)), closes [#3262](https://github.com/aws/jsii/issues/3262) + ## [1.51.0](https://github.com/aws/jsii/compare/v1.50.0...v1.51.0) (2022-01-06) diff --git a/lerna.json b/lerna.json index 599048707e..3c691e9061 100644 --- a/lerna.json +++ b/lerna.json @@ -10,5 +10,5 @@ "rejectCycles": true } }, - "version": "1.51.0" + "version": "1.52.0" }