chore(release): 1.52.0 (#3325)

See [CHANGELOG](https://github.com/aws/jsii/blob/bump/1.52.0/CHANGELOG.md)

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.52.0](https://github.com/aws/jsii/compare/v1.51.0...v1.52.0) (2022-01-07)
+
+
+### Bug Fixes
+
+* **rosetta:** `transliterate` tries to recompile samples from tablets ([#3324](https://github.com/aws/jsii/issues/3324)) ([7aa69a7](https://github.com/aws/jsii/commit/7aa69a7c51b31ab1adb37b9d5cab682b5da6c715)), closes [#3262](https://github.com/aws/jsii/issues/3262)
+
 ## [1.51.0](https://github.com/aws/jsii/compare/v1.50.0...v1.51.0) (2022-01-06)
 
 

gh-pages/requirements-dev.txt

@@ -1,4 +1,4 @@
 mkdocs~=1.2.3
 mkdocs-awesome-pages-plugin~=2.6.0
-mkdocs-material~=8.1.3
+mkdocs-material~=8.1.4
 mkdocs-git-revision-date-plugin~=0.3.1

lerna.json

@@ -10,5 +10,5 @@
       "rejectCycles": true
     }
   },
-  "version": "1.51.0"
+  "version": "1.52.0"
 }

packages/@jsii/python-runtime/setup.py

@@ -32,7 +32,7 @@
     install_requires=[
         "attrs~=21.2",
         "cattrs~=1.0.0 ; python_version < '3.7'",
-        "cattrs>=1.8,<1.10 ; python_version >= '3.7'",
+        "cattrs>=1.8,<1.11 ; python_version >= '3.7'",
         "importlib_resources ; python_version < '3.7'",
         "python-dateutil",
         "typing_extensions>=3.7,<5.0",

packages/jsii-rosetta/bin/jsii-rosetta.ts

@@ -6,6 +6,7 @@ import * as yargs from 'yargs';
 
 import { TranslateResult, translateTypeScript, RosettaDiagnostic } from '../lib';
 import { translateMarkdown } from '../lib/commands/convert';
+import { checkCoverage } from '../lib/commands/coverage';
 import { extractAndInfuse, extractSnippets, ExtractOptions } from '../lib/commands/extract';
 import { infuse, DEFAULT_INFUSION_RESULTS_NAME } from '../lib/commands/infuse';
 import { readTablet } from '../lib/commands/read';
@@ -348,6 +349,21 @@ function main() {
         });
       }),
     )
+    .command(
+      'coverage [ASSEMBLY..]',
+      'Check the translation coverage of implicit tablets for the given assemblies',
+      (command) =>
+        command.positional('ASSEMBLY', {
+          type: 'string',
+          string: true,
+          default: ['.'],
+          describe: 'Assembly or directory to search',
+        }),
+      wrapHandler(async (args) => {
+        const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map((x) => path.resolve(x));
+        await checkCoverage(absAssemblies);
+      }),
+    )
     .command(
       'read <TABLET> [KEY] [LANGUAGE]',
       'Display snippets in a language tablet file',

packages/jsii-rosetta/lib/commands/coverage.ts

@@ -0,0 +1,35 @@
+import { loadAssemblies, allTypeScriptSnippets, loadAllDefaultTablets } from '../jsii/assemblies';
+import * as logging from '../logging';
+import { RosettaTranslator } from '../rosetta-translator';
+import { formatLocation } from '../snippet';
+
+export async function checkCoverage(assemblyLocations: readonly string[]): Promise<void> {
+  logging.info(`Loading ${assemblyLocations.length} assemblies`);
+  const assemblies = await loadAssemblies(assemblyLocations, false);
+
+  const snippets = Array.from(allTypeScriptSnippets(assemblies, true));
+
+  const translator = new RosettaTranslator({
+    assemblies: assemblies.map((a) => a.assembly),
+    allowDirtyTranslations: true,
+  });
+  translator.addTabletsToCache(...Object.values(await loadAllDefaultTablets(assemblies)));
+
+  process.stdout.write(`- ${snippets.length} total snippets.\n`);
+  process.stdout.write(`- ${translator.cache.count} translations in cache.\n`);
+  process.stdout.write('\n');
+
+  const results = translator.readFromCache(snippets, true, true);
+  process.stdout.write(`- ${results.translations.length - results.dirtyCount} successful cache hits.\n`);
+  process.stdout.write(`     ${results.infusedCount} infused.\n`);
+  process.stdout.write(`- ${results.dirtyCount} translations in cache but dirty (ok for pacmak, transliterate)\n`);
+  process.stdout.write(`     ${results.dirtySourceCount} sources have changed.\n`);
+  process.stdout.write(`     ${results.dirtyTranslatorCount} translator has changed.\n`);
+  process.stdout.write(`     ${results.dirtyTypesCount} types have changed.\n`);
+  process.stdout.write(`     ${results.dirtyDidntCompile} did not successfully compile.\n`);
+  process.stdout.write(`- ${results.remaining.length} snippets untranslated.\n`);
+
+  for (const remaining of results.remaining) {
+    process.stdout.write(`     ${formatLocation(remaining.location)}\n`);
+  }
+}

packages/jsii-rosetta/lib/commands/extract.ts

@@ -58,6 +58,13 @@ export interface ExtractOptions {
    * @default false
    */
   readonly loose?: boolean;
+
+  /**
+   * Accept dirty translations from the cache
+   *
+   * @default false
+   */
+  readonly allowDirtyTranslations?: boolean;
 }
 
 export async function extractAndInfuse(assemblyLocations: string[], options: ExtractOptions): Promise<ExtractResult> {
@@ -96,6 +103,7 @@ export async function extractSnippets(
   const translatorOptions: RosettaTranslatorOptions = {
     includeCompilerDiagnostics: options.includeCompilerDiagnostics ?? false,
     assemblies: assemblies.map((a) => a.assembly),
+    allowDirtyTranslations: options.allowDirtyTranslations,
   };
 
   const translator = options.translatorFactory

packages/jsii-rosetta/lib/commands/transliterate.ts

@@ -60,6 +60,7 @@ export async function transliterateAssembly(
     loose: options.loose,
     cacheFromFile: options.tablet,
     writeToImplicitTablets: false,
+    allowDirtyTranslations: true,
   });
 
   // Now do a regular "tablet reader" cycle, expecting everything to be translated already,

packages/jsii-rosetta/lib/rosetta-translator.ts

@@ -33,6 +33,13 @@ export interface RosettaTranslatorOptions {
    * @default false
    */
   readonly includeCompilerDiagnostics?: boolean;
+
+  /**
+   * Allow reading dirty translations from cache
+   *
+   * @default false
+   */
+  readonly allowDirtyTranslations?: boolean;
 }
 
 /**
@@ -49,13 +56,16 @@ export class RosettaTranslator {
    */
   public readonly tablet = new LanguageTablet();
 
+  public readonly cache = new LanguageTablet();
+
   private readonly fingerprinter: TypeFingerprinter;
-  private readonly cache = new LanguageTablet();
   private readonly includeCompilerDiagnostics: boolean;
+  private readonly allowDirtyTranslations: boolean;
 
   public constructor(options: RosettaTranslatorOptions = {}) {
     this.fingerprinter = new TypeFingerprinter(options?.assemblies ?? []);
     this.includeCompilerDiagnostics = options.includeCompilerDiagnostics ?? false;
+    this.allowDirtyTranslations = options.allowDirtyTranslations ?? false;
   }
 
   /**
@@ -90,25 +100,58 @@ export class RosettaTranslator {
    * Will remove the cached snippets from the input array.
    */
   public readFromCache(snippets: TypeScriptSnippet[], addToTablet = true, compiledOnly = false): ReadFromCacheResults {
-    const remaining = [...snippets];
     const translations = new Array<TranslatedSnippet>();
+    const remaining = new Array<TypeScriptSnippet>();
+
+    let infusedCount = 0;
+    let dirtyCount = 0;
+    let dirtySourceCount = 0;
+    let dirtyTypesCount = 0;
+    let dirtyTranslatorCount = 0;
+    let dirtyDidntCompile = 0;
 
-    let i = 0;
-    while (i < remaining.length) {
-      const fromCache = tryReadFromCache(remaining[i], this.cache, this.fingerprinter);
-      // If compiledOnly is set, do not consider cached snippets that do not compile
-      if (fromCache && (!compiledOnly || fromCache.snippet.didCompile)) {
-        if (addToTablet) {
-          this.tablet.addSnippet(fromCache);
-        }
-        remaining.splice(i, 1);
-        translations.push(fromCache);
-      } else {
-        i += 1;
+    for (const snippet of snippets) {
+      const fromCache = tryReadFromCache(snippet, this.cache, this.fingerprinter, compiledOnly);
+      switch (fromCache.type) {
+        case 'hit':
+          if (addToTablet) {
+            this.tablet.addSnippet(fromCache.snippet);
+          }
+          translations.push(fromCache.snippet);
+
+          infusedCount += fromCache.infused ? 1 : 0;
+          break;
+
+        case 'dirty':
+          dirtyCount += 1;
+          dirtySourceCount += fromCache.dirtySource ? 1 : 0;
+          dirtyTranslatorCount += fromCache.dirtyTranslator ? 1 : 0;
+          dirtyTypesCount += fromCache.dirtyTypes ? 1 : 0;
+          dirtyDidntCompile += fromCache.dirtyDidntCompile ? 1 : 0;
+
+          if (this.allowDirtyTranslations) {
+            translations.push(fromCache.translation);
+          } else {
+            remaining.push(snippet);
+          }
+          break;
+
+        case 'miss':
+          remaining.push(snippet);
+          break;
       }
     }
 
-    return { translations, remaining };
+    return {
+      translations,
+      remaining,
+      infusedCount,
+      dirtyCount,
+      dirtySourceCount,
+      dirtyTranslatorCount,
+      dirtyTypesCount,
+      dirtyDidntCompile,
+    };
   }
 
   public async translateAll(snippets: TypeScriptSnippet[], addToTablet = true): Promise<TranslateAllResult> {
@@ -146,32 +189,75 @@ export class RosettaTranslator {
  * doesn't really make a lot of difference.  So, for simplification's sake,
  * we'll regen all translations if there's at least one that's outdated.
  */
-function tryReadFromCache(sourceSnippet: TypeScriptSnippet, cache: LanguageTablet, fingerprinter: TypeFingerprinter) {
+function tryReadFromCache(
+  sourceSnippet: TypeScriptSnippet,
+  cache: LanguageTablet,
+  fingerprinter: TypeFingerprinter,
+  compiledOnly: boolean,
+): CacheHit {
   const fromCache = cache.tryGetSnippet(snippetKey(sourceSnippet));
 
+  if (!fromCache) {
+    return { type: 'miss' };
+  }
+
   // infused snippets won't pass the full source check or the fingerprinter
   // but there is no reason to try to recompile it, so return cached snippet
   // if there exists one.
   if (isInfused(sourceSnippet)) {
-    return fromCache;
+    return { type: 'hit', snippet: fromCache, infused: true };
   }
 
-  const cacheable =
-    fromCache &&
-    completeSource(sourceSnippet) === fromCache.snippet.fullSource &&
-    Object.entries(TARGET_LANGUAGES).every(
-      ([lang, translator]) => fromCache.snippet.translations?.[lang]?.version === translator.version,
-    ) &&
-    fingerprinter.fingerprintAll(fromCache.fqnsReferenced()) === fromCache.snippet.fqnsFingerprint;
+  const dirtySource = completeSource(sourceSnippet) !== fromCache.snippet.fullSource;
+  const dirtyTranslator = !Object.entries(TARGET_LANGUAGES).every(
+    ([lang, translator]) => fromCache.snippet.translations?.[lang]?.version === translator.version,
+  );
+  const dirtyTypes = fingerprinter.fingerprintAll(fromCache.fqnsReferenced()) !== fromCache.snippet.fqnsFingerprint;
+  const dirtyDidntCompile = compiledOnly && !fromCache.snippet.didCompile;
 
-  return cacheable ? fromCache : undefined;
+  if (dirtySource || dirtyTranslator || dirtyTypes || dirtyDidntCompile) {
+    return { type: 'dirty', translation: fromCache, dirtySource, dirtyTranslator, dirtyTypes, dirtyDidntCompile };
+  }
+  return { type: 'hit', snippet: fromCache, infused: false };
 }
 
+export type CacheHit =
+  | { readonly type: 'miss' }
+  | { readonly type: 'hit'; readonly snippet: TranslatedSnippet; readonly infused: boolean }
+  | {
+      readonly type: 'dirty';
+      readonly translation: TranslatedSnippet;
+      readonly dirtySource: boolean;
+      readonly dirtyTranslator: boolean;
+      readonly dirtyTypes: boolean;
+      readonly dirtyDidntCompile: boolean;
+    };
+
 function isInfused(snippet: TypeScriptSnippet) {
   return snippet.parameters?.infused !== undefined;
 }
 
 export interface ReadFromCacheResults {
+  /**
+   * Successful translations
+   */
   readonly translations: TranslatedSnippet[];
+
+  /**
+   * Successful but dirty hits
+   */
   readonly remaining: TypeScriptSnippet[];
+
+  /**
+   * How many successfully hit translations were infused
+   */
+  readonly infusedCount: number;
+
+  readonly dirtyCount: number;
+
+  // Counts for dirtiness (a single snippet may be dirty for more than one reason)
+  readonly dirtySourceCount: number;
+  readonly dirtyTranslatorCount: number;
+  readonly dirtyTypesCount: number;
+  readonly dirtyDidntCompile: number;
 }

packages/jsii-rosetta/test/commands/transliterate.test.ts

@@ -1,11 +1,13 @@
-import { SPEC_FILE_NAME } from '@jsii/spec';
+import { Assembly, SPEC_FILE_NAME } from '@jsii/spec';
 import * as fs from 'fs-extra';
 import * as jsii from 'jsii';
 import * as path from 'path';
 
+import { extractSnippets } from '../../lib/commands/extract';
 import { transliterateAssembly } from '../../lib/commands/transliterate';
 import { TargetLanguage } from '../../lib/languages/target-language';
-import { withTemporaryDirectory } from '../testutil';
+import { TabletSchema } from '../../lib/tablets/schema';
+import { withTemporaryDirectory, TestJsiiModule, DUMMY_JSII_CONFIG } from '../testutil';
 
 jest.setTimeout(60_000);
 
@@ -1345,3 +1347,47 @@ export class ClassName implements IInterface {
       }),
     ).resolves.not.toThrow();
   }));
+
+test('will read translations from cache even if they are dirty', async () => {
+  const infusedAssembly = await TestJsiiModule.fromSource(
+    {
+      'index.ts': `
+        /**
+         * ClassA
+         *
+         * @example x
+         */
+        export class ClassA {
+          public someMethod() {
+          }
+        }
+        `,
+    },
+    {
+      name: 'my_assembly',
+      jsii: DUMMY_JSII_CONFIG,
+    },
+  );
+  try {
+    // Run an extract
+    await extractSnippets([infusedAssembly.moduleDirectory]);
+
+    // Mess up the extracted source file
+    const schema: TabletSchema = await fs.readJson(path.join(infusedAssembly.moduleDirectory, '.jsii.tabl.json'));
+    for (const snippet of Object.values(schema.snippets)) {
+      snippet.translations[TargetLanguage.PYTHON] = {
+        source: 'oops',
+        version: '999',
+      };
+    }
+    await fs.writeJson(path.join(infusedAssembly.moduleDirectory, '.jsii.tabl.json'), schema);
+
+    // Run a transliterate, should have used the translation from the cache even though the version is wrong
+    await transliterateAssembly([infusedAssembly.moduleDirectory], [TargetLanguage.PYTHON]);
+
+    const translated: Assembly = await fs.readJson(path.join(infusedAssembly.moduleDirectory, '.jsii.python'));
+    expect(translated.types?.['my_assembly.ClassA'].docs?.example).toEqual('oops');
+  } finally {
+    await infusedAssembly.cleanup();
+  }
+});