Merge pull request #159 from thomasmattheussen/feature/parser-italics

add a parser function to add fontStyle to typography tokens

.changeset/tiny-poets-doubt.md

@@ -0,0 +1,5 @@
+---
+'@tokens-studio/sd-transforms': minor
+---
+
+BREAKING: add parser that extracts fontStyle from fontWeight and adds it as a separate property on Typography tokens object values.

.changeset/wild-actors-roll.md

@@ -0,0 +1,5 @@
+---
+'@tokens-studio/sd-transforms': patch
+---
+
+Properly take into account fontStyle inside fontWeights values, in both the fontWeights and CSS typography shorthand transforms.

src/checkAndEvaluateMath.ts

@@ -1,5 +1,6 @@
 import { Parser } from 'expr-eval';
 import { parse, reduceExpression } from 'postcss-calc-ast-parser';
+import { isAlreadyQuoted } from './css/transformTypography.js';
 
 const mathChars = ['+', '-', '*', '/'];
 
@@ -86,11 +87,19 @@ function parseAndReduce(expr: string): string {
   let evaluated;
   try {
     evaluated = parser.evaluate(unitlessExpr);
+
+    // In CSS shorthand with white-spaced font-family that has single quotes around it
+    // parser ends up evaluating that and removing the single quotes. If that happens, revert that change.
+    if (evaluated && isAlreadyQuoted(unitlessExpr)) {
+      evaluated = `'${evaluated}'`;
+    }
   } catch (ex) {
     return expr;
   }
   // Put back the px unit if needed and if reduced doesn't come with one
-  return `${Number.parseFloat(evaluated.toFixed(3))}${unit ?? (hasPx ? 'px' : '')}`;
+  return `${typeof evaluated !== 'string' ? Number.parseFloat(evaluated.toFixed(3)) : evaluated}${
+    unit ?? (hasPx ? 'px' : '')
+  }`;
 }
 
 export function checkAndEvaluateMath(expr: string | number | undefined): string | undefined {

src/css/transformTypography.ts

@@ -8,7 +8,7 @@ export function hasWhiteSpace(value: string): boolean {
   return whiteSpaceRegex.test(value);
 }
 
-function isAlreadyQuoted(value: string): boolean {
+export function isAlreadyQuoted(value: string): boolean {
   return value.startsWith("'") && value.endsWith("'");
 }
 
@@ -48,12 +48,15 @@ export function transformTypographyForCSS(
   }
 
   let { fontFamily, fontWeight, fontSize, lineHeight } = value;
+  const { fontStyle } = value;
   fontSize = transformDimension(checkAndEvaluateMath(fontSize));
   lineHeight = checkAndEvaluateMath(lineHeight);
   fontWeight = transformFontWeights(fontWeight);
   fontFamily = processFontFamily(fontFamily as string | undefined);
 
-  return `${isNothing(fontWeight) ? 400 : fontWeight} ${isNothing(fontSize) ? '16px' : fontSize}/${
+  return `${isNothing(fontWeight) ? 400 : fontWeight}${
+    isNothing(fontStyle) ? '' : ` ${fontStyle}`
+  } ${isNothing(fontSize) ? '16px' : fontSize}/${
     isNothing(lineHeight) ? 1 : lineHeight
   } ${fontFamily}`;
 }

src/parsers/add-font-styles.ts

@@ -0,0 +1,58 @@
+import { DeepKeyTokenMap, SingleToken, TokenTypographyValue } from '@tokens-studio/types';
+// @ts-expect-error no type exported for this function
+import getReferences from 'style-dictionary/lib/utils/references/getReferences.js';
+// @ts-expect-error no type exported for this function
+import usesReference from 'style-dictionary/lib/utils/references/usesReference.js';
+import { fontWeightReg } from '../transformFontWeights.js';
+
+function recurse(
+  slice: DeepKeyTokenMap<false>,
+  boundGetRef: (ref: string) => Array<SingleToken<false>>,
+) {
+  for (const key in slice) {
+    const token = slice[key];
+    const { type, value } = token;
+    if (type === 'typography') {
+      if (typeof value !== 'object') {
+        continue;
+      }
+      let fontWeight = value.fontWeight;
+
+      if (usesReference(fontWeight)) {
+        let ref = { value: fontWeight } as SingleToken<false>;
+        while (ref && ref.value && typeof ref.value === 'string' && usesReference(ref.value)) {
+          try {
+            ref = Object.fromEntries(
+              Object.entries(boundGetRef(ref.value)[0]).map(([k, v]) => [k, v]),
+            ) as SingleToken<false>;
+          } catch (e) {
+            console.warn(`Warning: could not resolve reference ${ref.value}`);
+            return;
+          }
+        }
+        fontWeight = ref.value as string;
+      }
+
+      // cast it to TokenTypographyValue now that we've resolved references all the way, we know it cannot be a string anymore.
+      const tokenValue = value as TokenTypographyValue;
+
+      if (fontWeight) {
+        const fontStyleMatch = fontWeight.match(fontWeightReg);
+        if (fontStyleMatch?.groups?.weight && fontStyleMatch.groups.style) {
+          // @ts-expect-error fontStyle is not a property that exists on Typography Tokens, we just add it ourselves
+          tokenValue.fontStyle = fontStyleMatch.groups.style.toLowerCase();
+          tokenValue.fontWeight = fontStyleMatch?.groups?.weight;
+        }
+      }
+    } else if (typeof token === 'object') {
+      recurse(token as unknown as DeepKeyTokenMap<false>, boundGetRef);
+    }
+  }
+}
+
+export function addFontStyles(dictionary: DeepKeyTokenMap<false>): DeepKeyTokenMap<false> {
+  const copy = { ...dictionary };
+  const boundGetRef = getReferences.bind({ properties: copy });
+  recurse(copy, boundGetRef);
+  return copy;
+}

src/parsers/expand-composites.ts

@@ -29,6 +29,7 @@ const typeMaps = {
     fontWeight: 'fontWeights',
     lineHeight: 'lineHeights',
     fontSize: 'fontSizes',
+    fontStyle: 'fontStyles',
   },
 };
 

src/registerTransforms.ts

@@ -15,6 +15,7 @@ import { TransformOptions } from './TransformOptions.js';
 import { expandComposites } from './parsers/expand-composites.js';
 import { excludeParentKeys } from './parsers/exclude-parent-keys.js';
 import { transformOpacity } from './transformOpacity.js';
+import { addFontStyles } from './parsers/add-font-styles.js';
 
 const isBrowser = typeof window === 'object';
 
@@ -59,7 +60,8 @@ export async function registerTransforms(sd: Core, transformOpts?: TransformOpti
       parse: ({ filePath, contents }) => {
         const obj = JSON.parse(contents);
         const excluded = excludeParentKeys(obj, transformOpts);
-        const expanded = expandComposites(excluded, filePath, transformOpts);
+        const withFontStyles = addFontStyles(excluded);
+        const expanded = expandComposites(withFontStyles, filePath, transformOpts);
         return expanded;
       },
     });

src/transformFontWeights.ts

@@ -25,6 +25,8 @@ export const fontWeightMap = {
   extrafett: 900,
 };
 
+export const fontWeightReg = /(?<weight>.+?)\s?(?<style>italic|oblique|normal)?$/i;
+
 /**
  * Helper: Transforms fontweight keynames to fontweight numbers (100, 200, 300 ... 900)
  */
@@ -34,7 +36,15 @@ export function transformFontWeights(
   if (value === undefined) {
     return value;
   }
-  const mapped = fontWeightMap[`${value}`.toLowerCase()];
+  const match = `${value}`.match(fontWeightReg);
+
+  let mapped;
+  if (match?.groups?.weight) {
+    mapped = fontWeightMap[match?.groups?.weight.toLowerCase()];
+    if (match.groups.style) {
+      mapped = `${mapped} ${match.groups.style.toLowerCase()}`;
+    }
+  }
 
   return mapped ?? value;
 }

test/integration/expand-composition.test.ts

@@ -51,6 +51,7 @@ describe('expand composition tokens', () => {
 
   it('only expands composition tokens by default', async () => {
     const file = await promises.readFile(outputFilePath, 'utf-8');
+
     expect(file).to.include(
       `
   --sdCompositionSize: 24px;
@@ -61,11 +62,12 @@ describe('expand composition tokens', () => {
   --sdCompositionHeaderFontFamilies: Roboto;
   --sdCompositionHeaderFontSizes: 96px;
   --sdCompositionHeaderFontWeights: 700;
-  --sdTypography: 500 26px/1.25 Arial;
+  --sdTypography: 800 italic 26px/1.25 Arial;
+  --sdFontWeightRef: 800 italic;
   --sdBorder: 4px solid #FFFF00;
   --sdShadowSingle: inset 0 4px 10px 0 rgba(0,0,0,0.4);
   --sdShadowDouble: inset 0 4px 10px 0 rgba(0,0,0,0.4), 0 8px 12px 5px rgba(0,0,0,0.4);
-  --sdRef: 500 26px/1.25 Arial;`,
+  --sdRef: 800 italic 26px/1.25 Arial;`,
     );
   });
 
@@ -91,14 +93,16 @@ describe('expand composition tokens', () => {
   --sdCompositionHeaderFontSizes: 96px;
   --sdCompositionHeaderFontWeights: 700;
   --sdTypographyFontFamily: Arial;
-  --sdTypographyFontWeight: 500;
+  --sdTypographyFontWeight: 800;
   --sdTypographyLineHeight: 1.25;
   --sdTypographyFontSize: 26px;
   --sdTypographyLetterSpacing: 0;
   --sdTypographyParagraphSpacing: 0;
   --sdTypographyParagraphIndent: 0;
   --sdTypographyTextDecoration: none;
   --sdTypographyTextCase: none;
+  --sdTypographyFontStyle: italic;
+  --sdFontWeightRef: 800 italic;
   --sdBorderColor: #FFFF00;
   --sdBorderWidth: 4px;
   --sdBorderStyle: solid;
@@ -136,23 +140,25 @@ describe('expand composition tokens', () => {
     expect(file).to.include(
       `
   --sdRefFontFamily: Arial;
-  --sdRefFontWeight: 500;
+  --sdRefFontWeight: 800;
   --sdRefLineHeight: 1.25;
   --sdRefFontSize: 26px;
   --sdRefLetterSpacing: 0;
   --sdRefParagraphSpacing: 0;
   --sdRefParagraphIndent: 0;
   --sdRefTextDecoration: none;
   --sdRefTextCase: none;
+  --sdRefFontStyle: italic;
   --sdDeepRefFontFamily: Arial;
-  --sdDeepRefFontWeight: 500;
+  --sdDeepRefFontWeight: 800;
   --sdDeepRefLineHeight: 1.25;
   --sdDeepRefFontSize: 26px;
   --sdDeepRefLetterSpacing: 0;
   --sdDeepRefParagraphSpacing: 0;
   --sdDeepRefParagraphIndent: 0;
   --sdDeepRefTextDecoration: none;
-  --sdDeepRefTextCase: none;`,
+  --sdDeepRefTextCase: none;
+  --sdDeepRefFontStyle: italic;`,
     );
   });
 });

test/integration/object-value-references.test.ts

@@ -45,9 +45,9 @@ describe('typography references', () => {
     const file = await promises.readFile(outputFilePath, 'utf-8');
     expect(file).to.include(
       `
-  --sdBefore: 700 36px/1 'Aria Sans';
-  --sdFontHeadingXxl: 700 36px/1 'Aria Sans';
-  --sdAfter: 700 36px/1 'Aria Sans';`,
+  --sdBefore: 400 italic 36px/1 'Aria Sans';
+  --sdFontHeadingXxl: 400 italic 36px/1 'Aria Sans';
+  --sdAfter: 400 italic 36px/1 'Aria Sans';`,
     );
   });
 

test/integration/tokens/expand-composition.tokens.json

@@ -32,7 +32,7 @@
   "typography": {
     "value": {
       "fontFamily": "Arial",
-      "fontWeight": "500",
+      "fontWeight": "{fontWeightRef}",
       "lineHeight": "1.25",
       "fontSize": "26",
       "letterSpacing": "0",
@@ -43,6 +43,10 @@
     },
     "type": "typography"
   },
+  "fontWeightRef": {
+    "value": "ExtraBold italic",
+    "type": "fontWeights"
+  },
   "border": {
     "value": {
       "color": "#FFFF00",

test/integration/tokens/object-value-references.tokens.json

@@ -8,7 +8,7 @@
       "xxl": {
         "value": {
           "fontFamily": "Aria Sans",
-          "fontWeight": "bold",
+          "fontWeight": "{fontWeightRef}",
           "lineHeight": "1",
           "fontSize": "36",
           "letterSpacing": "1",
@@ -39,6 +39,10 @@
   "shadowRef": {
     "value": "{shadow}"
   },
+  "fontWeightRef": {
+    "value": "Regular Italic",
+    "type": "fontWeights"
+  },
   "border": {
     "value": {
       "color": "#FFFF00",

test/spec/checkAndEvaluateMath.spec.ts

@@ -56,4 +56,10 @@ describe('check and evaluate math', () => {
     expect(checkAndEvaluateMath('min(10, 24, 5, 12, 6) 8 * 14px')).to.equal('5 112px');
     expect(checkAndEvaluateMath('ceil(roundTo(16/1.2,0)/2)*2')).to.equal('14');
   });
+
+  it('does not unnecessarily remove wrapped quotes around font-family values', () => {
+    expect(checkAndEvaluateMath(`800 italic 16px/1 'Arial Black'`)).to.equal(
+      `800 italic 16px/1 'Arial Black'`,
+    );
+  });
 });

test/spec/css/transformTypographyForCSS.spec.ts

@@ -1,7 +1,6 @@
 import { expect } from '@esm-bundle/chai';
 import {
   transformTypographyForCSS,
-  hasWhiteSpace,
   isCommaSeparated,
 } from '../../../src/css/transformTypography.js';
 import { runTransformSuite } from '../../suites/transform-suite.spec.js';
@@ -69,4 +68,23 @@ describe('transform typography', () => {
       }),
     ).to.equal('300 20px/1.5 sans-serif');
   });
+
+  it('includes fontStyle if included in the fontWeight', () => {
+    expect(
+      transformTypographyForCSS({
+        fontWeight: 'light Italic',
+        fontSize: '20',
+        lineHeight: '1.5',
+      }),
+    ).to.equal('300 italic 20px/1.5 sans-serif');
+
+    expect(
+      transformTypographyForCSS({
+        fontWeight: 'light',
+        fontSize: '20',
+        lineHeight: '1.5',
+        fontStyle: 'italic',
+      }),
+    ).to.equal('300 italic 20px/1.5 sans-serif');
+  });
 });