Miscellaneous historical and para-IPA modifier letters #2290

Workflow file for this run

.github/workflows/cli-build-instructions.yml at c6ecabc

	name: build.md

	on:
	push:
	branches:
	- '*'
	pull_request:
	branches:
	- '*'

	env:
	CURRENT_UVERSION: 17.0.0 # FIX_FOR_NEW_VERSION
	PREVIOUS_UVERSION: 16.0.0

	jobs:

	# Using the Java style formatter google-java-style provided by the Spotless
	# plugin configured in the root pom.xml using 4-space indents (AOSP style).
	# Spotless is configured to run only on files in this branch (PR) that differ
	# from origin/main
	formatter:
	name: Formatter + Style checker
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Unicode Tools
	uses: actions/checkout@v3
	with:
	fetch-depth: 0 # fetch all branches so that Spotless can resolve `origin/main`
	- name: Check Java style
	run: mvn spotless:check \|\| (echo "Style checker failed. Formatting changes can be applied by 'mvn spotless:apply'" && exit 1)

	# Only run 1 or 2 commands for in-source build instructions, just to demonstrate
	# what translating between an out-of-source build and an in-source build would
	# look like. Rely on out-of-source build steps as the canonical way to
	# execute tools in CI and write corresponding documentation.
	in-source-build:
	name: In-source Instructions (Examples)
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Unicode Tools
	uses: actions/checkout@v3
	- name: Get the CLDR_REF from pom.xml
	id: cldr_ref
	run: echo "CLDR_REF="$(mvn help:evaluate -Dexpression=cldr.version -q -DforceStdout \| cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
	- name: Verify CLDR checkout ref
	run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
	- name: Cache CLDR repository
	uses: actions/cache@v3
	with:
	path: cldr
	key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
	restore-keys: \|
	cldr
	- name: Check out CLDR
	uses: actions/checkout@v3
	with:
	repository: unicode-org/cldr
	path: cldr
	ref: main
	fetch-depth: 0
	- name: Switch CLDR to CLDR_REF
	run: cd cldr && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
	- name: Backup Unicodetools and CLDR for jsps # this is needed only for the Docker build
	run:
	mkdir -p UnicodeJsps/target && tar -cpz --exclude=.git -f UnicodeJsps/target/cldr-unicodetools.tgz ./cldr ./unicodetools
	- name: Symlink CLDR working copy to be sibling of Unicode Tools
	run: \|
	ln -s $(pwd)/cldr ..
	- name: Set up JDK 11
	uses: actions/setup-java@v1
	with:
	java-version: 11
	- name: Cache local Maven repository
	uses: actions/cache@v2
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Set up in-source output dir
	run: \|
	mkdir -p Generated/BIN

	# Since these are just examples to smoke-test the in-source build process,
	# let’s not run the whole build and test suite, which is quite slow (6 min
	# 26 s as of this writing). Just run the invariant tests and smoke-test
	# MakeUnicodeFiles. We don’t even check that MakeUnicodeFiles doesn’t
	# change anything, which makes little sense; but that is the job of the
	# other job.
	- name: Run invariant tests
	run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Run command - Make Unicode Files
	run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


	# Out-of-source build.
	ucd-and-smoke-tests:
	name: Check UCD consistency, invariants, smoke-test generators
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Unicode Tools
	uses: actions/checkout@v3
	with:
	path: unicodetools/mine/src
	- name: Get the CLDR_REF from pom.xml
	id: cldr_ref
	run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout \| cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
	- name: Verify CLDR checkout ref
	run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
	- name: Cache CLDR repository
	uses: actions/cache@v3
	with:
	path: cldr/mine/src
	key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
	restore-keys: \|
	cldr
	- name: Check out CLDR
	uses: actions/checkout@v3
	with:
	repository: unicode-org/cldr
	path: cldr/mine/src
	ref: main
	fetch-depth: 0
	- name: Switch CLDR to CLDR_REF
	run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
	- name: Set up JDK 11
	uses: actions/setup-java@v1
	with:
	java-version: 11
	- name: Cache local Maven repository
	uses: actions/cache@v2
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Set up out-of-source output dir
	run: \|
	mkdir -p unicodetools/mine/Generated/BIN

	- name: Run command - Make Unicode Files
	run: \|
	cd unicodetools/mine/src
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Check that UCD files are consistent
	run: \|
	cd unicodetools/mine/src
	./py/copygenerateducd.py --out-of-source -y
	git diff --compact-summary --exit-code \|\| {
	git diff --compact-summary \|
	awk '{
	if (previous) {
	print "::error file="previous",title=File must be regenerated::Run org.unicode.text.UCD.Main build MakeUnicodeFiles and copy any changed files to unicodetools/data/ucd/dev."
	}
	previous=$1
	}'
	exit 1
	}

	# Only test once we know the UCD is internally consistent.
	# MakeUnicodeFiles is much faster than this anyway.
	- name: Run command - Build and Test
	run: \|
	cd unicodetools/mine/src
	MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -Dtest=!TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DEMIT_GITHUB_ERRORS
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Upload UnicodeTestResults artifact
	if: always()
	uses: actions/upload-artifact@v3
	with:
	name: unicode-test-results
	path: \|
	unicodetools/mine/Generated/UnicodeTestResults.*

	# https://github.com/unicode-org/unicodetools/blob/main/docs/emoji/aac.md#aacorderjava
	- name: Run command - AAC Order
	run: \|
	cd unicodetools/mine/src
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.tools.AacOrder" -Dexec.args="version $CURRENT_UVERSION build MakeUnicodeFiles" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md
	- name: Run command - IDNA
	run: \|
	cd unicodetools/mine/src
	# Confirm that the directory for the bin files exists
	compgen -G "../Generated/BIN"
	# "Delete all the bin files" as per instructions
	rm -rf ../Generated/BIN/*
	# run GenerateIdna
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.idna.GenerateIdna" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# https://github.com/unicode-org/unicodetools/blob/main/docs/idna.md
	- name: Run command - Generate IDNA Test
	run: \|
	cd unicodetools/mine/src
	# run GenerateIdnaTest
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.idna.GenerateIdnaTest" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# https://github.com/unicode-org/unicodetools/blob/main/docs/newunicodeproperties.md#run-generateenumsjava
	- name: Run command - Generate Enums Test
	run: \|
	cd unicodetools/mine/src
	# run GenerateEnums
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.props.GenerateEnums" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	# apply formatting because generated file will not pass Java formatter
	mvn spotless:apply '-DspotlessFiles=.*[\\/]org[\\/]unicode[\\/]props[\\/]UcdProperty(Values)?.java'
	# Fail if we haven't committed changes from Generate Enums.
	echo "Check if GenerateEnums output has been committed to repo"
	echo "git diff --compact-summary"
	if git diff --compact-summary --exit-code; then
	echo "Success: Any output from GenerateEnums has already been committed."
	exit 0
	else
	echo "Failure: GenerateEnums has changes that need to be committed."
	exit 1
	fi
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# https://github.com/unicode-org/unicodetools/blob/main/docs/security.md#generating
	- name: Run command - Generate Confusables
	run: \|
	cd unicodetools/mine/src
	# run GenerateConfusables
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.GenerateConfusables" -Dexec.args="-c -b" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	# run Build & Test command again to rerun TestSecurity test to verify output is okay
	MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestSecurity -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# https://github.com/unicode-org/unicodetools/blob/main/docs/unihan.md#run-generateunihancollators
	- name: Run command - Generate Unihan Collators
	run: \|
	cd unicodetools/mine/src
	# run GenerateUnihanCollators
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.draft.GenerateUnihanCollators" -Dexec.args="" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# https://github.com/unicode-org/unicodetools/blob/main/docs/newunicodeproperties.md#checking-other-properties
	- name: Run command - Checking Other Properties
	run: \|
	cd unicodetools/mine/src
	# run CheckProperties
	# Note: the test-compile phase/target needs to precede `compile exec:java` because
	# the source code is in src/test/java, not src/main/java, and we need the
	# code to compile before it is executed.
	mvn -s .github/workflows/mvn-settings.xml -Dexec.mainClass="org.unicode.propstest.CheckProperties" -Dexec.classpathScope=test test-compile -Dexec.args="COMPARE ALL $PREVIOUS_UVERSION" compile exec:java -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# Out-of-source build.
	uca:
	name: Check UCA data
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Unicode Tools
	uses: actions/checkout@v3
	with:
	repository: unicode-org/unicodetools
	path: unicodetools/mine/src
	- name: Checkout base UnicodeData.txt
	if: ${{ github.event_name == 'pull_request'}}
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.base.sha }}
	path: base
	sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
	- name: Compare repertoire
	if: ${{ github.event_name == 'pull_request'}}
	run: \|
	# Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
	sed 's/^$[^;];[^;]$;.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
	sed 's/^$[^;];[^;]$;.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
	set +e
	diff base-repertoire.txt merged-repertoire.txt
	echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
	- name: Get the CLDR_REF from pom.xml
	id: cldr_ref
	run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout \| cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
	- name: Verify CLDR checkout ref
	run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
	- name: Cache CLDR repository
	uses: actions/cache@v3
	with:
	path: cldr/mine/src
	key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
	restore-keys: \|
	cldr
	- name: Check out CLDR
	uses: actions/checkout@v3
	with:
	repository: unicode-org/cldr
	path: cldr/mine/src
	ref: main
	fetch-depth: 0
	- name: Switch CLDR to CLDR_REF
	run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
	- name: Set up JDK 11
	uses: actions/setup-java@v1
	with:
	java-version: 11
	- name: Cache local Maven repository
	uses: actions/cache@v2
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Set up out-of-source output dir
	run: \|
	mkdir -p unicodetools/mine/Generated/BIN

	# https://github.com/unicode-org/unicodetools/blob/main/docs/uca/index.md#tools--tests
	# Note: Not running desuffixucd.py in UCA jobs because no version numbers detected in data file names
	- name: Run command - UCA - collation validity log
	run: \|
	cd unicodetools/mine/src
	echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
	if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
	then set +e
	fi
	# invoke main() in class ...UCA.Main
	mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main" -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION
	# check for output file
	compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# Out-of-source build.
	security:
	name: Check security data invariants
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Unicode Tools
	uses: actions/checkout@v3
	with:
	repository: unicode-org/unicodetools
	path: unicodetools/mine/src
	- name: Checkout base UnicodeData.txt
	if: ${{ github.event_name == 'pull_request'}}
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.base.sha }}
	path: base
	sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
	- name: Compare repertoire
	if: ${{ github.event_name == 'pull_request'}}
	run: \|
	# Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
	sed 's/^$[^;];[^;]$;.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
	sed 's/^$[^;];[^;]$;.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
	set +e
	diff base-repertoire.txt merged-repertoire.txt
	echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
	- name: Get the CLDR_REF from pom.xml
	id: cldr_ref
	run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout \| cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
	- name: Verify CLDR checkout ref
	run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
	- name: Cache CLDR repository
	uses: actions/cache@v3
	with:
	path: cldr/mine/src
	key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
	restore-keys: \|
	cldr
	- name: Check out CLDR
	uses: actions/checkout@v3
	with:
	repository: unicode-org/cldr
	path: cldr/mine/src
	ref: main
	fetch-depth: 0
	- name: Switch CLDR to CLDR_REF
	run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
	- name: Set up JDK 11
	uses: actions/setup-java@v1
	with:
	java-version: 11
	- name: Cache local Maven repository
	uses: actions/cache@v2
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Set up out-of-source output dir
	run: \|
	mkdir -p unicodetools/mine/Generated/BIN

	- name: Run invariant tests
	run: \|
	cd unicodetools/mine/src
	echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
	if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
	then ERROR="::notice"
	else ERROR="::error"
	fi
	MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd) -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false -DEMIT_GITHUB_ERRORS 2>&1 \| sed "s/^::error/$ERROR/"
	STATUS=${PIPESTATUS[0]}
	if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
	then exit 0
	else exit $STATUS
	fi
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Miscellaneous historical and para-IPA modifier letters #2290

Workflow file

Miscellaneous historical and para-IPA modifier letters #2290

Jobs

Run details

Workflow file for this run