Generate component OWL files from templates

docs/DronReleaseWorkflow.md

@@ -2,50 +2,34 @@
 
 To create a new Dron release:
 
-1. [Update the external components](#Update-the-external-components)
-2. [Import the external components](#Import-the-external-components)
-3. [Build the release files](#Build-the-release-files-and-create-a-GitHub-release)
-4. [Create a GitHub release](#Build-the-release-files-and-create-a-GitHub-release)
+1. [Update the components](#Update-the-components)
+2. [Build the release files](#Build-the-release-files-and-create-a-GitHub-release)
+3. [Create a GitHub release](#Build-the-release-files-and-create-a-GitHub-release)
 
-## Update the external components
-DrOn has three large external components:
+## Update the components
+
+DrOn has three components:
 
 1. dron-rxnorm
 2. dron-ndc
 3. dron-ingredient
 
-Due to their size, they are not in GitHub version control. 
-
-The process of preparing a release works as follows:
-
-- Build the components using the DrOn builder application (as of 2021, done by Matt McConnell).
-- Upload the components to their s3 bucket. *Important: After the upload, all three components must be available from the following locations*:
-    - https://drugontology.s3.amazonaws.com/dron-ingredient.owl
-    - https://drugontology.s3.amazonaws.com/dron-ndc.owl
-    - https://drugontology.s3.amazonaws.com/dron-rxnorm.owl
-- (If the location has to change, make sure that update the `DRON_RELEASE_LOCATION` variable in `src/ontology/dron.Makefile` before running an ODK release.)
-
-The Dron Builder code is currently still bundled in a Scala module managed by Matt McConnell. 
-
-## Import the external components
+They are generated from the templates in `src/templates/`.
+The templates are updated from RxNorm monthly releases.
 
-Before running the normal ODK release workflow, the release manager (as of 2021, Bill Hogan) has to download the components into their local DrOn repo. This is achieved as follows:
+TODO: RxNorm updates are not implemented in this version of the code.
+That functionality will be added soon!
 
-- In your Terminal, go to your DrOn edit directory, for example:
+Build them with the `all_components` task:
 
 ```
 cd dron/src/ontology
+sh run.sh make all_components -B
 ```
 
-- To download the external components previously uploaded to their s3 bucket: 
-
-```
-sh run.sh make download_components
-```
-
-This command will now download dron-ingredient.owl, dron-ndc.owl, dron-rxnorm.owl to your local machine (and overwrite any old versions of these three files), so that the ODK will recognise them as part of the upcoming release.
+The files will be created in `src/ontology/components/` and will be imported into `dron-edit.owl`.
 
-## 3+4. Build the release files and create a GitHub release
+## 2+3. Build the release files and create a GitHub release
 
 Follow the [default ODK release instructions](odk-workflows/ReleaseWorkflow.md).
 

src/ontology/dron.Makefile

@@ -23,29 +23,61 @@ dron-lite.owl: $(TMPDIR)/dron-edit_lite.owl
 		reduce -r ELK \
 		annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@
 
-##################################
-#### Create tempates from OWL ####
-##################################
+###################################
+#### Create OWL from templates ####
+###################################
 
 $(TMPDIR)/ldtab.jar: | $(TMPDIR)
 	wget https://github.com/ontodev/ldtab.clj/releases/download/v2023-12-21/ldtab.jar -O $@
 
 LDTAB := java -jar $(TMPDIR)/ldtab.jar
 
-$(TMPDIR)/reverse/:
-	mkdir -p $@
+# Load DrOn templates into SQLite.
+$(TMPDIR)/dron.db: $(SCRIPTSDIR)/create-dron-tables.sql $(SCRIPTSDIR)/load-dron-tables.sql
+	rm -f $@
+	sqlite3 $@ < $<
+	sqlite3 $@ < $(word 2,$^)
+
+# Convert DrOn template tables to LDTab format tables.
+$(TMPDIR)/ldtab.db: $(TMPDIR)/dron.db $(SCRIPTSDIR)/prefix.tsv $(SCRIPTSDIR)/create-statement-table.sql $(SCRIPTSDIR)/convert-dron-ldtab.sql | $(TMPDIR)/ldtab.jar
+	rm -f $@
+	$(LDTAB) init $@
+	$(LDTAB) prefix $@ $(word 2,$^)
+	sed 's/statement/dron_ingredient/' $(word 3,$^) | sqlite3 $@
+	sed 's/statement/dron_rxnorm/' $(word 3,$^) | sqlite3 $@
+	sed 's/statement/dron_ndc/' $(word 3,$^) | sqlite3 $@
+	sqlite3 $@ < $(word 4,$^)
+
+# Export an LDTab table to Turtle format.
+$(COMPONENTSDIR)/dron-%.ttl: $(TMPDIR)/ldtab.db | $(COMPONENTSDIR)
+	rm -f $@
+	$(LDTAB) export $< $@ --table dron_$*
 
-# Convert components from OFN to OWL for LDTab.
-$(TMPDIR)/reverse/dron-%.owl: $(COMPONENTSDIR)/dron-%.owl | $(TMPDIR)/reverse/
+# Convert a Turtle file to an OWL file in RDFXML format.
+$(COMPONENTSDIR)/%.owl: $(COMPONENTSDIR)/%.ttl
 	$(ROBOT) convert -i $< -o $@
 
+# Override the all_components task.
+all_components: $(COMPONENTSDIR)/dron-ingredient.ttl $(COMPONENTSDIR)/dron-rxnorm.ttl $(COMPONENTSDIR)/dron-ndc.ttl $(COMPONENTSDIR)/dron-ingredient.owl $(COMPONENTSDIR)/dron-rxnorm.owl $(COMPONENTSDIR)/dron-ndc.owl
+
+###################################
+#### Create templates from OWL ####
+###################################
+
 # Convert component OWL files to DrOn templates.
 # This should be a one-time action
 # when migrating away from the old Scala DrOn build system,
 # but it could also be helpful for testing changes to new build scripts.
-.PHONY: reverse
-reverse: $(SCRIPTSDIR)/prefix.tsv $(SCRIPTSDIR)/create-dron-tables.sql $(SCRIPTSDIR)/convert-ldtab-dron.sql $(SCRIPTSDIR)/save-dron-tables.sql $(TMPDIR)/reverse/dron-ingredient.owl $(TMPDIR)/reverse/dron-rxnorm.owl $(TMPDIR)/reverse/dron-ndc.owl | $(TMPDIR)/ldtab.jar
-	$(eval DB=$(TMPDIR)/reverse.db)
+
+$(TMPDIR)/reverse/:
+	mkdir -p $@
+
+# Convert components from OFN to OWL for LDTab to import.
+$(TMPDIR)/reverse/dron-%.owl: components.bk/dron-%.owl | $(TMPDIR)/reverse/
+	$(ROBOT) convert -i $< -o $@
+
+$(TMPDIR)/reverse.db: $(SCRIPTSDIR)/prefix.tsv $(SCRIPTSDIR)/create-dron-tables.sql $(SCRIPTSDIR)/convert-ldtab-dron.sql $(SCRIPTSDIR)/save-dron-tables.sql $(TMPDIR)/reverse/dron-ingredient.owl $(TMPDIR)/reverse/dron-rxnorm.owl $(TMPDIR)/reverse/dron-ndc.owl | $(TMPDIR)/ldtab.jar
+	$(eval DB=$@)
 	rm -f $(DB)
 	$(LDTAB) init $(DB) --table dron_ingredient
 	$(LDTAB) init $(DB) --table dron_rxnorm
@@ -55,6 +87,47 @@ reverse: $(SCRIPTSDIR)/prefix.tsv $(SCRIPTSDIR)/create-dron-tables.sql $(SCRIPTS
 	$(LDTAB) import $(DB) --table dron_ingredient $(TMPDIR)/reverse/dron-ingredient.owl
 	$(LDTAB) import $(DB) --table dron_rxnorm $(TMPDIR)/reverse/dron-rxnorm.owl
 	$(LDTAB) import $(DB) --table dron_ndc $(TMPDIR)/reverse/dron-ndc.owl
+
+.PHONY: reverse
+reverse: $(TMPDIR)/reverse.db
+	# Convert LDTab tables to DrOn templates
 	sqlite3 $(DB) < $(word 3,$^)
+	# Save template tables to TSV
 	cd $(TMPDIR)/reverse/ && sqlite3 ../../$(DB) < ../../$(word 4,$^)
+	# Copy to src/templates/
 	cp $(TMPDIR)/reverse/*.tsv $(TEMPLATEDIR)
+
+###########################
+#### Roundtrip testing ####
+###########################
+
+# Export LDTab tables as TSV and sort them for diffing.
+
+.PRECIOUS: $(COMPONENTSDIR)/dron-%.tsv
+$(COMPONENTSDIR)/dron-%.tsv: $(COMPONENTSDIR)/dron-%.owl
+	$(eval DB=$(TMPDIR)/ldtab.db)
+	rm -f $@
+	$(LDTAB) export $(DB) $@ --table dron_$*
+	head -n1 $@ > $@.head
+	tail -n+2 $@ | sort > $@.rest
+	cat $@.head $@.rest > $@
+	rm $@.head $@.rest
+
+.PRECIOUS: $(TMPDIR)/reverse/dron-%.tsv
+$(TMPDIR)/reverse/dron-%.tsv: | $(TMPDIR)/reverse.db
+	$(eval DB=$|)
+	rm -f $@
+	$(LDTAB) export $(DB) $@ --table dron_$*
+	head -n1 $@ > $@.head
+	tail -n+2 $@ | sort > $@.rest
+	cat $@.head $@.rest > $@
+	rm $@.head $@.rest
+
+$(TMPDIR)/%.tsv.diff: $(TMPDIR)/reverse/%.tsv $(COMPONENTSDIR)/%.tsv
+	-diff -u $^ > $@
+
+$(TMPDIR)/%.owl.diff: $(TMPDIR)/reverse/%.owl $(COMPONENTSDIR)/%.owl
+	-diff -u $^ > $@
+
+.PHONY: roundtrip
+roundtrip: $(COMPONENTSDIR)/dron-ingredient.ttl $(COMPONENTSDIR)/dron-rxnorm.ttl $(COMPONENTSDIR)/dron-ndc.ttl $(TMPDIR)/dron-ingredient.owl.diff $(TMPDIR)/dron-ingredient.tsv.diff $(TMPDIR)/dron-rxnorm.owl.diff $(TMPDIR)/dron-rxnorm.tsv.diff $(TMPDIR)/dron-ndc.owl.diff $(TMPDIR)/dron-ndc.tsv.diff