gbif-taxonomy-for-la

#!/bin/bash

set -e

T=/data/lucene/target
B=/data/lucene/target/backbone
DATE=$(date +%Y-%m-%d_%H-%M)

CMD=$(basename "$0")
FIND_DOCOPTS=$(which docopts)

# Logs to console and file
# https://stackoverflow.com/questions/18460186/writing-outputs-to-log-file-and-console
LOG_FILE=$(basename "$CMD" .sh)-$DATE.log
LOG_FILE_LINK=$(basename "$CMD" .sh).log
exec > >(tee ${LOG_FILE}) 2>&1
ln -sf "$LOG_FILE" "$LOG_FILE_LINK"

if [[ -z $FIND_DOCOPTS ]]
then
  echo "ERROR: Please install docopts https://github.com/docopt/docopts an copy it in your PATH"
  exit 1
fi

eval "$(docopts -V - -h - : "$@" <<EOF
Usage: $CMD [options] <release-date>

Options:
      --backbone                    Download GBIF backbone taxonomy
      --name-authors                Split name and authors from the GBIF backbone
      --prepare-tests               Prepare tests
      --tests                       Run tests
      --filter_lang=<langs>         Filter VernacularName.tsv file for given language [default: ].
      --namematching-distri=<nmv>   Download ALA namematching-distribution version [default: 4.3].
      --namematching-index          Generate namematching index
      --namematching-index-legacy   Generate namematching index legacy (pre namemaching-service)
      --dwca                        Regenerate the dwca zip
      --help                        Show help options.
      --version                     Print program version.
----
$CMD 0.0.1
Copyright (C) 2022 vjrj
License MIT
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
EOF
)"

if [[ ! -d "$T" ]]
then
  mkdir -p "$T/tmp"
fi

if ($backbone)
then
  wget -O $T/backbone.zip https://hosted-datasets.gbif.org/datasets/backbone/current/backbone.zip
  if [[ -d "$T/backbone" ]]
  then
    mv "$T/backbone" "$T/backbone-pre-$DATE"
  fi
  echo "Unzipping backbone"
  (cd $T; mkdir backbone; cd backbone; unzip -q $T/backbone.zip)
  rm "$T/backbone.zip"

  if [[ ! -f "$B/Taxon.tsv.orig" ]]; then
    echo "Taxon.tsv.orig doesn't exist. Creating it"
    mv "$B/Taxon.tsv" "$B/Taxon.tsv.orig"
  fi
  if [ ! -z "${filter_lang}" ]; then
    if [[ ! -f "$B/VernacularName.tsv.orig" ]]; then
      echo "VernacularName.tsv.orig doesn't exist. Creating it"
      mv "$B/VernacularName.tsv" "$B/VernacularName.tsv.orig"
    fi
    input_file="$B/VernacularName.tsv.orig"
    output_file="$B/VernacularName.tsv"
    column_name="language"        # Specify the column name
    language_codes=$filter_lang   # Specify the comma-separated list of language codes

    # Extract the column number dynamically based on the header
    column_number=$(awk -F'\t' -v column_name="$column_name" 'NR==1 {for (i=1; i<=NF; i++) if ($i == column_name) {print i; exit}}' "$input_file")

    # Check if the column name exists
    if [ -z "$column_number" ]; then
        echo "Error: Column '$column_name' not found in the input file."
        exit 1
    fi

    # Convert the language codes to a pattern for awk
    language_pattern=$(echo "$language_codes" | tr ',' '|')

    # Print the header to the output file
    awk -F'\t' 'NR==1 {print; exit}' "$input_file" > "$output_file"

    # Filter rows based on the specified column and case-insensitive language codes, and append to the output file
    awk -v column_number="$column_number" -v language_pattern="$language_pattern" -F'\t' 'tolower($column_number) ~ tolower(language_pattern)' "$input_file" >> "$output_file"

    echo "Filtered data saved to $output_file"
  fi

  #echo Fixing: https://github.com/gbif/portal-feedback/issues/3781
  #egrep -v '^2925549	\\.*The Woody Plants of Korea' "$B/VernacularName.tsv" > "$B/VernacularName.tsv.fixed"
  #mv "$B/VernacularName.tsv.fixed" "$B/VernacularName.tsv"

  # echo Trying to patch col_vernacular.txt
  # cp col_vernacular.txt.patch /data/lucene/sources
  # OUT="$(cd /data/lucene/sources; patch -p0 -r - --forward < col_vernacular.txt.patch)" || echo "${OUT}" | grep "Skipping patch" -q || (echo "$OUT" && false);
  # echo $OUT

fi

if ($prepare_tests)
then
  echo "Creating a test file with the first 10000 lines and names included in tests"
  head -10000 "$B/Taxon.tsv.orig" > "$B/Taxon-tests.tsv"
  grep -oP 'grepName\("\K[^"]*' test/taxonTransformTests.js | while read -r line
  do
    output=$(grep -m 1 "$line" "$B/Taxon.tsv.orig" || true)
    if [ -z "$output" ]; then
      echo "The specie '$line' was not found, so the test will fail"
    else
      echo "$output" >> "$B/Taxon-tests.tsv"
    fi
  done
  echo "Splitting scientificName and scientificNameAuthorship"
  node main.js "$B/Taxon-tests.tsv" "$B/issues.tsv" > "$B/Taxon.tsv"
fi

if ($name_authors)
then
  echo "Trying to install node deps"
  npm install

  echo "Splitting scientificName and scientificNameAuthorship"
  node main.js "$B/Taxon.tsv.orig" "$B/issues.tsv" > "$B/Taxon.tsv"
fi

if ($tests)
then
  ln -sf "$T/namematching-gbif-backbone-lucene-8-$release_date/" /data/lucene/namematching-nm
  env TAXON_FILE="$B/Taxon-tests.tsv" mocha
fi

# https://github.com/AtlasOfLivingAustralia/ala-name-matching
if [[ ! -d "$T/ala-name-matching-distri-$namematching_distri" ]]
then
  mkdir "$T/ala-name-matching-distri-$namematching_distri"
  wget -O "$T/ala-name-matching-distri-$namematching_distri/ala-name-matching-distribution-$namematching_distri-distribution.zip" "https://nexus.ala.org.au/service/local/repositories/releases/content/au/org/ala/ala-name-matching-distribution/$namematching_distri/ala-name-matching-distribution-$namematching_distri-distribution.zip"
  (cd "$T/ala-name-matching-distri-$namematching_distri/"; unzip -q "ala-name-matching-distribution-$namematching_distri-distribution.zip")
fi

if ($namematching_index_legacy)
then
  NTLO="namematching-gbif-backbone-lucene-6-$release_date"
  NTL="namematching-gbif-backbone-lucene-6-$release_date.tgz"

  if [[ -d "$T/$NTLO" ]]; then
    rm -rf "$T/$NTLO"
  fi

  if [[ -f "$T/$NTL" ]]; then
    rm "$T/$NTL"
  fi

#  java -Dlog4j.configuration=file:/usr/lib/nameindexer/log4j.xml -Dfile.encoding=UTF8 -Djava.util.Arrays.useLegacyMergeSort=true -Xmx8g -Xms1g \
#    -jar /usr/lib/nameindexer/nameindexer.*jar -all --dwca $B --irmng /data/lucene/sources/IRMNG_DWC_HOMONYMS

  /usr/lib/nameindexer/nameindexer --all --dwca "$B" --target /data/lucene/namematching --irmng /data/lucene/sources/IRMNG_DWC_HOMONYMS/ --common "$B/VernacularName.tsv"

  cp -a /data/lucene/namematching "$T/$NTLO"
  (cd "$T"; tar cvfz "$NTL" "$NTLO")
fi

if ($namematching_index)
then
  TMPDIR="$T/tmp/"
  mkdir -p $TMPDIR
  export JAVA_OPTIONS="-Xmx3g -Xms3g -Djava.io.tmpdir=$TMPDIR"

  if [[ ! -d "$T/ala-name-matching-distri-$namematching_distri" ]]
  then
    echo "Use --namematching-distri=4.2 or similar before this"
    exit 1
  fi

  NTO="namematching-gbif-backbone-lucene-8-$release_date/"
  NT="namematching-gbif-backbone-lucene-8-$release_date.tgz"

  if [[ -d "$T/$NTO" ]]
  then
    rm -rf "$T/$NTO"
  fi

  if [[ -f "$T/$NT" ]]
  then
    rm "$T/$NT"
  fi

  (cd "$T/ala-name-matching-distri-$namematching_distri/"; chmod +x ./index.sh ; ./index.sh --all --dwca "$B" --target "$T/$NTO" --irmng /data/lucene/sources/IRMNG_DWC_HOMONYMS/ ) # --common "$B/VernacularName.tsv")

  (cd "$T"; tar cvfz "$NT" "$NTO")
fi

if ($dwca)
then
  DWCA_FILE="gbif-backbone-$release_date.zip"
  if [[ -f "$T/$DWCA_FILE" ]]
  then
    rm "$T/$DWCA_FILE"
  fi

  (cd $B; zip "$T/$DWCA_FILE" * --exclude *.orig *-tests.tsv issues.tsv *-sorted )
fi