e2e-tests.sh

#!/bin/bash

# This script executes end-to-end tests.
#
# It is meant to be run in the CI on a daily basis and provide a report of the
# endpoint health status.
#
# It will in order:
# - define a function for computing a hurl summary output
# - index every .hurl file in the examples/ directory,
# - execute it and keep track of its success state,
# - output an inline summary in the console,
# - generate a summary.md that is reused in the CI job summary,
# - exit with an error code in case any hurl test fails.

# Prints a markdown report for the given hurl file.
#
# @param $1 - the path of the hurl file to execute
print_hurl_report () {
  # Compute the group and method associated to the given hurl file
  group=$(echo "$1" | awk -F'/' '{print $(NF-1)}')
  method_name=$(echo "$1" | awk -F'/' '{print $NF}' | sed 's/.hurl//')

  # Execute the hurl file and keep track of it result and status
  result=$(hurl --test --max-time=30 $1 2>&1)
  status=$?

  # Define the "actual" output
  actual=$(hurl --ignore-asserts $1 2>&1)

  # Define the status badge
  if [ $status -eq 0 ]
  then
      status_badge="✅"
  else
      status_badge="❌"
  fi

  echo -e "\
<details>\n
<summary>$method_name $status_badge</summary>\n\
\n\
**Response**\n\
\`\`\`json\n\
$actual\n\
\`\`\`\n\
\n\
  "

  if [ $status -ne 0 ]
  then
    # note: devnet implementation and [JSON body assertions](https://hurl.dev/docs/asserting-response.html#json-body)
    # would allow to improve the expectation output by showing the expected body
    echo -e "\
**Result**\n\
\`\`\`json\n\
$result\n\
\`\`\`\n\
\n\
    "
  fi

  echo -e "</details>\n"
}


# Generate a markdown summary file
markdown_file="summary.md"

# Write the summary header
echo -e "## Hurl Test Execution Summary\n\
\n\
This is a summary of the different methods tested.\n\
\n\
### Methods\n\
\n\
" > "$markdown_file"

# Initialize empty arrays to keep track of succeeding and failing files
succeeding_files=()
failing_files=()

# Use find to get all .hurl files in the examples directory and its subdirectories
all_files=$(find examples -name "*.hurl" | sort)

# Count all files, removing leading spaces
total_files=$(echo "$all_files" | wc -l | tr -d ' ')

# Initialize counter
count=0

# Define a variable to receive the hurl summary outputs
outputs=""

# Loop over all .hurl files
while IFS= read -r file
do
    # Increment counter
    ((count++))

    # Display progress
    echo -e "\n[${count}/${total_files}] Executing: $file"

    # Execute the file with hurl and capture its ouput and exit status
    output=$(hurl --test --max-time=30 --very-verbose $file 2>&1)
    status=$?

    # todo: reformat
    if [ $status -eq 0 ]
    then
        # If the exit status is 0 (success), add file to succeeding_files
        succeeding_files+=("$file")
        status_badge="✅"
    else
        # If the exit status is non-zero (failure), add file to failing_files
        failing_files+=("$file")
        status_badge="❌"
    fi

    # Only add failing tests to the output, otherwise it exceeds github md report limit
    if [ $status -ne 0 ]
    then
        outputs+=$(echo -e "\
<details>\n
<summary>$file $status_badge</summary>\n\
\n\
**Response**\n\
\`\`\`json\n\
$output\n\
\`\`\`\n\
\n\
</details>") >> "$markdown_file"
    fi
done <<< "$all_files"

# Display failing files
if [ ${#failing_files[@]} -gt 0 ]
then
    echo "Failing files:"
    printf '%s\n' "${failing_files[@]}"
fi

# Display a one-line summary
echo "Summary: ${#failing_files[@]} failing files out of $total_files"

# Generate a markdown summary file
markdown_file="summary.md"

# Write the summary header
echo -e "## Hurl Test Execution Summary\n\
\n\
This is a summary of the different methods tested.\n\
\n\
### Failing Methods\n\
\n\
| Group | Method | Status |\n\
| --- | --- | --- |" > "$markdown_file"

# Add failing methods to the table
for method in "${failing_files[@]}"; do
    group=$(echo "$method" | awk -F'/' '{print $(NF-1)}')
    method_name=$(echo "$method" | awk -F'/' '{print $NF}' | sed 's/.hurl//')
    echo "| $group | $method_name | ❌ |" >> "$markdown_file"
done

# Add the succeeding methods section
echo -e "\n\
### Succeeding Methods\n\
\n\
| Group | Method | Status |\n\
| --- | --- | --- |" >> "$markdown_file"

# Add succeeding methods to the table
for method in "${succeeding_files[@]}"; do
    group=$(echo "$method" | awk -F'/' '{print $(NF-1)}')
    method_name=$(echo "$method" | awk -F'/' '{print $NF}' | sed 's/.hurl//')
    echo "| $group | $method_name | ✅ |" >> "$markdown_file"
done

echo -e "\n\
### Execution Summary\n\
\n\
$outputs\
" >> "$markdown_file"

# If there are any failing files, return an error
if [ ${#failing_files[@]} -gt 0 ]
then
    # TODO: As of 02/08/2023 some hurl tests are consistently failing. Once those are fixed enable back exiting with an error code
    # exit 1
    exit 0
fi

exit 0