util_runtpcds.sh

#!/bin/bash

function timedate() {
    TZ="America/Los_Angeles" date
    echo ""
}

function usageExit() {
    echo "Usage: sh util_runtpcds.sh SCALE FORMAT"
    echo "SCALE must be greater than 1"
    echo "FORMAT must be either 'orc' | 'parquet'"
    exit 1
}

function setupRun() {
    ID=`TZ='America/Los_Angeles' date +"%m.%d.%Y-%H.%M.%S"`

    # --- QUERY FILE NAME ---
    QUERY_BASE_NAME="sample-queries-tpcds/query"
    QUERY_FILE_EXT=".sql"
    
    # --- SETTINGS ---
    SETTINGS_PATH="settings.sql"

    # --- REPORT NAME ---
    REPORT_NAME="time_elapsed_tpcds"

    # --- DATABASE ---
    DATABASE="tpcds_bin_partitioned_${FORMAT}_${SCALE}"

    # --- CLOCK ---
    CLOCK_FILE="aaa_clocktime.txt"

    if [[ -f $CLOCK_FILE ]]; then
        rm $CLOCK_FILE
        echo "Old clock removed"
    fi
    echo "Created new clock"

    # generate time report
    rm $REPORT_NAME*".csv"
    echo "Old report removed"
    echo "query #", "secs elapsed", "status" > $REPORT_NAME".csv"
    echo "New report generated"

    # remove old llapio_summary
    rm "llapio_summary"*".csv"
    echo "Old llapio_summary*.csv removed"

    # clear and make new log directory
    if [[ -d log_query/ ]]; then
        rm -r log_query/
        echo "Old logs removed"
    fi
    mkdir log_query/
    echo "Log folder generated"

    # make executable
    chmod +x util_internalGetPAT.sh
    chmod +x util_internalRunQuery.sh
    chmod -R +x PAT/

    # absolute path
    CURR_DIR="`pwd`/"
}

function runBenchmark() {
    echo "Run queries for TPC-DS ${FORMAT} at scale ${SCALE}" > $CLOCK_FILE
    timedate >> $CLOCK_FILE
    
    # range of queries
    START=1
    END=99
    REPEAT=1
    for (( QUERY_NUM = $START; QUERY_NUM <= $END; QUERY_NUM++ )); do
        for (( j = 0; j < $REPEAT; j++ )); do
            query_path=(${QUERY_BASE_NAME}${QUERY_NUM}${QUERY_FILE_EXT})
            LOG_PATH="log_query/logquery${QUERY_NUM}.${j}.txt"

            ./util_internalRunQuery.sh "$DATABASE" "$CURR_DIR$SETTINGS_PATH" "$CURR_DIR$query_path" "$CURR_DIR$LOG_PATH" "$QUERY_NUM" "$CURR_DIR$REPORT_NAME.csv"

            # ./util_internalGetPAT.sh /$CURR_DIR/util_internalRunQuery.sh "$DATABASE" "$CURR_DIR$SETTINGS_PATH" "$CURR_DIR$query_path" "$CURR_DIR$LOG_PATH" "$QUERY_NUM" "$CURR_DIR$REPORT_NAME.csv" tpcdsPAT"$ID"/query"$i"/
        done
    done

    echo "Finished" >> $CLOCK_FILE
    timedate >> $CLOCK_FILE
}

function generateZipReport() {
    python3 parselog.py
    mv $REPORT_NAME".csv" $REPORT_NAME$ID".csv"
    zip -j log_query.zip log_query/*
    zip -r "tpcds-"$SCALE"GB-"$ID".zip" log_query.zip PAT/PAT-collecting-data/results/tpcdsPAT"$ID"/* $REPORT_NAME$ID".csv" "llapio_summary"*".csv"
    rm log_query.zip 
}

# --- SCRIPT START ---
SCALE=$1
FORMAT=$2

if [[ "X$SCALE" == "X" || $SCALE -eq 1 ]]; then
    usageExit
fi
if ! [[ "$SCALE" =~ ^[0-9]+$ ]]; then
    echo "'$SCALE' is not a number!"
    usageExit
fi
if [[ "$FORMAT" != "orc" && "$FORMAT" != "parquet" ]]; then
    usageExit
fi

setupRun

runBenchmark

generateZipReport