DO NOT MERGE - Pipeline performance test project

performance-test/.gitignore

@@ -0,0 +1,151 @@
+##########################
+# KEDRO PROJECT
+
+# ignore all local configuration
+conf/local/**
+!conf/local/.gitkeep
+
+# ignore potentially sensitive credentials files
+conf/**/*credentials*
+
+# ignore everything in the following folders
+data/**
+
+# except their sub-folders
+!data/**/
+
+# also keep all .gitkeep files
+!.gitkeep
+
+# keep also the example dataset
+!data/01_raw/*
+
+
+##########################
+# Common files
+
+# IntelliJ
+.idea/
+*.iml
+out/
+.idea_modules/
+
+### macOS
+*.DS_Store
+.AppleDouble
+.LSOverride
+.Trashes
+
+# Vim
+*~
+.*.swo
+.*.swp
+
+# emacs
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# C extensions
+*.so
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+.static_storage/
+.media/
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

performance-test/.viz/stats.json

@@ -0,0 +1 @@
+{}

performance-test/README.md

@@ -0,0 +1,19 @@
+# performance-test
+
+## Overview
+
+This is a test project meant to simulate delays in specific parts of a Kedro pipeline. It's supposed to be a tool to gauge pipeline performance and be used to compare in-development changes to Kedro with an already stable release version.
+
+## Usage
+
+There are three delay parameters that can be set in this project:
+
+**hook_delay** - Simulates slow-loading hooks due to it performing complex operations or accessing external services that can suffer from latency.
+
+**load_delay** - Simulates a delay in loading a dataset, because of a large size or connection latency, for example.
+
+**save_delay** - Simulates a delay in saving an output file, because of, for example, connection delay in accessing remote storage.
+
+When invoking the `kedro run` command, you can pass the desired value in seconds for each delay as a parameter using the `--params` flag. For example:
+
+`kedro run --params=hook_delay=5,load_delay=5,save_delay=5`

performance-test/conf/base/catalog.yml

@@ -0,0 +1,58 @@
+congress_expenses:
+  type: spark.SparkDataset
+  filepath: data/gastos-deputados.csv
+  file_format: csv
+  load_args:
+    header: True
+    inferSchema: True
+
+expenses_per_party:
+  type: spark.SparkDataset
+  filepath: data/output/expenses_per_party.csv
+  file_format: csv
+  save_args:
+    sep: ','
+    header: True
+    mode: overwrite
+  load_args:
+    header: True
+    inferSchema: True
+
+largest_expense_source:
+  type: spark.SparkDataset
+  filepath: data/output/largest_expense_source.parquet
+  file_format: parquet
+  save_args:
+    sep: ','
+    header: True
+    mode: overwrite
+
+top_spender_per_party:
+  type: spark.SparkDataset
+  filepath: data/output/top_spender_per_party.csv
+  file_format: csv
+  save_args:
+    sep: ','
+    header: True
+    mode: overwrite
+  load_args:
+    header: True
+    inferSchema: True
+
+top_overall_spender:
+  type: spark.SparkDataset
+  filepath: data/output/top_overall_spender.parquet
+  file_format: parquet
+  save_args:
+    sep: ','
+    header: True
+    mode: overwrite
+
+top_spending_party:
+  type: spark.SparkDataset
+  filepath: data/output/top_spending_party.parquet
+  file_format: parquet
+  save_args:
+    sep: ','
+    header: True
+    mode: overwrite

performance-test/conf/base/parameters.yml

@@ -0,0 +1,3 @@
+hook_delay: 0
+load_delay: 0
+save_delay: 0

performance-test/conf/base/parameters_expense_analysis.yml

@@ -0,0 +1,5 @@
+# This is a boilerplate parameters config generated for pipeline 'expense_analysis'
+# using Kedro 0.19.8.
+#
+# Documentation for this file format can be found in "Parameters"
+# Link: https://docs.kedro.org/en/0.19.8/configuration/parameters.html

performance-test/conf/base/spark.yml

@@ -0,0 +1,8 @@
+# You can define spark specific configuration here.
+
+spark.driver.maxResultSize: 3g
+spark.hadoop.fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
+spark.sql.execution.arrow.pyspark.enabled: true
+
+# https://docs.kedro.org/en/stable/integrations/pyspark_integration.html#tips-for-maximising-concurrency-using-threadrunner
+spark.scheduler.mode: FAIR