Release 1.4.0

.github/workflows/publish.yml

@@ -4,16 +4,26 @@ on:
     paths:
       - 'setup.py'
 
-
 jobs:
   Pipeline:
-    if: github.ref == 'refs/heads/master' || contains(github.ref, 'hotfix/')
-
-    runs-on: ubuntu-16.04
-    container: quintoandar/python-3-7-java
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+
+    - uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: microsoft
+
+    - uses: vemonet/setup-spark@v1
+      with:
+        spark-version: '3.5.1'
+        hadoop-version: '3'
 
     - name: Install dependencies
       run: make ci-install
@@ -24,13 +34,18 @@ jobs:
     - name: Get version
       run: echo "version=$(grep __version__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2)" >> $GITHUB_ENV
 
+    - name: Get release notes
+      id: get_release_notes
+      uses: ffurrer2/extract-release-notes@v1
+
     - name: Create release
       uses: actions/create-release@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         tag_name: ${{ env.version }}
         release_name: Release ${{ env.version }}
+        body: ${{ steps.get_release_notes.outputs.release_notes }}
 
     - name: Release already exist
       if: ${{ failure() }}

.github/workflows/staging.yml

@@ -0,0 +1,55 @@
+name: "Publish Dev Package"
+on:
+  push:
+    paths:
+      - "setup.py"
+
+jobs:
+  Pipeline:
+    if: github.ref == 'refs/heads/staging'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: microsoft
+
+      - uses: vemonet/setup-spark@v1
+        with:
+          spark-version: '3.5.1'
+          hadoop-version: '3'
+
+      - name: Install dependencies
+        run: make ci-install
+
+      - name: Get version
+        run: echo "version=$(grep __version__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2 )" >> $GITHUB_ENV
+
+      - name: Build package
+        run: make package
+
+      - name: Create release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.version }}
+          release_name: Release ${{ env.version }}
+          prerelease: true
+
+      - name: Release already exist
+        if: ${{ failure() }}
+        run: echo Release already exist
+
+      - name: Publish release to pypi.org
+        if: ${{ success() }}
+        env:
+          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: PYTHONPATH=./pip/deps python -m twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD --verbose dist/*

.github/workflows/test.yml

@@ -3,16 +3,29 @@ on:
   push:
     branches:
       - master
+      - staging
       - hotfix/**
   pull_request:
 
 jobs:
   Pipeline:
-    runs-on: ubuntu-16.04
-    container: quintoandar/python-3-7-java
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+
+    - uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: microsoft
+
+    - uses: vemonet/setup-spark@v1
+      with:
+        spark-version: '3.5.1'
+        hadoop-version: '3'
 
     - name: Install dependencies
       run: make ci-install

.gitignore

@@ -47,6 +47,7 @@ coverage.xml
 *.cover
 .hypothesis/
 *cov.xml
+test_folder/
 
 # Translations
 *.mo
@@ -65,6 +66,7 @@ instance/
 
 # PyBuilder
 target/
+pip/
 
 # Jupyter Notebook
 .ipynb_checkpoints

CHANGELOG.md

@@ -3,6 +3,108 @@ All notable changes to this project will be documented in this file.
 
 Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each release or unreleased log for a better organization.
 
+## [Unreleased]
+
+## [1.4.0](https://github.com/quintoandar/butterfree/releases/tag/1.4.0)
+* Add Delta support ([#370](https://github.com/quintoandar/butterfree/pull/370))
+
+## [1.3.5](https://github.com/quintoandar/butterfree/releases/tag/1.3.5)
+* Auto create feature sets ([#368](https://github.com/quintoandar/butterfree/pull/368))
+
+## [1.3.4](https://github.com/quintoandar/butterfree/releases/tag/1.3.4)
+* Fix Cassandra Config and tests ([#366](https://github.com/quintoandar/butterfree/pull/366))
+
+## [1.3.3](https://github.com/quintoandar/butterfree/releases/tag/1.3.3)
+* Fix Cassandra Config and Numpy version ([#364](https://github.com/quintoandar/butterfree/pull/364))
+
+## [1.3.2](https://github.com/quintoandar/butterfree/releases/tag/1.3.2)
+* Fix publish script ([#362](https://github.com/quintoandar/butterfree/pull/362))
+
+## [1.3.2](https://github.com/quintoandar/butterfree/releases/tag/1.3.2)
+* Fix publish script ([#360](https://github.com/quintoandar/butterfree/pull/362))
+
+## [1.3.1](https://github.com/quintoandar/butterfree/releases/tag/1.3.1)
+* Timestamp NTZ available ([#360](https://github.com/quintoandar/butterfree/pull/360))
+
+## [1.3.0](https://github.com/quintoandar/butterfree/releases/tag/1.3.0)
+* Bump versions  ([#355](https://github.com/quintoandar/butterfree/pull/355))
+* Sphinx version ([#356](https://github.com/quintoandar/butterfree/pull/356))
+
+## [1.2.4](https://github.com/quintoandar/butterfree/releases/tag/1.2.4)
+* Auto create feature sets ([#351](https://github.com/quintoandar/butterfree/pull/351))
+
+## [1.2.3](https://github.com/quintoandar/butterfree/releases/tag/1.2.3)
+* Optional params ([#347](https://github.com/quintoandar/butterfree/pull/347))
+
+### Changed
+* Optional row count validation ([#340](https://github.com/quintoandar/butterfree/pull/340))
+
+## [1.2.2](https://github.com/quintoandar/butterfree/releases/tag/1.2.2)
+
+### Changed
+* Optional row count validation ([#284](https://github.com/quintoandar/butterfree/pull/284))
+* Bump several libs versions ([#333](https://github.com/quintoandar/butterfree/pull/333))
+
+## [1.2.1](https://github.com/quintoandar/butterfree/releases/tag/1.2.1)
+
+### Changed
+* Update README.md ([#331](https://github.com/quintoandar/butterfree/pull/331))
+* Update Github Actions Workflow runner ([#332](https://github.com/quintoandar/butterfree/pull/332))
+* Delete sphinx version. ([#334](https://github.com/quintoandar/butterfree/pull/334))
+
+### Fixed
+* Add the missing link for H3 geohash ([#330](https://github.com/quintoandar/butterfree/pull/330))
+
+## [1.2.0](https://github.com/quintoandar/butterfree/releases/tag/1.2.0)
+
+### Added
+* [MLOP-636] Create migration classes ([#282](https://github.com/quintoandar/butterfree/pull/282))
+* [MLOP-635] Rebase Incremental Job/Interval Run branch for test on selected feature sets ([#278](https://github.com/quintoandar/butterfree/pull/278))
+* Allow slide selection ([#293](https://github.com/quintoandar/butterfree/pull/293))
+* [MLOP-637] Implement diff method ([#292](https://github.com/quintoandar/butterfree/pull/292))
+* [MLOP-640] Create CLI with migrate command ([#298](https://github.com/quintoandar/butterfree/pull/298))
+* [MLOP-645] Implement query method, cassandra ([#291](https://github.com/quintoandar/butterfree/pull/291))
+* [MLOP-671] Implement get_schema on Spark client ([#301](https://github.com/quintoandar/butterfree/pull/301))
+* [MLOP-648] Implement query method, metastore ([#294](https://github.com/quintoandar/butterfree/pull/294))
+* [MLOP-647] / [MLOP-646] Apply migrations ([#300](https://github.com/quintoandar/butterfree/pull/300))
+* [MLOP-639] Track logs in S3 ([#306](https://github.com/quintoandar/butterfree/pull/306))
+* [MLOP-702] Debug mode for Automate Migration ([#322](https://github.com/quintoandar/butterfree/pull/322))
+
+### Changed
+* Keep milliseconds when using 'from_ms' argument in timestamp feature ([#284](https://github.com/quintoandar/butterfree/pull/284))
+* Read and write consistency level options ([#309](https://github.com/quintoandar/butterfree/pull/309))
+* [MLOP-691] Include step to add partition to SparkMetastore during writing of Butterfree ([#327](https://github.com/quintoandar/butterfree/pull/327))
+
+### Fixed
+* [BUG] Apply create_partitions to historical validate ([#303](https://github.com/quintoandar/butterfree/pull/303))
+* [BUG] Fix key path for validate read ([#304](https://github.com/quintoandar/butterfree/pull/304))
+* [FIX] Add Partition types for Metastore ([#305](https://github.com/quintoandar/butterfree/pull/305))
+* Change solution for tracking logs ([#308](https://github.com/quintoandar/butterfree/pull/308))
+* [BUG] Fix Cassandra Connect Session ([#316](https://github.com/quintoandar/butterfree/pull/316))
+* Fix method to generate agg feature name. ([#326](https://github.com/quintoandar/butterfree/pull/326))
+
+## [1.1.3](https://github.com/quintoandar/butterfree/releases/tag/1.1.3)
+### Added
+* [MLOP-599] Apply mypy to ButterFree ([#273](https://github.com/quintoandar/butterfree/pull/273))
+
+### Changed
+* [MLOP-634] Butterfree dev workflow, set triggers for branches staging and master ([#280](https://github.com/quintoandar/butterfree/pull/280))
+* Keep milliseconds when using 'from_ms' argument in timestamp feature ([#284](https://github.com/quintoandar/butterfree/pull/284))
+* [MLOP-633] Butterfree dev workflow, update documentation ([#281](https://github.com/quintoandar/butterfree/commit/74278986a49f1825beee0fd8df65a585764e5524))
+* [MLOP-632] Butterfree dev workflow, automate release description ([#279](https://github.com/quintoandar/butterfree/commit/245eaa594846166972241b03fddc61ee5117b1f7))
+
+### Fixed
+* Change trigger for pipeline staging ([#287](https://github.com/quintoandar/butterfree/pull/287))
+
+## [1.1.2](https://github.com/quintoandar/butterfree/releases/tag/1.1.2)
+### Fixed
+* [HOTFIX] Add both cache and count back to Butterfree ([#274](https://github.com/quintoandar/butterfree/pull/274))
+* [MLOP-606] Change docker image in Github Actions Pipeline ([#275](https://github.com/quintoandar/butterfree/pull/275))
+* FIX Read the Docs build ([#272](https://github.com/quintoandar/butterfree/pull/272))
+* [BUG] Fix style ([#271](https://github.com/quintoandar/butterfree/pull/271))
+* [MLOP-594] Remove from_column in some transforms ([#270](https://github.com/quintoandar/butterfree/pull/270))
+* [MLOP-536] Rename S3 config to Metastore config ([#269](https://github.com/quintoandar/butterfree/pull/269))
+
 ## [1.1.1](https://github.com/quintoandar/butterfree/releases/tag/1.2.0)
 ### Added
 * [MLOP-590] Adapt KafkaConfig to receive a custom topic name ([#266](https://github.com/quintoandar/butterfree/pull/266))
@@ -15,11 +117,11 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each
 * Update README ([#257](https://github.com/quintoandar/butterfree/pull/257))
 
 ### Fixed
-* Fix Butterfree's workflow ([#262](https://github.com/quintoandar/butterfree/pull/262)) 
+* Fix Butterfree's workflow ([#262](https://github.com/quintoandar/butterfree/pull/262))
 * [FIX] Downgrade Python Version in Pyenv ([#227](https://github.com/quintoandar/butterfree/pull/227))
-* [FIX] Fix docs ([#229](https://github.com/quintoandar/butterfree/pull/229)) 
+* [FIX] Fix docs ([#229](https://github.com/quintoandar/butterfree/pull/229))
 * [FIX] Fix Docs - Add more dependencies ([#230](https://github.com/quintoandar/butterfree/pull/230))
-* Fix broken notebook URL ([#236](https://github.com/quintoandar/butterfree/pull/236)) 
+* Fix broken notebook URL ([#236](https://github.com/quintoandar/butterfree/pull/236))
 * Issue #77 Fix ([#245](https://github.com/quintoandar/butterfree/pull/245))
 
 ## [1.0.2](https://github.com/quintoandar/butterfree/releases/tag/1.0.2)
@@ -30,7 +132,7 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each
 * [MLOP-426] Change branching strategy on butterfree to use only master branch ([#216](https://github.com/quintoandar/butterfree/pull/216))
 
 ### Fixed
-* [MLOP-440] Python 3.7 bump and Fixing Dependencies ([#220](https://github.com/quintoandar/butterfree/pull/220)) 
+* [MLOP-440] Python 3.7 bump and Fixing Dependencies ([#220](https://github.com/quintoandar/butterfree/pull/220))
 
 ## [1.0.1](https://github.com/quintoandar/butterfree/releases/tag/1.0.1)
 ### Added
@@ -229,4 +331,4 @@ Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each
 * [MLOP-143] Fix Bugs for HouseMain FeatureSet ([#62](https://github.com/quintoandar/butterfree/pull/62))
 
 ## [0.1.0](https://github.com/quintoandar/butterfree/releases/tag/0.1.0)
-* First modules and entities of butterfree package.
+* First modules and entities of butterfree package.

Makefile

@@ -1,8 +1,16 @@
+# globals
+
+PACKAGE_NAME := $(shell grep __package_name__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2)
+VERSION := $(shell grep __version__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2)
+
+
+#custom targets
+
 .PHONY: environment
 ## create virtual environment for butterfree
 environment:
-	@pyenv install -s 3.7.6
-	@pyenv virtualenv 3.7.6 butterfree
+	@pyenv install -s 3.9.19
+	@pyenv virtualenv 3.9.19 butterfree
 	@pyenv local butterfree
 	@PYTHONPATH=. python -m pip install --upgrade pip
 
@@ -28,7 +36,7 @@ minimum-requirements:
 
 .PHONY: requirements
 ## install all requirements
-requirements: requirements-test requirements-lint dev-requirements minimum-requirements
+requirements: minimum-requirements dev-requirements requirements-test requirements-lint
 
 .PHONY: ci-install
 ci-install:
@@ -68,7 +76,7 @@ style-check:
 	@echo "Code Style"
 	@echo "=========="
 	@echo ""
-	@python -m black --check -t py36 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" . && echo "\n\nSuccess" || (echo "\n\nFailure\n\nYou need to run \"make apply-style\" to apply style formatting to your code"; exit 1)
+	@python -m black --check -t py39 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" . && echo "\n\nSuccess" || (echo "\n\nFailure\n\nYou need to run \"make apply-style\" to apply style formatting to your code"; exit 1)
 
 .PHONY: quality-check
 ## run code quality checks with flake8
@@ -96,8 +104,8 @@ checks: style-check quality-check type-check
 .PHONY: apply-style
 ## fix stylistic errors with black
 apply-style:
-	@python -m black -t py36 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" .
-	@python -m isort -rc butterfree/ tests/
+	@python -m black -t py39 --exclude="build/|buck-out/|dist/|_build/|pip/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" .
+	@python -m isort --atomic butterfree/ tests/
 
 .PHONY: clean
 ## clean unused artifacts
@@ -114,32 +122,38 @@ clean:
 	@find ./ -type f -name 'coverage.xml' -exec rm -f {} \;
 	@find ./ -type f -name '.coverage*' -exec rm -f {} \;
 	@find ./ -type f -name '*derby.log' -exec rm -f {} \;
+	@find ./ -type f -name 'logging.json' -exec rm -f {} \;
 	@find ./ -name '*.pyc' -exec rm -f {} \;
 	@find ./ -name '*.pyo' -exec rm -f {} \;
 	@find ./ -name '*~' -exec rm -f {} \;
 
 .PHONY: version
-## dump package name into VERSION env variable and show
+## show version
 version:
-	@export VERSION=$(grep __version__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2)
-	@$(info VERSION is [${VERSION}])
+	@echo "VERSION: $(VERSION)"
+
+.PHONY: change-version
+## change the version to string received in the NEW_VERSION variable and show
+change-version:
+	@sed -i 's/$(VERSION)/$(NEW_VERSION)/g' setup.py
+	@echo "VERSION: $(NEW_VERSION)"
 
 .PHONY: package-name
-## dump package name into PACKAGE_NAME env variable and show
+## show package name
 package-name:
-	@PACKAGE_NAME=$(grep __package_name__ setup.py | head -1 | cut -d \" -f2 | cut -d \' -f2 | sed 's/.*/&${build}/')
-	@echo $PACKAGE_NAME
+	@echo "PACKAGE_NAME: $(PACKAGE_NAME)"
 
 .PHONY: package
 ## build butterfree package wheel
 package:
+	@PYTHONPATH=. pip3 install wheel
 	@PYTHONPATH=. python -m setup sdist bdist_wheel
 
 .PHONY: update-docs
 ## update Butterfree API docs
 update-docs:
 	cd ./docs; rm -rf source/butterfree.*
-	cd ./docs; sphinx-apidoc -T -E -o source/ ../butterfree
+	cd ./docs; sphinx-apidoc -o source/ ../butterfree
 	cd ./docs; make coverage
 
 .PHONY: docs
@@ -208,4 +222,4 @@ help:
 		} \
 		printf "\n"; \
 	}' \
-	| more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')
+	| more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')