Skip to content
LSTM Pipeline

LSTM Pipeline #261

Sign in to view logs

LSTM Pipeline

LSTM Pipeline #261

Workflow file for this run

.github/workflows/lstm_pipeline.yaml at beb4fc3
name: LSTM Pipeline
on:
push:
branches:
- main
tags:
- '*'
schedule:
- cron: "0 9 * * *"
# 16 - 7 = 9
jobs:
scraping_extracting:
name: Scraping and Extracting
runs-on: ubuntu-latest
environment: Scraping and Extraction
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
lfs: true
persist-credentials: false
fetch-depth: 1
- name: Scraping Yahoo Finance
run: |
mkdir datasets
go run scraper.go \
--symbols-file=./postman/symbols.json
- name: Zip Datasets
run: zip -r datasets.zip datasets
- name: Store Datasets to Google Drive
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: datasets.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: datasets.zip
overwrite: "true"
- name: Upload Artifact (datasets)
uses: actions/upload-artifact@v3
with:
name: datasets
path: datasets.zip
- name: Remove Temporarary Files and Directories
run: |
rm datasets.zip
rm -rf datasets
preprocessing_training:
name: Preprocessing and Training
runs-on: ubuntu-latest
needs: scraping_extracting
environment: Preprocessing and Training
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
lfs: true
persist-credentials: false
fetch-depth: 1
- name: Install Libraries
run: pip install -r dev.requirements.txt
- name: Download Artifact (datasets)
uses: actions/download-artifact@v3
with:
name: datasets
- name: Install Cython and Build ML Module
run: |
sudo apt-get update && \
sudo apt-get install -y \
gcc python3-dev gnupg
pip install cython
cd training && \
python setup.py build_ext --inplace && cd ..
- name: Modeling and Training
run: |
unzip datasets.zip
mkdir models
mkdir pickles
mkdir posttrained
python trainingcli.py \
--epochs=200 \
--batchs=32 \
--sequences=60 \
--algorithm=LSTM
- name: Zip Posttrained, Models, and Pickles
run: |
zip -r models.zip models
zip -r pickles.zip pickles
zip -r posttrained.zip posttrained
- name: Store Models to Google Drive
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: models.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: models.zip
overwrite: "true"
- name: Store Pickles to Google Drive
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: pickles.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: pickles.zip
overwrite: "true"
- name: Store Posttrained to Google Drive
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: posttrained.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: posttrained.zip
overwrite: "true"
- name: Remove Temporarary Files and Directories
run: |
rm models.zip
rm pickles.zip
rm datasets.zip
rm posttrained.zip
rm -rf models
rm -rf pickles
rm -rf datasets
rm -rf posttrained
tebakaja_crypto_space-0:
name: crypto-forecast-svc-0
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-0
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "alfariqyraihan@gmail.com"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-1:
name: crypto-forecast-svc-1
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-1
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "alfariqyraihan@gmail.com"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-2:
name: crypto-forecast-svc-2
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-2
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "alfariqyraihan@gmail.com"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-3:
name: crypto-forecast-svc-3
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-3
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "alfariqyraihan@gmail.com"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-4:
name: crypto-forecast-svc-4
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-4
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "alfariqyraihan@gmail.com"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME main --force
functional_testing:
name: Functional Testing
runs-on: ubuntu-latest
needs:
- tebakaja_crypto_space-0
- tebakaja_crypto_space-1
- tebakaja_crypto_space-2
- tebakaja_crypto_space-3
- tebakaja_crypto_space-4
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1
- name: Testing Proxy Endpoints
run: |
sleep 60
chmod +x endpoints_test.sh && ./endpoints_test.sh