From e39bdf3f06a934f5e875d88f3e133b0da5e1b7e8 Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Sat, 24 Feb 2024 18:22:40 +0530 Subject: [PATCH] first commit --- .github/workflows/deploy.yaml | 32 ++++++++++++++++++++++++++++++ README.md | 9 +++++++++ resources/Deployment/deployment.sh | 19 ++++++++++++++++++ resources/Storage/apcha.csv | 9 +++++++++ 4 files changed, 69 insertions(+) create mode 100644 .github/workflows/deploy.yaml create mode 100644 README.md create mode 100644 resources/Deployment/deployment.sh create mode 100644 resources/Storage/apcha.csv diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..55f6e96 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,32 @@ +name: Deploy to GCS + +on: + push: + branches: + - main + +jobs: + deploy: + runs-on: ubuntu-latest + + strategy: + fail-fast: true + + env: + GCP_AUTH: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_SECRET: ${{ secrets.GCS_SECRET }} + GCS_SECRET: ${{ secrets.GCS_SECRET }} + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Google Cloud SDK + uses: google-github-actions/setup-gcloud@v0.3.0 + with: + service_account_key: $GCP_AUTH + project_id: + export_default_credentials: true + + - name: Execute deployment script + run: bash deployment.sh $GCP_PROJECT_SECRET $GCS_SECRET diff --git a/README.md b/README.md new file mode 100644 index 0000000..0b28aa7 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +# Sensex-Data-Analysis + +[![Automated Data Extraction and Loading](https://github.com/AbhishekSingh1180/sensex-data-analysis/actions/workflows/main.yml/badge.svg)](https://github.com/AbhishekSingh1180/sensex-data-analysis/actions/workflows/main.yml) + +[![Deploy DDL SQL with SnowSQL](https://github.com/AbhishekSingh1180/sensex-data-analysis/actions/workflows/deploy.yml/badge.svg)](https://github.com/AbhishekSingh1180/sensex-data-analysis/actions/workflows/deploy.yml) + +![Diagram](https://github.com/AbhishekSingh1180/sensex-data-analysis/blob/main/Diagram/sensex-data-analysis.png) + +Leveraged GitHub Actions for streamlined data extraction from Yahoo finance API, Snowflake and SnowSQL for extraction, storage, and transformation for Sensex Data. diff --git a/resources/Deployment/deployment.sh b/resources/Deployment/deployment.sh new file mode 100644 index 0000000..d650e11 --- /dev/null +++ b/resources/Deployment/deployment.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +$FOLDER_PATH = 'resources/Storage/apcha.csv' + +# Set project ID and region from Project Secrets +read -r PROJECT_NAME REGION <<< $(echo "$1" | tr ',' ' ') + +# Set default project and region +gcloud config set project $PROJECT_NAME +gcloud config set compute/region $REGION + +# Set GCS Variables from GCS Secrets +read -r BUCKET_NAME GCS_FOLDER <<< $(echo "$2" | tr ',' ' ') + +# Create GCS bucket +gsutil mb gs://$BUCKET_NAME + +# cp local folder to GCS bucket +gsutil -m cp -r $FOLDER_PATH gs://$BUCKET_NAME/$GCS_FOLDER \ No newline at end of file diff --git a/resources/Storage/apcha.csv b/resources/Storage/apcha.csv new file mode 100644 index 0000000..cca613b --- /dev/null +++ b/resources/Storage/apcha.csv @@ -0,0 +1,9 @@ +"index","symbol","timestamp","open","close","high","low","volume" +"Sensex","^BSESN","2024-02-21T03:45:00Z","73219.11","73101.91","73230.75","73029.81","0" +"Sensex","^BSESN","2024-02-21T04:45:00Z","73103.56","73091.73","73155.62","72930.95","0" +"Sensex","^BSESN","2024-02-21T05:45:00Z","73093.81","73098.48","73135.98","72997.32","0" +"Sensex","^BSESN","2024-02-21T06:45:00Z","73098.99","73139.31","73193.04","73080.79","0" +"Sensex","^BSESN","2024-02-21T07:45:00Z","73139.09","72793.88","73143.08","72760.83","0" +"Sensex","^BSESN","2024-02-21T08:45:00Z","72791.60","72764.45","72876.99","72450.74","0" +"Sensex","^BSESN","2024-02-21T09:45:00Z","72759.91","72623.09","72759.91","72623.09","0" +"Sensex","^BSESN","2024-02-21T10:00:00Z","72623.09","72623.09","72623.09","72623.09","0" \ No newline at end of file