databricks.yml

# This is a Databricks asset bundle definition for invaas.
# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
bundle:
  name: invaas
sync:
  include:
    - "./setup.py"
variables:
  email:
    default: daneisburgh@gmail.com
  schedule_pause_status:
    default: "PAUSED"
  root_path:
    default: "/Shared/.bundle/prod/invaas"
common_settings: &common_settings
  timeout_seconds: 900
  max_concurrent_runs: 1
  email_notifications:
    on_failure:
      - ${var.email}
  job_clusters:
    - job_cluster_key: job_cluster
      new_cluster:
        spark_version: 13.3.x-scala2.12
        node_type_id: Standard_F4s
        data_security_mode: LEGACY_SINGLE_USER_STANDARD
        spark_conf:
          spark.databricks.cluster.profile: singleNode
          spark.master: "local[*,4]"
        spark_env_vars:
          APP_ENV: ${bundle.target}
          APP_FILES_PATH: /Workspace${var.root_path}/files
        custom_tags:
          ResourceClass: SingleNode
        azure_attributes:
          availability: ON_DEMAND_AZURE
          first_on_demand: 1
          spot_bid_max_price: -1
resources:
  jobs:
    coinbase:
      name: ${bundle.name}-coinbase
      <<:
        - *common_settings
      schedule:
        quartz_cron_expression: "0 0 0 * * ?"
        timezone_id: UTC
        # pause_status: ${var.schedule_pause_status}
        pause_status: "PAUSED"
      tasks:
        - task_key: coinbase
          job_cluster_key: job_cluster
          notebook_task:
            notebook_path: "invaas/coinbase/coinbase_notebook.ipynb"
            base_parameters:
              job_id: "{{job.id}}"
    schwab:
      name: ${bundle.name}-schwab
      <<:
        - *common_settings
      schedule:
        quartz_cron_expression: "0 30 9 ? * MON,TUE,WED,THU,FRI *"
        timezone_id: America/New_York
        # pause_status: ${var.schedule_pause_status}
        pause_status: "PAUSED"
      tasks:
        - task_key: schwab
          job_cluster_key: job_cluster
          notebook_task:
            notebook_path: "invaas/schwab/schwab_notebook.ipynb"
            base_parameters:
              job_id: "{{job.id}}"
targets:
  dev:
    mode: development
    default: true
    run_as:
      user_name: ${var.email}
    variables:
      schedule_pause_status: "PAUSED"
      root_path: "/Shared/.bundle/dev/${var.email}/invaas"
    workspace:
      host: https://adb-6718246255739942.2.azuredatabricks.net
      root_path: ${var.root_path}
  prod:
    mode: production
    run_as:
      user_name: ${var.email}
    workspace:
      host: https://adb-6718246255739942.2.azuredatabricks.net
      root_path: ${var.root_path}