From 708258013c67d090e16ad5266b2c2c984ecf6636 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Thu, 31 Oct 2024 14:29:02 -0700 Subject: [PATCH 01/23] Added modified ETL logic for BCTS transformation for annual developed volume report --- .../BCTS_ACCESS_CATALOG.csv | 31 ++++++ shared/bcts_access_management/Dockerfile | 15 +++ shared/bcts_access_management/README.md | 2 + .../bcts_acces_apply_grants.py | 86 ++++++++++++++++ .../bcts_access_export_master_file.py | 54 ++++++++++ .../lrm_local_secrets.yaml | 18 ++++ .../bcts_access_management/requirements.txt | 2 + .../Annual_Developed_Volume_Query.sql | 98 ++++++++++++++++--- .../Dockerfile | 0 .../bcts_etl.py | 0 .../bcts_reports_etl/lrm_local_secrets.yaml | 18 ++++ 11 files changed, 308 insertions(+), 16 deletions(-) create mode 100644 shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv create mode 100644 shared/bcts_access_management/Dockerfile create mode 100644 shared/bcts_access_management/README.md create mode 100644 shared/bcts_access_management/bcts_acces_apply_grants.py create mode 100644 shared/bcts_access_management/bcts_access_export_master_file.py create mode 100644 shared/bcts_access_management/lrm_local_secrets.yaml create mode 100644 shared/bcts_access_management/requirements.txt rename shared/{pg-transformations => bcts_reports_etl}/Annual_Developed_Volume_Query.sql (62%) rename shared/{pg-transformations => bcts_reports_etl}/Dockerfile (100%) rename shared/{pg-transformations => bcts_reports_etl}/bcts_etl.py (100%) create mode 100644 shared/bcts_reports_etl/lrm_local_secrets.yaml diff --git a/shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv b/shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv new file mode 100644 index 0000000..b6f6fb3 --- /dev/null +++ b/shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv @@ -0,0 +1,31 @@ +KEY,SCHEMA,TABLE_NAME,AUDIT_FOR_PI,BCTS_ETL_ROLE,BCTS_DEV_ROLE,BCTS_STAGE_ANALYST_ROLE,BCTS_STAGE_ANALYST_PI_ROLE,BCTS_ANALYST_ROLE,BCTS_ANALYST_PI_ROLE +1,LRM_REPLICATION,DIVISION,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +2,LRM_REPLICATION,BLOCK_ALLOCATION,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +3,LRM_REPLICATION,MANAGEMENT_UNIT,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +4,LRM_REPLICATION,LICENCE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +5,LRM_REPLICATION,BLOCK_ADMIN_ZONE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +6,LRM_REPLICATION,DIVISION_CODE_LOOKUP,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +7,LRM_REPLICATION,CODE_LOOKUP,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +8,LRM_REPLICATION,TENURE_TYPE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +9,LRM_REPLICATION,CUT_PERMIT,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +10,LRM_REPLICATION,MARK,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +11,LRM_REPLICATION,CUT_BLOCK,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +12,LRM_REPLICATION,ACTIVITY_CLASS,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +13,LRM_REPLICATION,ACTIVITY_TYPE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +14,LRM_REPLICATION,ACTIVITY,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny +15,BCTS_STAGING,DIVISION,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +16,BCTS_STAGING,BLOCK_ALLOCATION,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +17,BCTS_STAGING,MANAGEMENT_UNIT,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +18,BCTS_STAGING,LICENCE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +19,BCTS_STAGING,BLOCK_ADMIN_ZONE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +20,BCTS_STAGING,DIVISION_CODE_LOOKUP,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +21,BCTS_STAGING,CODE_LOOKUP,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +22,BCTS_STAGING,TENURE_TYPE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +23,BCTS_STAGING,CUT_PERMIT,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +24,BCTS_STAGING,MARK,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +25,BCTS_STAGING,CUT_BLOCK,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +26,BCTS_STAGING,ACTIVITY_CLASS,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +27,BCTS_STAGING,ACTIVITY_TYPE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +28,BCTS_STAGING,ACTIVITY,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +29,BCTS_STAGING,ANNUAL_DEVELOPED_VOLUME,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny +30,BCTS_REPORTING,ANNUAL_DEVELOPED_VOLUME,No PI,Read/Write,Read/Write,Deny,Deny,Read,Deny diff --git a/shared/bcts_access_management/Dockerfile b/shared/bcts_access_management/Dockerfile new file mode 100644 index 0000000..fbc20f6 --- /dev/null +++ b/shared/bcts_access_management/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11.4-slim-buster + +RUN apt-get update \ + && apt-get -y install libpq-dev gcc \ + && pip install psycopg2 + +WORKDIR /app + +COPY requirements.txt . + +# Install the required Python packages +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + diff --git a/shared/bcts_access_management/README.md b/shared/bcts_access_management/README.md new file mode 100644 index 0000000..e724795 --- /dev/null +++ b/shared/bcts_access_management/README.md @@ -0,0 +1,2 @@ +- This package is used to implement grant management for BCTS data in ODS. The DAG `export-grants-BCTS` is used to read the master access management file, maintained by the BCTS service team, and export the contents to `master_access` table in `LRM_REPLICATION` schema in ODS. The DAG `apply-grants-BCTS` is used to read the various grants assigned to various roles in ODS and apply these grants. +- THe master access management file defines which role has read/ write access to what tables and other objects in the BCTS owned schemas in ODS; LRM_REPLICATION, BCTS_STAGING, and BCTS_REPORTING. diff --git a/shared/bcts_access_management/bcts_acces_apply_grants.py b/shared/bcts_access_management/bcts_acces_apply_grants.py new file mode 100644 index 0000000..5e3574b --- /dev/null +++ b/shared/bcts_access_management/bcts_acces_apply_grants.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# coding: utf-8 + +#Imports +import os, time +import logging +import sys +import pandas as pd +from sqlalchemy import create_engine, text + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler() + ] +) + +# Retrieve Postgres database configuration +postgres_username = os.environ['ODS_USERNAME'] +postgres_password = os.environ['ODS_PASSWORD'] +postgres_host = os.environ['ODS_HOST'] +postgres_port = os.environ['ODS_PORT'] +postgres_database = os.environ['ODS_DATABASE'] + + +# Database connection details +DATABASE_URI = f'postgresql+psycopg2://{postgres_username}:{postgres_password}@{postgres_host}:{postgres_port}/{postgres_database}' + +def apply_grants(): + # Connect to the database + engine = create_engine(DATABASE_URI) + + # Query the master_access table and load data into a DataFrame + query = f"SELECT * FROM lrm_replication.master_access" + df = pd.read_sql(query, engine) + + # Generate all grant statements for each row in the DataFrame + all_statements = [] + for _, row in df.iterrows(): + all_statements.extend(generate_grant_statements(row)) + + # Execute the grant statements in PostgreSQL + with engine.begin() as connection: + for statement in all_statements: + logging.info(statement) + connection.execute(text(statement)) + + logging.info("Grant statements executed successfully.") + +# Function to generate grant statements based on role permissions +def generate_grant_statements(row): + statements = [] + schema = row['SCHEMA'] + table_name = row['TABLE_NAME'] + + # List of roles and their permissions in the DataFrame + role_permissions = { + 'BCTS_ETL_ROLE': row['BCTS_ETL_ROLE'], + 'BCTS_DEV_ROLE': row['BCTS_DEV_ROLE'], + 'BCTS_STAGE_ANALYST_ROLE': row['BCTS_STAGE_ANALYST_ROLE'], + 'BCTS_STAGE_ANALYST_PI_ROLE': row['BCTS_STAGE_ANALYST_PI_ROLE'], + 'BCTS_ANALYST_ROLE': row['BCTS_ANALYST_ROLE'], + 'BCTS_ANALYST_PI_ROLE': row['BCTS_ANALYST_PI_ROLE'] + } + + # Generate grant statements based on role permissions + for role, permission in role_permissions.items(): + if permission == "Read": + statements.append(f"GRANT SELECT ON {schema}.{table_name} TO {role};") + elif permission == "Read/Write": + statements.append(f"GRANT SELECT, INSERT, UPDATE, DELETE ON {schema}.{table_name} TO {role};") + # Ignore 'Deny' permissions + + return statements + + +if __name__ == "__main__": + apply_grants() + + + + + + diff --git a/shared/bcts_access_management/bcts_access_export_master_file.py b/shared/bcts_access_management/bcts_access_export_master_file.py new file mode 100644 index 0000000..356f025 --- /dev/null +++ b/shared/bcts_access_management/bcts_access_export_master_file.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# coding: utf-8 + +#Imports +import os, time +import logging +import pandas as pd +from sqlalchemy import create_engine + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler() + ] +) + +# Retrieve Postgres database configuration +postgres_username = os.environ['ODS_USERNAME'] +postgres_password = os.environ['ODS_PASSWORD'] +postgres_host = os.environ['ODS_HOST'] +postgres_port = os.environ['ODS_PORT'] +postgres_database = os.environ['ODS_DATABASE'] + + +MASTER_FILE_PATH = "./BCTS_ACCESS_CATALOG.csv" + + +# Database connection details +DATABASE_URI = f'postgresql+psycopg2://{postgres_username}:{postgres_password}@{postgres_host}:{postgres_port}/{postgres_database}' + + +def export_access_catalog(): + # Create SQLAlchemy engine + engine = create_engine(DATABASE_URI) + + # Read the master grant file into a pandas DataFrame + df = pd.read_csv(MASTER_FILE_PATH) + + # Write DataFrame to the target table in PostgreSQL, replacing the table if it exists + df.to_sql( + 'master_access', # Target table name + engine, + schema='lrm_replication', + if_exists='replace', + index=False # Do not write DataFrame index as a column + ) + + logging.info("Data has been successfully written to the master_access table in PostgreSQL.") + +if __name__ == "__main__": + export_access_catalog() + \ No newline at end of file diff --git a/shared/bcts_access_management/lrm_local_secrets.yaml b/shared/bcts_access_management/lrm_local_secrets.yaml new file mode 100644 index 0000000..c74cbbb --- /dev/null +++ b/shared/bcts_access_management/lrm_local_secrets.yaml @@ -0,0 +1,18 @@ +# These secrets are used in the LOCAL evelopment environment. For the DAP environments, these secrets should be created in Openshift. +apiVersion: v1 +kind: Secret +metadata: + name: lrm-ods-database +type: Opaque +stringData: + ODS_USERNAME: ods_proxy_user_lrm + ODS_PASSWORD: ods_proxy_user_lrm + ODS_HOST: 172.28.0.10 + ODS_PORT: '5432' + ODS_DATABASE: ods + MSTR_SCHEMA: ods_data_management + APP_NAME: lrm + CONCUR_TASKS: '50' + + + \ No newline at end of file diff --git a/shared/bcts_access_management/requirements.txt b/shared/bcts_access_management/requirements.txt new file mode 100644 index 0000000..5803cd6 --- /dev/null +++ b/shared/bcts_access_management/requirements.txt @@ -0,0 +1,2 @@ +pandas==2.2.3 +SQLAlchemy==2.0.36 diff --git a/shared/pg-transformations/Annual_Developed_Volume_Query.sql b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql similarity index 62% rename from shared/pg-transformations/Annual_Developed_Volume_Query.sql rename to shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql index a6a894f..ba6534b 100644 --- a/shared/pg-transformations/Annual_Developed_Volume_Query.sql +++ b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql @@ -1,6 +1,67 @@ -DROP TABLE IF EXISTS bcts_reporting.annual_developed_volume; +DROP TABLE IF EXISTS bcts_staging.DIVISION; +CREATE TABLE bcts_staging.DIVISION AS + SELECT * FROM lrm_replication.DIVISION; -CREATE TABLE bcts_reporting.annual_developed_volume AS +DROP TABLE IF EXISTS bcts_staging.BLOCK_ALLOCATION; +CREATE TABLE bcts_staging.BLOCK_ALLOCATION AS + SELECT * FROM lrm_replication.BLOCK_ALLOCATION; + +DROP TABLE IF EXISTS bcts_staging.MANAGEMENT_UNIT; +CREATE TABLE bcts_staging.MANAGEMENT_UNIT AS + SELECT * FROM lrm_replication.MANAGEMENT_UNIT; + +DROP TABLE IF EXISTS bcts_staging.LICENCE; +CREATE TABLE bcts_staging.LICENCE AS + SELECT * FROM lrm_replication.LICENCE; + +DROP TABLE IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; +CREATE TABLE bcts_staging.BLOCK_ADMIN_ZONE AS + SELECT * FROM lrm_replication.BLOCK_ADMIN_ZONE; + +DROP TABLE IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE TABLE bcts_staging.DIVISION_CODE_LOOKUP AS + SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; + +DROP TABLE IF EXISTS bcts_staging.CODE_LOOKUP; +CREATE TABLE bcts_staging.CODE_LOOKUP AS + SELECT * FROM lrm_replication.CODE_LOOKUP; + +DROP TABLE IF EXISTS bcts_staging.TENURE_TYPE; +CREATE TABLE bcts_staging.TENURE_TYPE AS + SELECT * FROM lrm_replication.TENURE_TYPE; + +DROP TABLE IF EXISTS bcts_staging.CUT_PERMIT; +CREATE TABLE bcts_staging.CUT_PERMIT AS + SELECT * FROM lrm_replication.CUT_PERMIT; + +DROP TABLE IF EXISTS bcts_staging.MARK; +CREATE TABLE bcts_staging.MARK AS + SELECT * FROM lrm_replication.MARK; + +DROP TABLE IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE TABLE bcts_staging.DIVISION_CODE_LOOKUP AS + SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; + +DROP TABLE IF EXISTS bcts_staging.CUT_BLOCK; +CREATE TABLE bcts_staging.CUT_BLOCK AS + SELECT * FROM lrm_replication.CUT_BLOCK; + +DROP TABLE IF EXISTS bcts_staging.ACTIVITY_CLASS; +CREATE TABLE bcts_staging.ACTIVITY_CLASS AS + SELECT * FROM lrm_replication.ACTIVITY_CLASS; + +DROP TABLE IF EXISTS bcts_staging.ACTIVITY_TYPE; +CREATE TABLE bcts_staging.ACTIVITY_TYPE AS + SELECT * FROM lrm_replication.ACTIVITY_TYPE; + +DROP TABLE IF EXISTS bcts_staging.ACTIVITY; +CREATE TABLE bcts_staging.ACTIVITY AS + SELECT * FROM lrm_replication.ACTIVITY; + + +DROP TABLE IF EXISTS bcts_staging.annual_developed_volume; + +CREATE TABLE bcts_staging.annual_developed_volume AS WITH annual_developed_volume AS ( SELECT DISTINCT @@ -48,31 +109,31 @@ WITH annual_developed_volume AS b.cutb_seq_nbr FROM lrm_replication.division d - INNER JOIN lrm_replication.block_allocation ba + INNER JOIN bcts_staging.block_allocation ba ON d.divi_div_nbr = ba.divi_div_nbr - INNER JOIN lrm_replication.management_unit mu + INNER JOIN bcts_staging.management_unit mu ON ba.manu_seq_nbr = mu.manu_seq_nbr - INNER JOIN lrm_replication.licence l + INNER JOIN bcts_staging.licence l ON ba.licn_seq_nbr = l.licn_seq_nbr - LEFT OUTER JOIN lrm_replication.block_admin_zone z + LEFT OUTER JOIN bcts_staging.block_admin_zone z ON l.divi_div_nbr = z.divi_div_nbr AND l.blaz_admin_zone_id = z.blaz_admin_zone_id AND ba.licn_seq_nbr = l.licn_seq_nbr AND l.divi_div_nbr = z.divi_div_nbr AND l.blaz_admin_zone_id = z.blaz_admin_zone_id - LEFT OUTER JOIN lrm_replication.division_code_lookup dcl + LEFT OUTER JOIN bcts_staging.division_code_lookup dcl ON l.licn_field_team_id = dcl.colu_lookup_id AND l.divi_div_nbr = dcl.divi_div_nbr - LEFT OUTER JOIN lrm_replication.code_lookup cl + LEFT OUTER JOIN bcts_staging.code_lookup cl ON dcl.colu_lookup_type = cl.colu_lookup_type AND dcl.colu_lookup_id = cl.colu_lookup_id - LEFT JOIN lrm_replication.tenure_type tn + LEFT JOIN bcts_staging.tenure_type tn ON l.tent_seq_nbr = tn.tent_seq_nbr - LEFT OUTER JOIN lrm_replication.cut_permit cp + LEFT OUTER JOIN bcts_staging.cut_permit cp ON ba.perm_seq_nbr = cp.perm_seq_nbr - LEFT JOIN lrm_replication.mark m + LEFT JOIN bcts_staging.mark m ON ba.mark_seq_nbr = m.mark_seq_nbr - INNER JOIN lrm_replication.cut_block b + INNER JOIN bcts_staging.cut_block b ON ba.cutb_seq_nbr = b.cutb_seq_nbr INNER JOIN ( @@ -83,9 +144,9 @@ WITH annual_developed_volume AS MAX(CASE WHEN ACTT_KEY_IND = 'DVS' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVS_Done, MAX(CASE WHEN ACTT_KEY_IND = 'DVC' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVC_Done FROM - lrm_replication.ACTIVITY_CLASS C, - lrm_replication.ACTIVITY_TYPE T, - lrm_replication.ACTIVITY A + bcts_staging.ACTIVITY_CLASS C, + bcts_staging.ACTIVITY_TYPE T, + bcts_staging.ACTIVITY A WHERE C.ACCL_SEQ_NBR = T.ACCL_SEQ_NBR AND T.ACTT_SEQ_NBR = A.ACTT_SEQ_NBR @@ -111,4 +172,9 @@ ORDER BY permit, mark, block -; \ No newline at end of file +; + +DROP TABLE IF EXISTS bcts_reporting.annual_developed_volume; + +CREATE TABLE bcts_reporting.annual_developed_volume AS + SELECT * FROM bcts_staging.annual_developed_volume; \ No newline at end of file diff --git a/shared/pg-transformations/Dockerfile b/shared/bcts_reports_etl/Dockerfile similarity index 100% rename from shared/pg-transformations/Dockerfile rename to shared/bcts_reports_etl/Dockerfile diff --git a/shared/pg-transformations/bcts_etl.py b/shared/bcts_reports_etl/bcts_etl.py similarity index 100% rename from shared/pg-transformations/bcts_etl.py rename to shared/bcts_reports_etl/bcts_etl.py diff --git a/shared/bcts_reports_etl/lrm_local_secrets.yaml b/shared/bcts_reports_etl/lrm_local_secrets.yaml new file mode 100644 index 0000000..c74cbbb --- /dev/null +++ b/shared/bcts_reports_etl/lrm_local_secrets.yaml @@ -0,0 +1,18 @@ +# These secrets are used in the LOCAL evelopment environment. For the DAP environments, these secrets should be created in Openshift. +apiVersion: v1 +kind: Secret +metadata: + name: lrm-ods-database +type: Opaque +stringData: + ODS_USERNAME: ods_proxy_user_lrm + ODS_PASSWORD: ods_proxy_user_lrm + ODS_HOST: 172.28.0.10 + ODS_PORT: '5432' + ODS_DATABASE: ods + MSTR_SCHEMA: ods_data_management + APP_NAME: lrm + CONCUR_TASKS: '50' + + + \ No newline at end of file From ffe2df1b4f553d1317afbf93acece2e3f11281cd Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Tue, 5 Nov 2024 10:02:39 -0800 Subject: [PATCH 02/23] Added DDL for initializing BCTS roles and schemas --- bcts/access_management/ddl/bcts_roles.sql | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bcts/access_management/ddl/bcts_roles.sql diff --git a/bcts/access_management/ddl/bcts_roles.sql b/bcts/access_management/ddl/bcts_roles.sql new file mode 100644 index 0000000..0d4df06 --- /dev/null +++ b/bcts/access_management/ddl/bcts_roles.sql @@ -0,0 +1,13 @@ +-- Create roles for ETL and data consumption +CREATE ROLE BCTS_ETL_ROLE; +CREATE ROLE BCTS_DEV_ROLE; +CREATE ROLE BCTS_DEV_ROLE; +CREATE ROLE BCTS_STAGE_ANALYST_ROLE; +CREATE ROLE BCTS_STAGE_ANALYST_PI_ROLE; +CREATE ROLE BCTS_ANALYST_ROLE; +CREATE ROLE BCTS_ANALYST_PI_ROLE; + +-- Create BCTS schemas in ODS +CREATE SCHEMA IF NOT EXISTS lrm_replication; +CREATE SCHEMA IF NOT EXISTS bcts_reporting; +CREATE SCHEMA IF NOT EXISTS bcts_staging; From 0c4c22577ae78bd7ec623fcc66ce40d0d43b1baf Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Tue, 5 Nov 2024 11:42:29 -0800 Subject: [PATCH 03/23] Added DDL for initializing BCTS roles and schemas --- bcts/access_management/ddl/bcts_roles.sql | 41 ++++++++++++++++++-- bcts/lrm/forest/ddl/activity.sql | 2 + bcts/lrm/forest/ddl/activity_class.sql | 2 + bcts/lrm/forest/ddl/activity_type.sql | 2 + bcts/lrm/forest/ddl/block_admin_zone.sql | 2 + bcts/lrm/forest/ddl/block_allocation.sql | 2 + bcts/lrm/forest/ddl/code_lookup.sql | 2 + bcts/lrm/forest/ddl/cut_block.sql | 3 ++ bcts/lrm/forest/ddl/cut_permit.sql | 2 + bcts/lrm/forest/ddl/division.sql | 2 + bcts/lrm/forest/ddl/division_code_lookup.sql | 2 + bcts/lrm/forest/ddl/licence.sql | 2 + bcts/lrm/forest/ddl/management_unit.sql | 2 + bcts/lrm/forest/ddl/mark.sql | 2 + bcts/lrm/forest/ddl/tenure_type.sql | 2 + 15 files changed, 66 insertions(+), 4 deletions(-) diff --git a/bcts/access_management/ddl/bcts_roles.sql b/bcts/access_management/ddl/bcts_roles.sql index 0d4df06..40cb580 100644 --- a/bcts/access_management/ddl/bcts_roles.sql +++ b/bcts/access_management/ddl/bcts_roles.sql @@ -1,3 +1,11 @@ +-- Create BCTS ETL executor role. Password should be available only to the DBA +-- 1. Create the etl superuser +CREATE USER bcts_etl_user WITH PASSWORD ''; + +CREATE SCHEMA lrm_replication; +CREATE SCHEMA bcts_staging; +CREATE SCHEMA bcts_reporting; + -- Create roles for ETL and data consumption CREATE ROLE BCTS_ETL_ROLE; CREATE ROLE BCTS_DEV_ROLE; @@ -7,7 +15,32 @@ CREATE ROLE BCTS_STAGE_ANALYST_PI_ROLE; CREATE ROLE BCTS_ANALYST_ROLE; CREATE ROLE BCTS_ANALYST_PI_ROLE; --- Create BCTS schemas in ODS -CREATE SCHEMA IF NOT EXISTS lrm_replication; -CREATE SCHEMA IF NOT EXISTS bcts_reporting; -CREATE SCHEMA IF NOT EXISTS bcts_staging; +-- 2. Grant access to schemas A, B, and C +GRANT USAGE ON SCHEMA lrm_replication TO bcts_etl_user; +GRANT USAGE ON SCHEMA bcts_staging TO bcts_etl_user; +GRANT USAGE ON SCHEMA bcts_reporting TO bcts_etl_user; + +-- 3. Grant read and write access to existing tables in schemas A, B, and C +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA lrm_replication TO bcts_etl_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_staging TO bcts_etl_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_reporting TO bcts_etl_user; + +-- 4. Grant permission to create new tables, functions, etc., in schemas A, B, and C +GRANT CREATE ON SCHEMA lrm_replication TO bcts_etl_user; +GRANT CREATE ON SCHEMA bcts_staging TO bcts_etl_user; +GRANT CREATE ON SCHEMA bcts_reporting TO bcts_etl_user; + +-- 5. Grant privileges to automatically apply on any new tables created in schemas A, B, and C +ALTER DEFAULT PRIVILEGES IN SCHEMA lrm_replication GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO bcts_etl_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA bcts_staging GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO bcts_etl_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA bcts_reporting GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO bcts_etl_user; + +-- Grant usage on sequences if needed for ID generation or other purposes +GRANT USAGE ON ALL SEQUENCES IN SCHEMA lrm_replication TO bcts_etl_user; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA bcts_staging TO bcts_etl_user; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA bcts_reporting TO bcts_etl_user; + +-- Set default privileges for sequences +ALTER DEFAULT PRIVILEGES IN SCHEMA lrm_replication GRANT USAGE ON SEQUENCES TO bcts_etl_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA bcts_staging GRANT USAGE ON SEQUENCES TO bcts_etl_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA bcts_reporting GRANT USAGE ON SEQUENCES TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/activity.sql b/bcts/lrm/forest/ddl/activity.sql index 8d1fcfa..3d4d5fb 100644 --- a/bcts/lrm/forest/ddl/activity.sql +++ b/bcts/lrm/forest/ddl/activity.sql @@ -33,3 +33,5 @@ CREATE TABLE lrm_replication.activity ( PRIMARY KEY (acti_seq_nbr) ); +ALTER TABLE lrm_replication.activity OWNER TO bcts_etl_user; + diff --git a/bcts/lrm/forest/ddl/activity_class.sql b/bcts/lrm/forest/ddl/activity_class.sql index 4b3be61..a4da5dc 100644 --- a/bcts/lrm/forest/ddl/activity_class.sql +++ b/bcts/lrm/forest/ddl/activity_class.sql @@ -13,3 +13,5 @@ CREATE TABLE lrm_replication.activity_class ( createdusing VARCHAR(120) NULL, PRIMARY KEY (accl_seq_nbr) ); + +ALTER TABLE lrm_replication.activity_class OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/activity_type.sql b/bcts/lrm/forest/ddl/activity_type.sql index ed62134..ed265a2 100644 --- a/bcts/lrm/forest/ddl/activity_type.sql +++ b/bcts/lrm/forest/ddl/activity_type.sql @@ -28,3 +28,5 @@ CREATE TABLE lrm_replication.activity_type ( createdusing VARCHAR(120) NULL, PRIMARY KEY (actt_seq_nbr) ); + +ALTER TABLE lrm_replication.activity_type OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/block_admin_zone.sql b/bcts/lrm/forest/ddl/block_admin_zone.sql index c232f9f..e10ac83 100644 --- a/bcts/lrm/forest/ddl/block_admin_zone.sql +++ b/bcts/lrm/forest/ddl/block_admin_zone.sql @@ -11,3 +11,5 @@ CREATE TABLE lrm_replication.block_admin_zone ( createdusing VARCHAR(120) NULL, PRIMARY KEY (blaz_admin_zone_id, divi_div_nbr) ); + +ALTER TABLE lrm_replication.block_admin_zone OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/block_allocation.sql b/bcts/lrm/forest/ddl/block_allocation.sql index 9931664..bf86fa1 100644 --- a/bcts/lrm/forest/ddl/block_allocation.sql +++ b/bcts/lrm/forest/ddl/block_allocation.sql @@ -29,3 +29,5 @@ CREATE TABLE lrm_replication.block_allocation ( blal_data_source VARCHAR(120) NULL, PRIMARY KEY (blal_seq_nbr) ); + +ALTER TABLE lrm_replication.block_allocation OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/code_lookup.sql b/bcts/lrm/forest/ddl/code_lookup.sql index 8fcbbf5..857039e 100644 --- a/bcts/lrm/forest/ddl/code_lookup.sql +++ b/bcts/lrm/forest/ddl/code_lookup.sql @@ -15,3 +15,5 @@ CREATE TABLE lrm_replication.code_lookup ( hq_display_ind VARCHAR(1) NULL, PRIMARY KEY (colu_lookup_type, colu_lookup_id) ); + +ALTER TABLE lrm_replication.code_lookup OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/cut_block.sql b/bcts/lrm/forest/ddl/cut_block.sql index cf3297a..2cd8bbd 100644 --- a/bcts/lrm/forest/ddl/cut_block.sql +++ b/bcts/lrm/forest/ddl/cut_block.sql @@ -129,3 +129,6 @@ CREATE TABLE lrm_replication.cut_block ( safety_concern VARCHAR(4000) NULL, PRIMARY KEY (cutb_seq_nbr) ); + + +ALTER TABLE lrm_replication.cut_block OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/cut_permit.sql b/bcts/lrm/forest/ddl/cut_permit.sql index d8e7b6f..9883652 100644 --- a/bcts/lrm/forest/ddl/cut_permit.sql +++ b/bcts/lrm/forest/ddl/cut_permit.sql @@ -68,3 +68,5 @@ CREATE TABLE lrm_replication.cut_permit ( createdusing VARCHAR(120) NULL, PRIMARY KEY (perm_seq_nbr) ); + +ALTER TABLE lrm_replication.cut_permit OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/division.sql b/bcts/lrm/forest/ddl/division.sql index d2349b3..5e9fa4a 100644 --- a/bcts/lrm/forest/ddl/division.sql +++ b/bcts/lrm/forest/ddl/division.sql @@ -31,3 +31,5 @@ CREATE TABLE lrm_replication.division ( PRIMARY KEY (divi_div_nbr) ); +ALTER TABLE lrm_replication.division OWNER TO bcts_etl_user; + diff --git a/bcts/lrm/forest/ddl/division_code_lookup.sql b/bcts/lrm/forest/ddl/division_code_lookup.sql index 151289a..2f399e5 100644 --- a/bcts/lrm/forest/ddl/division_code_lookup.sql +++ b/bcts/lrm/forest/ddl/division_code_lookup.sql @@ -10,3 +10,5 @@ CREATE TABLE lrm_replication.division_code_lookup ( createdusing VARCHAR(120) NULL, PRIMARY KEY (colu_lookup_type, colu_lookup_id, divi_div_nbr) ); + +ALTER TABLE lrm_replication.division_code_lookup OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/licence.sql b/bcts/lrm/forest/ddl/licence.sql index ad9f294..0864914 100644 --- a/bcts/lrm/forest/ddl/licence.sql +++ b/bcts/lrm/forest/ddl/licence.sql @@ -42,3 +42,5 @@ CREATE TABLE lrm_replication.licence ( licn_archive_date TIMESTAMP NULL, PRIMARY KEY (licn_seq_nbr) ); + +ALTER TABLE lrm_replication.licence OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/management_unit.sql b/bcts/lrm/forest/ddl/management_unit.sql index b2f3253..205f177 100644 --- a/bcts/lrm/forest/ddl/management_unit.sql +++ b/bcts/lrm/forest/ddl/management_unit.sql @@ -24,3 +24,5 @@ CREATE TABLE lrm_replication.management_unit ( non_bcts_aac_ind NUMERIC(1) NULL, PRIMARY KEY (manu_seq_nbr) ); + +ALTER TABLE lrm_replication.management_unit OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/mark.sql b/bcts/lrm/forest/ddl/mark.sql index b42cea7..77d401a 100644 --- a/bcts/lrm/forest/ddl/mark.sql +++ b/bcts/lrm/forest/ddl/mark.sql @@ -18,3 +18,5 @@ CREATE TABLE lrm_replication.mark ( createdusing VARCHAR(120) NULL, PRIMARY KEY (mark_seq_nbr) ); + +ALTER TABLE lrm_replication.mark OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/tenure_type.sql b/bcts/lrm/forest/ddl/tenure_type.sql index 88408da..f658194 100644 --- a/bcts/lrm/forest/ddl/tenure_type.sql +++ b/bcts/lrm/forest/ddl/tenure_type.sql @@ -13,3 +13,5 @@ CREATE TABLE lrm_replication.tenure_type ( createdusing VARCHAR(120) NULL, PRIMARY KEY (tent_seq_nbr) ); + +ALTER TABLE lrm_replication.tenure_type OWNER TO bcts_etl_user; From c555480edf75c10682ccd738ae79bcf743084878 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Tue, 5 Nov 2024 14:16:56 -0800 Subject: [PATCH 04/23] Added DDL for initializing BCTS roles and schemas --- bcts/access_management/ddl/bcts_roles.sql | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bcts/access_management/ddl/bcts_roles.sql b/bcts/access_management/ddl/bcts_roles.sql index 40cb580..b019c1e 100644 --- a/bcts/access_management/ddl/bcts_roles.sql +++ b/bcts/access_management/ddl/bcts_roles.sql @@ -15,22 +15,26 @@ CREATE ROLE BCTS_STAGE_ANALYST_PI_ROLE; CREATE ROLE BCTS_ANALYST_ROLE; CREATE ROLE BCTS_ANALYST_PI_ROLE; --- 2. Grant access to schemas A, B, and C +-- 2. Grant access to schemas lrm_replication, bcts_staging, and bcts_reporting GRANT USAGE ON SCHEMA lrm_replication TO bcts_etl_user; GRANT USAGE ON SCHEMA bcts_staging TO bcts_etl_user; GRANT USAGE ON SCHEMA bcts_reporting TO bcts_etl_user; --- 3. Grant read and write access to existing tables in schemas A, B, and C +-- 3. Grant read and write access to existing tables in schemas lrm_replication, bcts_staging, and bcts_reporting GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA lrm_replication TO bcts_etl_user; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_staging TO bcts_etl_user; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_reporting TO bcts_etl_user; --- 4. Grant permission to create new tables, functions, etc., in schemas A, B, and C +GRANT USAGE ON SCHEMA ods_data_management TO bcts_etl_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON ods_data_management.cdc_master_table_list TO bcts_etl_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON ods_data_management.audit_batch_status TO bcts_etl_user; + +-- 4. Grant permission to create new tables, functions, etc., in schemas lrm_replication, bcts_staging, and bcts_reporting GRANT CREATE ON SCHEMA lrm_replication TO bcts_etl_user; GRANT CREATE ON SCHEMA bcts_staging TO bcts_etl_user; GRANT CREATE ON SCHEMA bcts_reporting TO bcts_etl_user; --- 5. Grant privileges to automatically apply on any new tables created in schemas A, B, and C +-- 5. Grant privileges to automatically apply on any new tables created in schemas lrm_replication, bcts_staging, and bcts_reporting ALTER DEFAULT PRIVILEGES IN SCHEMA lrm_replication GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO bcts_etl_user; ALTER DEFAULT PRIVILEGES IN SCHEMA bcts_staging GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO bcts_etl_user; ALTER DEFAULT PRIVILEGES IN SCHEMA bcts_reporting GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO bcts_etl_user; From e986e791cad18ed2a123bea6f7ce04e3899fa592 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Thu, 14 Nov 2024 14:10:35 -0800 Subject: [PATCH 05/23] Added with grant option in grant statements for bcts_etl_user, combined lrm ddl scripts to a single file and removed transfer of ownership --- bcts/access_management/ddl/bcts_roles.sql | 12 +- bcts/lrm/forest/ddl/activity.sql | 37 -- bcts/lrm/forest/ddl/activity_class.sql | 17 - bcts/lrm/forest/ddl/activity_type.sql | 32 -- bcts/lrm/forest/ddl/bcts_lrm_forest_ddl.sql | 505 +++++++++++++++++++ bcts/lrm/forest/ddl/block_admin_zone.sql | 15 - bcts/lrm/forest/ddl/block_allocation.sql | 33 -- bcts/lrm/forest/ddl/code_lookup.sql | 19 - bcts/lrm/forest/ddl/cut_block.sql | 134 ----- bcts/lrm/forest/ddl/cut_permit.sql | 72 --- bcts/lrm/forest/ddl/division.sql | 35 -- bcts/lrm/forest/ddl/division_code_lookup.sql | 14 - bcts/lrm/forest/ddl/licence.sql | 46 -- bcts/lrm/forest/ddl/management_unit.sql | 28 - bcts/lrm/forest/ddl/mark.sql | 22 - bcts/lrm/forest/ddl/tenure_type.sql | 17 - 16 files changed, 511 insertions(+), 527 deletions(-) delete mode 100644 bcts/lrm/forest/ddl/activity.sql delete mode 100644 bcts/lrm/forest/ddl/activity_class.sql delete mode 100644 bcts/lrm/forest/ddl/activity_type.sql create mode 100644 bcts/lrm/forest/ddl/bcts_lrm_forest_ddl.sql delete mode 100644 bcts/lrm/forest/ddl/block_admin_zone.sql delete mode 100644 bcts/lrm/forest/ddl/block_allocation.sql delete mode 100644 bcts/lrm/forest/ddl/code_lookup.sql delete mode 100644 bcts/lrm/forest/ddl/cut_block.sql delete mode 100644 bcts/lrm/forest/ddl/cut_permit.sql delete mode 100644 bcts/lrm/forest/ddl/division.sql delete mode 100644 bcts/lrm/forest/ddl/division_code_lookup.sql delete mode 100644 bcts/lrm/forest/ddl/licence.sql delete mode 100644 bcts/lrm/forest/ddl/management_unit.sql delete mode 100644 bcts/lrm/forest/ddl/mark.sql delete mode 100644 bcts/lrm/forest/ddl/tenure_type.sql diff --git a/bcts/access_management/ddl/bcts_roles.sql b/bcts/access_management/ddl/bcts_roles.sql index b019c1e..deb7fd2 100644 --- a/bcts/access_management/ddl/bcts_roles.sql +++ b/bcts/access_management/ddl/bcts_roles.sql @@ -16,14 +16,14 @@ CREATE ROLE BCTS_ANALYST_ROLE; CREATE ROLE BCTS_ANALYST_PI_ROLE; -- 2. Grant access to schemas lrm_replication, bcts_staging, and bcts_reporting -GRANT USAGE ON SCHEMA lrm_replication TO bcts_etl_user; -GRANT USAGE ON SCHEMA bcts_staging TO bcts_etl_user; -GRANT USAGE ON SCHEMA bcts_reporting TO bcts_etl_user; +GRANT USAGE ON SCHEMA lrm_replication TO bcts_etl_user WITH GRANT OPTION; +GRANT USAGE ON SCHEMA bcts_staging TO bcts_etl_user WITH GRANT OPTION; +GRANT USAGE ON SCHEMA bcts_reporting TO bcts_etl_user WITH GRANT OPTION; -- 3. Grant read and write access to existing tables in schemas lrm_replication, bcts_staging, and bcts_reporting -GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA lrm_replication TO bcts_etl_user; -GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_staging TO bcts_etl_user; -GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_reporting TO bcts_etl_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA lrm_replication TO bcts_etl_user WITH GRANT OPTION; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_staging TO bcts_etl_user WITH GRANT OPTION; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA bcts_reporting TO bcts_etl_user WITH GRANT OPTION; GRANT USAGE ON SCHEMA ods_data_management TO bcts_etl_user; GRANT SELECT, INSERT, UPDATE, DELETE ON ods_data_management.cdc_master_table_list TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/activity.sql b/bcts/lrm/forest/ddl/activity.sql deleted file mode 100644 index 3d4d5fb..0000000 --- a/bcts/lrm/forest/ddl/activity.sql +++ /dev/null @@ -1,37 +0,0 @@ -CREATE TABLE lrm_replication.activity ( - acti_seq_nbr NUMERIC(15) NOT NULL, - cutb_seq_nbr NUMERIC(15) NULL, - perm_seq_nbr NUMERIC(15) NULL, - actt_seq_nbr NUMERIC(15) NOT NULL, - ctor_seq_nbr NUMERIC(15) NULL, - acti_status_ind VARCHAR(30) NULL, - acti_status_date TIMESTAMP NULL, - acti_comments VARCHAR(2000) NULL, - acti_cost NUMERIC(15, 2) NULL, - acti_harv_seas_id VARCHAR(40) NULL, - acti_responsibility VARCHAR(160) NULL, - acti_area NUMERIC(11, 6) NULL, - acti_harvest_vol NUMERIC(15, 6) NULL, - acti_int_reason VARCHAR(40) NULL, - acti_target_date TIMESTAMP NULL, - acti_fdp_reason VARCHAR(40) NULL, - acti_target_cost NUMERIC(15, 2) NULL, - plan_seq_nbr NUMERIC(15) NULL, - licn_seq_nbr NUMERIC(15) NULL, - mark_seq_nbr NUMERIC(15) NULL, - cloc_seq_nbr NUMERIC(15) NULL, - acti_digitized_ind VARCHAR(4) NULL, - acti_cost_uom VARCHAR(120) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - documentkey NUMERIC(9) NULL, - accl_description VARCHAR(160) NULL, - PRIMARY KEY (acti_seq_nbr) -); - -ALTER TABLE lrm_replication.activity OWNER TO bcts_etl_user; - diff --git a/bcts/lrm/forest/ddl/activity_class.sql b/bcts/lrm/forest/ddl/activity_class.sql deleted file mode 100644 index a4da5dc..0000000 --- a/bcts/lrm/forest/ddl/activity_class.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE TABLE lrm_replication.activity_class ( - accl_seq_nbr NUMERIC(15) NOT NULL, - accl_description VARCHAR(40) NULL, - accl_object_type VARCHAR(1) NULL, - accl_display_order NUMERIC(3) NULL, - divi_div_nbr NUMERIC(2) NULL, - accl_key_ind VARCHAR(12) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (accl_seq_nbr) -); - -ALTER TABLE lrm_replication.activity_class OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/activity_type.sql b/bcts/lrm/forest/ddl/activity_type.sql deleted file mode 100644 index ed265a2..0000000 --- a/bcts/lrm/forest/ddl/activity_type.sql +++ /dev/null @@ -1,32 +0,0 @@ -CREATE TABLE lrm_replication.activity_type ( - actt_seq_nbr NUMERIC(15) NOT NULL, - accl_seq_nbr NUMERIC(15) NOT NULL, - actt_description VARCHAR(200) NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - actt_default_ind VARCHAR(4) NULL, - actt_responsibility VARCHAR(160) NULL, - actt_display_order NUMERIC(2) NULL, - actt_key_ind VARCHAR(10) NULL, - actt_status_ind VARCHAR(4) NULL, - actt_date_relative VARCHAR(20) NULL, - ctor_contractor_id VARCHAR(40) NULL, - actt_view_level NUMERIC(2) NULL, - actt_system_ind VARCHAR(4) NULL, - ctor_seq_nbr NUMERIC(15) NULL, - actt_active_ind VARCHAR(4) NOT NULL, - actt_key_pair VARCHAR(40) NULL, - actt_harvs_ind VARCHAR(4) NULL, - actt_trvol_ind VARCHAR(4) NULL, - actt_indirect_cost_ind VARCHAR(8) NULL, - actt_default_cost NUMERIC(9, 2) NULL, - actt_cost_uom VARCHAR(120) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (actt_seq_nbr) -); - -ALTER TABLE lrm_replication.activity_type OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/bcts_lrm_forest_ddl.sql b/bcts/lrm/forest/ddl/bcts_lrm_forest_ddl.sql new file mode 100644 index 0000000..3d89f22 --- /dev/null +++ b/bcts/lrm/forest/ddl/bcts_lrm_forest_ddl.sql @@ -0,0 +1,505 @@ +-- Create target tables +CREATE TABLE IF NOT EXISTS lrm_replication.division ( + divi_div_nbr NUMERIC(2) NOT NULL, + divi_country_code VARCHAR(40) NOT NULL, + divi_prov_state_code VARCHAR(40) NOT NULL, + divi_division_name VARCHAR(50) NOT NULL, + divi_startup_date TIMESTAMP NULL, + divi_end_date TIMESTAMP NULL, + divi_short_code VARCHAR(15) NULL, + divi_line1_addr VARCHAR(120) NULL, + divi_line2_addr VARCHAR(120) NULL, + divi_line3_addr VARCHAR(120) NULL, + divi_city_name VARCHAR(200) NULL, + divi_postal_code VARCHAR(40) NULL, + divi_intl_routing_code VARCHAR(60) NULL, + divi_phone_nbr VARCHAR(40) NULL, + divi_fax_nbr VARCHAR(40) NULL, + divi_stmpg_acctcd VARCHAR(200) NULL, + divi_stmpg_offset_acctcd VARCHAR(200) NULL, + divi_walker_entity_acctcd VARCHAR(60) NULL, + divi_gst_pay_acctcd VARCHAR(200) NULL, + divi_gst_recovery_acctd VARCHAR(200) NULL, + divi_operation_location_ind VARCHAR(4) NULL, + divi_client_location_code VARCHAR(40) NULL, + divi_abbreviation_code VARCHAR(16) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (divi_div_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.block_allocation ( + cutb_seq_nbr NUMERIC(15) NOT NULL, + blal_seq_nbr NUMERIC(15) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + licn_seq_nbr NUMERIC(15) NULL, + perm_seq_nbr NUMERIC(15) NULL, + mark_seq_nbr NUMERIC(15) NULL, + blal_gross_ha_area NUMERIC(11, 6) NULL, + blal_merch_ha_area NUMERIC(11, 6) NULL, + blal_cruise_m3_vol NUMERIC(15, 6) NULL, + blal_harvested_m3_vol NUMERIC(15, 6) NULL, + blal_harvested_ha_area NUMERIC(11, 6) NULL, + blal_firs_cutblock_id VARCHAR(80) NULL, + blal_firs_timbermark_id VARCHAR(80) NULL, + blal_estimated_area NUMERIC(7, 2) NULL, + manu_seq_nbr NUMERIC(15) NULL, + blal_digi_ind VARCHAR(1) NULL, + blal_primary_ind VARCHAR(4) NULL, + blal_actual_partition_vol NUMERIC(15, 2) NULL, + blal_usr_cruise_m3_vol NUMERIC(15, 6) NULL, + blal_rw_vol NUMERIC(15, 6) NULL, + blal_rw_ha_area NUMERIC(11, 6) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + blal_data_source VARCHAR(120) NULL, + PRIMARY KEY (blal_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.management_unit ( + manu_seq_nbr NUMERIC(15) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + manu_id VARCHAR(60) NOT NULL, + manu_name VARCHAR(200) NULL, + manu_comment VARCHAR(4000) NULL, + manu_type_id VARCHAR(40) NULL, + manu_operating_zone VARCHAR(40) NULL, + manu_mgr_seq_nbr NUMERIC(15) NULL, + manu_fst_seq_nbr NUMERIC(15) NULL, + manu_digitize_ind VARCHAR(20) NULL, + manu_area NUMERIC(11, 6) NULL, + manu_net_area NUMERIC(11, 6) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + nat_res_area VARCHAR(1) NULL, + nat_res_region VARCHAR(2) NULL, + manu_aac_effect_date TIMESTAMP NULL, + manu_aac NUMERIC(10, 2) NULL, + non_bcts_aac_ind NUMERIC(1) NULL, + PRIMARY KEY (manu_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.licence ( + licn_seq_nbr NUMERIC(15) NOT NULL, + licn_licence_id VARCHAR(15) NOT NULL, + licn_licence_desc VARCHAR(53) NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + licn_crown_land CHAR NULL, + licn_annual_allowable_cut NUMERIC(9) NULL, + lsee_licensee_id VARCHAR(10) NULL, + tent_seq_nbr NUMERIC(15) NULL, + licn_licence_state VARCHAR(20) NULL, + licn_permit_exists_ind VARCHAR(4) NULL, + licn_salvage_ind VARCHAR(1) NULL, + licn_category_id VARCHAR(10) NULL, + licn_field_team_id VARCHAR(10) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + blaz_admin_zone_id VARCHAR(10) NULL, + licn_digi_ind VARCHAR(1) NULL, + licl_licence_class VARCHAR(10) NULL, + licn_parent_licence NUMERIC(15) NULL, + licn_crown_granted_ind VARCHAR(4) NULL, + licn_client_loc_code VARCHAR(20) NULL, + licn_category_type VARCHAR(16) NULL, + licn_licence_to_cut_code VARCHAR(10) NULL, + linc_cert_level_id VARCHAR(10) NULL, + licn_mgr_seq_nbr NUMERIC(15) NULL, + licn_fst_seq_nbr NUMERIC(15) NULL, + licn_gross_area NUMERIC(11, 6) NULL, + licn_net_area NUMERIC(11, 6) NULL, + licn_comment VARCHAR(4000) NULL, + licn_apportion_tenure_type VARCHAR(30) NULL, + team_seq_nbr NUMERIC(15) NULL, + licn_hammermark VARCHAR(60) NULL, + licn_client_location_code VARCHAR(40) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + licn_archive_ind VARCHAR(3) NULL, + licn_archive_date TIMESTAMP NULL, + PRIMARY KEY (licn_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.block_admin_zone ( + blaz_admin_zone_id VARCHAR(40) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + blaz_admin_zone_desc VARCHAR(200) NULL, + blaz_active_ind VARCHAR(1) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (blaz_admin_zone_id, divi_div_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.division_code_lookup ( + colu_lookup_type VARCHAR(4) NOT NULL, + colu_lookup_id VARCHAR(120) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (colu_lookup_type, colu_lookup_id, divi_div_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.code_lookup ( + colu_lookup_type VARCHAR(4) NOT NULL, + colu_lookup_id VARCHAR(30) NOT NULL, + colu_lookup_desc VARCHAR(150) NOT NULL, + colu_user_defined_ind VARCHAR(4) NULL, + colu_display_ind VARCHAR(1) NULL, + colu_display_order NUMERIC(10) NULL, + colu_comment VARCHAR(4000) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + hq_display_ind VARCHAR(1) NULL, + PRIMARY KEY (colu_lookup_type, colu_lookup_id) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.tenure_type ( + tent_seq_nbr NUMERIC(15) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + tent_tenure_id VARCHAR(40) NOT NULL, + tent_tenure_name VARCHAR(160) NULL, + tety_tenure_type VARCHAR(40) NULL, + tent_active_ind VARCHAR(4) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (tent_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.cut_permit ( + perm_seq_nbr NUMERIC(15) NOT NULL, + regn_region_id VARCHAR(10) NULL, + perm_permit_id VARCHAR(40) NOT NULL, + perm_revision_date TIMESTAMP NULL, + papr_point_of_apprsl_id VARCHAR(16) NULL, + lsee_licensee_id VARCHAR(10) NULL, + admin_dsct_district_name VARCHAR(60) NULL, + geo_dsct_district_name VARCHAR(240) NULL, + perm_mgmt_unit_code VARCHAR(12) NULL, + perm_tsb_name VARCHAR(200) NULL, + perm_expiry_date TIMESTAMP NULL, + perm_remote_oper_ind VARCHAR(4) NULL, + perm_basic_silvi_ind VARCHAR(4) NULL, + perm_road_mtce_ind VARCHAR(4) NULL, + perm_road_land_use_chrg NUMERIC(15, 2) NULL, + perm_cost_eff_date TIMESTAMP NULL, + perm_rate_eff_date TIMESTAMP NULL, + perm_water_transport_ind VARCHAR(4) NULL, + perm_primary_block_seq_nbr NUMERIC(15) NULL, + perm_primary_mark_id VARCHAR(40) NULL, + perm_haul_ind VARCHAR(4) NULL, + tsar_tsa_nbr NUMERIC(3) NULL, + sblk_supply_block_id VARCHAR(40) NULL, + sptc_support_centre_name VARCHAR(50) NULL, + divi_div_nbr NUMERIC(2) NULL, + perm_damage_status VARCHAR(20) NULL, + perm_support_centre_dist NUMERIC(6, 2) NULL, + opty_opening_type_id VARCHAR(40) NULL, + pers_seq_nbr NUMERIC(15) NULL, + perm_permit_state VARCHAR(80) NULL, + perm_logprod_external_ind VARCHAR(4) NULL, + perm_logprod_customer_id VARCHAR(60) NULL, + perm_cruise_res_source VARCHAR(4) NULL, + perm_jurisdiction VARCHAR(40) NULL, + perm_application_description VARCHAR(4000) NULL, + perm_primary_species VARCHAR(40) NULL, + mkme_marking_method_code VARCHAR(40) NULL, + mkin_marking_instrument_code VARCHAR(40) NULL, + perm_prov_forest_conflict VARCHAR(40) NULL, + perm_cruise_based_ind VARCHAR(40) NULL, + perm_lsee_representative NUMERIC(15) NULL, + perm_traverse_start_point VARCHAR(160) NULL, + perm_traverse_end_point VARCHAR(160) NULL, + perm_digi_ind VARCHAR(4) NULL, + perm_bdt_conversion NUMERIC(15, 7) NULL, + cpcl_permit_class VARCHAR(40) NULL, + perm_parent_permit NUMERIC(15) NULL, + perm_spuc VARCHAR(40) NULL, + perm_salvage_ind VARCHAR(4) NULL, + perm_location VARCHAR(800) NULL, + perm_status VARCHAR(40) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + perm_fst_seq_nbr NUMERIC(15) NULL, + perm_bid_amount NUMERIC(10, 2) NULL, + perm_total_cost NUMERIC(10, 2) NULL, + perm_legal_description VARCHAR(400) NULL, + perm_gross_area NUMERIC(15, 7) NULL, + perm_high_lvl_pln VARCHAR(120) NULL, + perm_lcn_hvst_auth VARCHAR(120) NULL, + perm_lcn_reg CHAR NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (perm_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.mark ( + mark_seq_nbr NUMERIC(15) NOT NULL, + mark_mark_id VARCHAR(15) NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + mark_mark_desc VARCHAR(160) NULL, + mark_mark_state VARCHAR(80) NULL, + mark_crown_granted_ind VARCHAR(4) NULL, + mark_aac_partition VARCHAR(40) NULL, + mark_apportionment VARCHAR(40) NULL, + mark_endemic_percent NUMERIC(4, 2) NULL, + mark_species_type VARCHAR(8) NULL, + mark_all_log_grades_ind VARCHAR(4) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (mark_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.cut_block ( + cutb_seq_nbr NUMERIC(15) NOT NULL, + cutb_block_id VARCHAR(20) NOT NULL, + cutb_block_number VARCHAR(15) NULL, + divi_div_nbr NUMERIC(2) NULL, + cutb_devt_plan_status_code VARCHAR(8) NULL, + cutb_devt_plan_apprvl_date TIMESTAMP NULL, + cutb_silvi_plan_status_code VARCHAR(8) NULL, + cutb_silvi_plan_apprvl_date TIMESTAMP NULL, + cutb_block_memo VARCHAR(4000) NULL, + cutb_woods_run_km_nbr NUMERIC(15, 6) NULL, + cutb_hauling_km_nbr NUMERIC(15, 6) NULL, + cutb_hlcptr_drop_cruise_ind VARCHAR(4) NULL, + cutb_logging_plan_status_code VARCHAR(8) NULL, + cutb_logging_plan_apprvl_date TIMESTAMP NULL, + opar_operating_area_id VARCHAR(10) NULL, + bioz_zone_id VARCHAR(40) NULL, + cutb_gross_ha_area NUMERIC(11, 6) NULL, + cutb_bcgs VARCHAR(15) NULL, + cutb_opening VARCHAR(15) NULL, + cutb_nts VARCHAR(60) NULL, + cutb_ext_ha_area NUMERIC(11, 6) NULL, + cutb_photos VARCHAR(800) NULL, + cutb_field_work_by VARCHAR(200) NULL, + cutb_field_work_date TIMESTAMP NULL, + cutb_site_class VARCHAR(20) NULL, + fdps_status_id VARCHAR(10) NULL, + cutb_site_index NUMERIC(5) NULL, + cutb_cpi_slope_pct NUMERIC(7, 4) NULL, + cutb_cpi_road_access_km NUMERIC(13, 3) NULL, + cutb_cpi_road_block_km NUMERIC(13, 3) NULL, + cutb_cpi_road_num_landings NUMERIC(10) NULL, + cutb_cpi_road_haul_km NUMERIC(13, 3) NULL, + sblk_supply_block_id VARCHAR(10) NULL, + cutb_formc_harvest_print_date TIMESTAMP NULL, + cutb_formc_regen_print_date TIMESTAMP NULL, + cutb_formc_freegrow_print_date TIMESTAMP NULL, + fund_funding_code VARCHAR(15) NULL, + cutb_highway_ind VARCHAR(1) NULL, + cutb_winter_road_ind CHAR NULL, + finz_forest_inventory_zone_id VARCHAR(40) NULL, + cutb_user_mapsheet_id VARCHAR(120) NULL, + cutb_cell_number VARCHAR(60) NULL, + fjap_fdp_joint_approval VARCHAR(40) NULL, + pmod_modifier_id VARCHAR(40) NULL, + cutb_traverse_method_code VARCHAR(40) NULL, + cutb_critical_date_ind CHAR NULL, + cutb_greenup_date TIMESTAMP NULL, + grns_greenup_source VARCHAR(40) NULL, + pmpo_operating_zone VARCHAR(40) NULL, + lsun_landscape_unit VARCHAR(40) NULL, + plan_seq_nbr NUMERIC(15) NULL, + cutb_forma_print_date TIMESTAMP NULL, + cutb_block_state VARCHAR(20) NULL, + cutb_digi_ind VARCHAR(4) NULL, + sttp_stand_type VARCHAR(40) NULL, + ttac_timbertype_age_class VARCHAR(40) NULL, + ttsc_timbertype_stock_class VARCHAR(40) NULL, + tthc_timbertype_height_class VARCHAR(40) NULL, + cutb_site_plan_exempt_ind VARCHAR(4) NULL, + sssc_source_code VARCHAR(40) NULL, + cutb_vol_data_source VARCHAR(40) NULL, + cutb_vol_data_source_type VARCHAR(16) NULL, + cutb_heli_flight_dist NUMERIC(10) NULL, + cutb_harvest_sequence NUMERIC(6) NULL, + cutb_prov_forest_conflict VARCHAR(40) NULL, + treg_seq_nbr NUMERIC(15) NULL, + bcat_category_code VARCHAR(40) NULL, + cutb_traverse_start_point VARCHAR(160) NULL, + cutb_traverse_end_point VARCHAR(160) NULL, + cutb_forma_printed VARCHAR(4) NULL, + cutb_system_id VARCHAR(15) NULL, + cutb_block_status VARCHAR(40) NULL, + cutb_latitude_bak VARCHAR(40) NULL, + cutb_longitude_bak VARCHAR(40) NULL, + cutb_damage_type VARCHAR(40) NULL, + cutb_opening_id NUMERIC(10) NULL, + silp_use_block_num_ind VARCHAR(1) NOT NULL, + cutb_location VARCHAR(800) NULL, + cutb_silv_act_harv_link VARCHAR(120) NULL, + cutb_file_id VARCHAR(72) NULL, + suop_subop_area_id VARCHAR(40) NULL, + cutb_selling_price_period TIMESTAMP NULL, + cutb_synch_status VARCHAR(120) NULL, + cutb_block_grouping_id VARCHAR(80) NULL, + pers_seq_nbr NUMERIC(15) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + cutb_variant_id VARCHAR(60) NULL, + cutb_season_id VARCHAR(120) NULL, + cutb_volume NUMERIC(15, 6) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + cutb_parent_seq NUMERIC(15) NULL, + cutb_prev_forma_print_date TIMESTAMP NULL, + sila_seq_nbr NUMERIC(15) NULL, + cutb_cruise_lock_ind VARCHAR(4) NULL, + cutb_ref_dec CHAR NULL, + cutb_reg_code VARCHAR(12) NULL, + cutb_apr_eff_date TIMESTAMP NULL, + siph_seq_nbr NUMERIC(15) NULL, + cdat_seq_nbr INTEGER NULL, + cutb_access_restriction VARCHAR(4000) NULL, + cutb_planned_vol NUMERIC(15, 6) NULL, + cutb_row_ind VARCHAR(4) NOT NULL, + cutb_tim_dev_date TIMESTAMP NULL, + cutb_ebm_ind VARCHAR(4) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + documentkey NUMERIC(9) NULL, + log_start_cascade_date_ind VARCHAR(12) NULL, + pricing_date TIMESTAMP NULL, + cutb_latitude_dd NUMERIC(9, 6) NULL, + cutb_longitude_dd NUMERIC(9, 6) NULL, + cutb_latitude VARCHAR(40) NULL, + cutb_longitude VARCHAR(40) NULL, + marked_for_del_ind NUMERIC(1) NULL, + marked_for_del_by VARCHAR(120) NULL, + marked_for_del_on TIMESTAMP NULL, + cutb_archive_reason VARCHAR(4) NULL, + cutb_archive_date TIMESTAMP NULL, + cutb_cprp_protection_ind VARCHAR(1) NULL, + cutb_rc_risk_rating VARCHAR(1) NULL, + cutb_rc_risk_source VARCHAR(5) NULL, + cutb_rc_risk_comments VARCHAR(2000) NULL, + operational_site_factor VARCHAR(4000) NULL, + safety_concern VARCHAR(4000) NULL, + PRIMARY KEY (cutb_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.activity_class ( + accl_seq_nbr NUMERIC(15) NOT NULL, + accl_description VARCHAR(40) NULL, + accl_object_type VARCHAR(1) NULL, + accl_display_order NUMERIC(3) NULL, + divi_div_nbr NUMERIC(2) NULL, + accl_key_ind VARCHAR(12) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (accl_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.activity_type ( + actt_seq_nbr NUMERIC(15) NOT NULL, + accl_seq_nbr NUMERIC(15) NOT NULL, + actt_description VARCHAR(200) NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + actt_default_ind VARCHAR(4) NULL, + actt_responsibility VARCHAR(160) NULL, + actt_display_order NUMERIC(2) NULL, + actt_key_ind VARCHAR(10) NULL, + actt_status_ind VARCHAR(4) NULL, + actt_date_relative VARCHAR(20) NULL, + ctor_contractor_id VARCHAR(40) NULL, + actt_view_level NUMERIC(2) NULL, + actt_system_ind VARCHAR(4) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + actt_active_ind VARCHAR(4) NOT NULL, + actt_key_pair VARCHAR(40) NULL, + actt_harvs_ind VARCHAR(4) NULL, + actt_trvol_ind VARCHAR(4) NULL, + actt_indirect_cost_ind VARCHAR(8) NULL, + actt_default_cost NUMERIC(9, 2) NULL, + actt_cost_uom VARCHAR(120) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (actt_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.activity ( + acti_seq_nbr NUMERIC(15) NOT NULL, + cutb_seq_nbr NUMERIC(15) NULL, + perm_seq_nbr NUMERIC(15) NULL, + actt_seq_nbr NUMERIC(15) NOT NULL, + ctor_seq_nbr NUMERIC(15) NULL, + acti_status_ind VARCHAR(30) NULL, + acti_status_date TIMESTAMP NULL, + acti_comments VARCHAR(2000) NULL, + acti_cost NUMERIC(15, 2) NULL, + acti_harv_seas_id VARCHAR(40) NULL, + acti_responsibility VARCHAR(160) NULL, + acti_area NUMERIC(11, 6) NULL, + acti_harvest_vol NUMERIC(15, 6) NULL, + acti_int_reason VARCHAR(40) NULL, + acti_target_date TIMESTAMP NULL, + acti_fdp_reason VARCHAR(40) NULL, + acti_target_cost NUMERIC(15, 2) NULL, + plan_seq_nbr NUMERIC(15) NULL, + licn_seq_nbr NUMERIC(15) NULL, + mark_seq_nbr NUMERIC(15) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + acti_digitized_ind VARCHAR(4) NULL, + acti_cost_uom VARCHAR(120) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + documentkey NUMERIC(9) NULL, + accl_description VARCHAR(160) NULL, + PRIMARY KEY (acti_seq_nbr) +); + diff --git a/bcts/lrm/forest/ddl/block_admin_zone.sql b/bcts/lrm/forest/ddl/block_admin_zone.sql deleted file mode 100644 index e10ac83..0000000 --- a/bcts/lrm/forest/ddl/block_admin_zone.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE lrm_replication.block_admin_zone ( - blaz_admin_zone_id VARCHAR(40) NOT NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - blaz_admin_zone_desc VARCHAR(200) NULL, - blaz_active_ind VARCHAR(1) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (blaz_admin_zone_id, divi_div_nbr) -); - -ALTER TABLE lrm_replication.block_admin_zone OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/block_allocation.sql b/bcts/lrm/forest/ddl/block_allocation.sql deleted file mode 100644 index bf86fa1..0000000 --- a/bcts/lrm/forest/ddl/block_allocation.sql +++ /dev/null @@ -1,33 +0,0 @@ -CREATE TABLE lrm_replication.block_allocation ( - cutb_seq_nbr NUMERIC(15) NOT NULL, - blal_seq_nbr NUMERIC(15) NOT NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - licn_seq_nbr NUMERIC(15) NULL, - perm_seq_nbr NUMERIC(15) NULL, - mark_seq_nbr NUMERIC(15) NULL, - blal_gross_ha_area NUMERIC(11, 6) NULL, - blal_merch_ha_area NUMERIC(11, 6) NULL, - blal_cruise_m3_vol NUMERIC(15, 6) NULL, - blal_harvested_m3_vol NUMERIC(15, 6) NULL, - blal_harvested_ha_area NUMERIC(11, 6) NULL, - blal_firs_cutblock_id VARCHAR(80) NULL, - blal_firs_timbermark_id VARCHAR(80) NULL, - blal_estimated_area NUMERIC(7, 2) NULL, - manu_seq_nbr NUMERIC(15) NULL, - blal_digi_ind VARCHAR(1) NULL, - blal_primary_ind VARCHAR(4) NULL, - blal_actual_partition_vol NUMERIC(15, 2) NULL, - blal_usr_cruise_m3_vol NUMERIC(15, 6) NULL, - blal_rw_vol NUMERIC(15, 6) NULL, - blal_rw_ha_area NUMERIC(11, 6) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - blal_data_source VARCHAR(120) NULL, - PRIMARY KEY (blal_seq_nbr) -); - -ALTER TABLE lrm_replication.block_allocation OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/code_lookup.sql b/bcts/lrm/forest/ddl/code_lookup.sql deleted file mode 100644 index 857039e..0000000 --- a/bcts/lrm/forest/ddl/code_lookup.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TABLE lrm_replication.code_lookup ( - colu_lookup_type VARCHAR(4) NOT NULL, - colu_lookup_id VARCHAR(30) NOT NULL, - colu_lookup_desc VARCHAR(150) NOT NULL, - colu_user_defined_ind VARCHAR(4) NULL, - colu_display_ind VARCHAR(1) NULL, - colu_display_order NUMERIC(10) NULL, - colu_comment VARCHAR(4000) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - hq_display_ind VARCHAR(1) NULL, - PRIMARY KEY (colu_lookup_type, colu_lookup_id) -); - -ALTER TABLE lrm_replication.code_lookup OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/cut_block.sql b/bcts/lrm/forest/ddl/cut_block.sql deleted file mode 100644 index 2cd8bbd..0000000 --- a/bcts/lrm/forest/ddl/cut_block.sql +++ /dev/null @@ -1,134 +0,0 @@ -CREATE TABLE lrm_replication.cut_block ( - cutb_seq_nbr NUMERIC(15) NOT NULL, - cutb_block_id VARCHAR(20) NOT NULL, - cutb_block_number VARCHAR(15) NULL, - divi_div_nbr NUMERIC(2) NULL, - cutb_devt_plan_status_code VARCHAR(8) NULL, - cutb_devt_plan_apprvl_date TIMESTAMP NULL, - cutb_silvi_plan_status_code VARCHAR(8) NULL, - cutb_silvi_plan_apprvl_date TIMESTAMP NULL, - cutb_block_memo VARCHAR(4000) NULL, - cutb_woods_run_km_nbr NUMERIC(15, 6) NULL, - cutb_hauling_km_nbr NUMERIC(15, 6) NULL, - cutb_hlcptr_drop_cruise_ind VARCHAR(4) NULL, - cutb_logging_plan_status_code VARCHAR(8) NULL, - cutb_logging_plan_apprvl_date TIMESTAMP NULL, - opar_operating_area_id VARCHAR(10) NULL, - bioz_zone_id VARCHAR(40) NULL, - cutb_gross_ha_area NUMERIC(11, 6) NULL, - cutb_bcgs VARCHAR(15) NULL, - cutb_opening VARCHAR(15) NULL, - cutb_nts VARCHAR(60) NULL, - cutb_ext_ha_area NUMERIC(11, 6) NULL, - cutb_photos VARCHAR(800) NULL, - cutb_field_work_by VARCHAR(200) NULL, - cutb_field_work_date TIMESTAMP NULL, - cutb_site_class VARCHAR(20) NULL, - fdps_status_id VARCHAR(10) NULL, - cutb_site_index NUMERIC(5) NULL, - cutb_cpi_slope_pct NUMERIC(7, 4) NULL, - cutb_cpi_road_access_km NUMERIC(13, 3) NULL, - cutb_cpi_road_block_km NUMERIC(13, 3) NULL, - cutb_cpi_road_num_landings NUMERIC(10) NULL, - cutb_cpi_road_haul_km NUMERIC(13, 3) NULL, - sblk_supply_block_id VARCHAR(10) NULL, - cutb_formc_harvest_print_date TIMESTAMP NULL, - cutb_formc_regen_print_date TIMESTAMP NULL, - cutb_formc_freegrow_print_date TIMESTAMP NULL, - fund_funding_code VARCHAR(15) NULL, - cutb_highway_ind VARCHAR(1) NULL, - cutb_winter_road_ind CHAR NULL, - finz_forest_inventory_zone_id VARCHAR(40) NULL, - cutb_user_mapsheet_id VARCHAR(120) NULL, - cutb_cell_number VARCHAR(60) NULL, - fjap_fdp_joint_approval VARCHAR(40) NULL, - pmod_modifier_id VARCHAR(40) NULL, - cutb_traverse_method_code VARCHAR(40) NULL, - cutb_critical_date_ind CHAR NULL, - cutb_greenup_date TIMESTAMP NULL, - grns_greenup_source VARCHAR(40) NULL, - pmpo_operating_zone VARCHAR(40) NULL, - lsun_landscape_unit VARCHAR(40) NULL, - plan_seq_nbr NUMERIC(15) NULL, - cutb_forma_print_date TIMESTAMP NULL, - cutb_block_state VARCHAR(20) NULL, - cutb_digi_ind VARCHAR(4) NULL, - sttp_stand_type VARCHAR(40) NULL, - ttac_timbertype_age_class VARCHAR(40) NULL, - ttsc_timbertype_stock_class VARCHAR(40) NULL, - tthc_timbertype_height_class VARCHAR(40) NULL, - cutb_site_plan_exempt_ind VARCHAR(4) NULL, - sssc_source_code VARCHAR(40) NULL, - cutb_vol_data_source VARCHAR(40) NULL, - cutb_vol_data_source_type VARCHAR(16) NULL, - cutb_heli_flight_dist NUMERIC(10) NULL, - cutb_harvest_sequence NUMERIC(6) NULL, - cutb_prov_forest_conflict VARCHAR(40) NULL, - treg_seq_nbr NUMERIC(15) NULL, - bcat_category_code VARCHAR(40) NULL, - cutb_traverse_start_point VARCHAR(160) NULL, - cutb_traverse_end_point VARCHAR(160) NULL, - cutb_forma_printed VARCHAR(4) NULL, - cutb_system_id VARCHAR(15) NULL, - cutb_block_status VARCHAR(40) NULL, - cutb_latitude_bak VARCHAR(40) NULL, - cutb_longitude_bak VARCHAR(40) NULL, - cutb_damage_type VARCHAR(40) NULL, - cutb_opening_id NUMERIC(10) NULL, - silp_use_block_num_ind VARCHAR(1) NOT NULL, - cutb_location VARCHAR(800) NULL, - cutb_silv_act_harv_link VARCHAR(120) NULL, - cutb_file_id VARCHAR(72) NULL, - suop_subop_area_id VARCHAR(40) NULL, - cutb_selling_price_period TIMESTAMP NULL, - cutb_synch_status VARCHAR(120) NULL, - cutb_block_grouping_id VARCHAR(80) NULL, - pers_seq_nbr NUMERIC(15) NULL, - ctor_seq_nbr NUMERIC(15) NULL, - cutb_variant_id VARCHAR(60) NULL, - cutb_season_id VARCHAR(120) NULL, - cutb_volume NUMERIC(15, 6) NULL, - cloc_seq_nbr NUMERIC(15) NULL, - cutb_parent_seq NUMERIC(15) NULL, - cutb_prev_forma_print_date TIMESTAMP NULL, - sila_seq_nbr NUMERIC(15) NULL, - cutb_cruise_lock_ind VARCHAR(4) NULL, - cutb_ref_dec CHAR NULL, - cutb_reg_code VARCHAR(12) NULL, - cutb_apr_eff_date TIMESTAMP NULL, - siph_seq_nbr NUMERIC(15) NULL, - cdat_seq_nbr INTEGER NULL, - cutb_access_restriction VARCHAR(4000) NULL, - cutb_planned_vol NUMERIC(15, 6) NULL, - cutb_row_ind VARCHAR(4) NOT NULL, - cutb_tim_dev_date TIMESTAMP NULL, - cutb_ebm_ind VARCHAR(4) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - documentkey NUMERIC(9) NULL, - log_start_cascade_date_ind VARCHAR(12) NULL, - pricing_date TIMESTAMP NULL, - cutb_latitude_dd NUMERIC(9, 6) NULL, - cutb_longitude_dd NUMERIC(9, 6) NULL, - cutb_latitude VARCHAR(40) NULL, - cutb_longitude VARCHAR(40) NULL, - marked_for_del_ind NUMERIC(1) NULL, - marked_for_del_by VARCHAR(120) NULL, - marked_for_del_on TIMESTAMP NULL, - cutb_archive_reason VARCHAR(4) NULL, - cutb_archive_date TIMESTAMP NULL, - cutb_cprp_protection_ind VARCHAR(1) NULL, - cutb_rc_risk_rating VARCHAR(1) NULL, - cutb_rc_risk_source VARCHAR(5) NULL, - cutb_rc_risk_comments VARCHAR(2000) NULL, - operational_site_factor VARCHAR(4000) NULL, - safety_concern VARCHAR(4000) NULL, - PRIMARY KEY (cutb_seq_nbr) -); - - -ALTER TABLE lrm_replication.cut_block OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/cut_permit.sql b/bcts/lrm/forest/ddl/cut_permit.sql deleted file mode 100644 index 9883652..0000000 --- a/bcts/lrm/forest/ddl/cut_permit.sql +++ /dev/null @@ -1,72 +0,0 @@ -CREATE TABLE lrm_replication.cut_permit ( - perm_seq_nbr NUMERIC(15) NOT NULL, - regn_region_id VARCHAR(10) NULL, - perm_permit_id VARCHAR(40) NOT NULL, - perm_revision_date TIMESTAMP NULL, - papr_point_of_apprsl_id VARCHAR(16) NULL, - lsee_licensee_id VARCHAR(10) NULL, - admin_dsct_district_name VARCHAR(60) NULL, - geo_dsct_district_name VARCHAR(240) NULL, - perm_mgmt_unit_code VARCHAR(12) NULL, - perm_tsb_name VARCHAR(200) NULL, - perm_expiry_date TIMESTAMP NULL, - perm_remote_oper_ind VARCHAR(4) NULL, - perm_basic_silvi_ind VARCHAR(4) NULL, - perm_road_mtce_ind VARCHAR(4) NULL, - perm_road_land_use_chrg NUMERIC(15, 2) NULL, - perm_cost_eff_date TIMESTAMP NULL, - perm_rate_eff_date TIMESTAMP NULL, - perm_water_transport_ind VARCHAR(4) NULL, - perm_primary_block_seq_nbr NUMERIC(15) NULL, - perm_primary_mark_id VARCHAR(40) NULL, - perm_haul_ind VARCHAR(4) NULL, - tsar_tsa_nbr NUMERIC(3) NULL, - sblk_supply_block_id VARCHAR(40) NULL, - sptc_support_centre_name VARCHAR(50) NULL, - divi_div_nbr NUMERIC(2) NULL, - perm_damage_status VARCHAR(20) NULL, - perm_support_centre_dist NUMERIC(6, 2) NULL, - opty_opening_type_id VARCHAR(40) NULL, - pers_seq_nbr NUMERIC(15) NULL, - perm_permit_state VARCHAR(80) NULL, - perm_logprod_external_ind VARCHAR(4) NULL, - perm_logprod_customer_id VARCHAR(60) NULL, - perm_cruise_res_source VARCHAR(4) NULL, - perm_jurisdiction VARCHAR(40) NULL, - perm_application_description VARCHAR(4000) NULL, - perm_primary_species VARCHAR(40) NULL, - mkme_marking_method_code VARCHAR(40) NULL, - mkin_marking_instrument_code VARCHAR(40) NULL, - perm_prov_forest_conflict VARCHAR(40) NULL, - perm_cruise_based_ind VARCHAR(40) NULL, - perm_lsee_representative NUMERIC(15) NULL, - perm_traverse_start_point VARCHAR(160) NULL, - perm_traverse_end_point VARCHAR(160) NULL, - perm_digi_ind VARCHAR(4) NULL, - perm_bdt_conversion NUMERIC(15, 7) NULL, - cpcl_permit_class VARCHAR(40) NULL, - perm_parent_permit NUMERIC(15) NULL, - perm_spuc VARCHAR(40) NULL, - perm_salvage_ind VARCHAR(4) NULL, - perm_location VARCHAR(800) NULL, - perm_status VARCHAR(40) NULL, - ctor_seq_nbr NUMERIC(15) NULL, - cloc_seq_nbr NUMERIC(15) NULL, - perm_fst_seq_nbr NUMERIC(15) NULL, - perm_bid_amount NUMERIC(10, 2) NULL, - perm_total_cost NUMERIC(10, 2) NULL, - perm_legal_description VARCHAR(400) NULL, - perm_gross_area NUMERIC(15, 7) NULL, - perm_high_lvl_pln VARCHAR(120) NULL, - perm_lcn_hvst_auth VARCHAR(120) NULL, - perm_lcn_reg CHAR NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (perm_seq_nbr) -); - -ALTER TABLE lrm_replication.cut_permit OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/division.sql b/bcts/lrm/forest/ddl/division.sql deleted file mode 100644 index 5e9fa4a..0000000 --- a/bcts/lrm/forest/ddl/division.sql +++ /dev/null @@ -1,35 +0,0 @@ -CREATE TABLE lrm_replication.division ( - divi_div_nbr NUMERIC(2) NOT NULL, - divi_country_code VARCHAR(40) NOT NULL, - divi_prov_state_code VARCHAR(40) NOT NULL, - divi_division_name VARCHAR(50) NOT NULL, - divi_startup_date TIMESTAMP NULL, - divi_end_date TIMESTAMP NULL, - divi_short_code VARCHAR(15) NULL, - divi_line1_addr VARCHAR(120) NULL, - divi_line2_addr VARCHAR(120) NULL, - divi_line3_addr VARCHAR(120) NULL, - divi_city_name VARCHAR(200) NULL, - divi_postal_code VARCHAR(40) NULL, - divi_intl_routing_code VARCHAR(60) NULL, - divi_phone_nbr VARCHAR(40) NULL, - divi_fax_nbr VARCHAR(40) NULL, - divi_stmpg_acctcd VARCHAR(200) NULL, - divi_stmpg_offset_acctcd VARCHAR(200) NULL, - divi_walker_entity_acctcd VARCHAR(60) NULL, - divi_gst_pay_acctcd VARCHAR(200) NULL, - divi_gst_recovery_acctd VARCHAR(200) NULL, - divi_operation_location_ind VARCHAR(4) NULL, - divi_client_location_code VARCHAR(40) NULL, - divi_abbreviation_code VARCHAR(16) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (divi_div_nbr) -); - -ALTER TABLE lrm_replication.division OWNER TO bcts_etl_user; - diff --git a/bcts/lrm/forest/ddl/division_code_lookup.sql b/bcts/lrm/forest/ddl/division_code_lookup.sql deleted file mode 100644 index 2f399e5..0000000 --- a/bcts/lrm/forest/ddl/division_code_lookup.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE lrm_replication.division_code_lookup ( - colu_lookup_type VARCHAR(4) NOT NULL, - colu_lookup_id VARCHAR(120) NOT NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (colu_lookup_type, colu_lookup_id, divi_div_nbr) -); - -ALTER TABLE lrm_replication.division_code_lookup OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/licence.sql b/bcts/lrm/forest/ddl/licence.sql deleted file mode 100644 index 0864914..0000000 --- a/bcts/lrm/forest/ddl/licence.sql +++ /dev/null @@ -1,46 +0,0 @@ -CREATE TABLE lrm_replication.licence ( - licn_seq_nbr NUMERIC(15) NOT NULL, - licn_licence_id VARCHAR(15) NOT NULL, - licn_licence_desc VARCHAR(53) NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - licn_crown_land CHAR NULL, - licn_annual_allowable_cut NUMERIC(9) NULL, - lsee_licensee_id VARCHAR(10) NULL, - tent_seq_nbr NUMERIC(15) NULL, - licn_licence_state VARCHAR(20) NULL, - licn_permit_exists_ind VARCHAR(4) NULL, - licn_salvage_ind VARCHAR(1) NULL, - licn_category_id VARCHAR(10) NULL, - licn_field_team_id VARCHAR(10) NULL, - ctor_seq_nbr NUMERIC(15) NULL, - cloc_seq_nbr NUMERIC(15) NULL, - blaz_admin_zone_id VARCHAR(10) NULL, - licn_digi_ind VARCHAR(1) NULL, - licl_licence_class VARCHAR(10) NULL, - licn_parent_licence NUMERIC(15) NULL, - licn_crown_granted_ind VARCHAR(4) NULL, - licn_client_loc_code VARCHAR(20) NULL, - licn_category_type VARCHAR(16) NULL, - licn_licence_to_cut_code VARCHAR(10) NULL, - linc_cert_level_id VARCHAR(10) NULL, - licn_mgr_seq_nbr NUMERIC(15) NULL, - licn_fst_seq_nbr NUMERIC(15) NULL, - licn_gross_area NUMERIC(11, 6) NULL, - licn_net_area NUMERIC(11, 6) NULL, - licn_comment VARCHAR(4000) NULL, - licn_apportion_tenure_type VARCHAR(30) NULL, - team_seq_nbr NUMERIC(15) NULL, - licn_hammermark VARCHAR(60) NULL, - licn_client_location_code VARCHAR(40) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - licn_archive_ind VARCHAR(3) NULL, - licn_archive_date TIMESTAMP NULL, - PRIMARY KEY (licn_seq_nbr) -); - -ALTER TABLE lrm_replication.licence OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/management_unit.sql b/bcts/lrm/forest/ddl/management_unit.sql deleted file mode 100644 index 205f177..0000000 --- a/bcts/lrm/forest/ddl/management_unit.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE TABLE lrm_replication.management_unit ( - manu_seq_nbr NUMERIC(15) NOT NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - manu_id VARCHAR(60) NOT NULL, - manu_name VARCHAR(200) NULL, - manu_comment VARCHAR(4000) NULL, - manu_type_id VARCHAR(40) NULL, - manu_operating_zone VARCHAR(40) NULL, - manu_mgr_seq_nbr NUMERIC(15) NULL, - manu_fst_seq_nbr NUMERIC(15) NULL, - manu_digitize_ind VARCHAR(20) NULL, - manu_area NUMERIC(11, 6) NULL, - manu_net_area NUMERIC(11, 6) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - nat_res_area VARCHAR(1) NULL, - nat_res_region VARCHAR(2) NULL, - manu_aac_effect_date TIMESTAMP NULL, - manu_aac NUMERIC(10, 2) NULL, - non_bcts_aac_ind NUMERIC(1) NULL, - PRIMARY KEY (manu_seq_nbr) -); - -ALTER TABLE lrm_replication.management_unit OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/mark.sql b/bcts/lrm/forest/ddl/mark.sql deleted file mode 100644 index 77d401a..0000000 --- a/bcts/lrm/forest/ddl/mark.sql +++ /dev/null @@ -1,22 +0,0 @@ -CREATE TABLE lrm_replication.mark ( - mark_seq_nbr NUMERIC(15) NOT NULL, - mark_mark_id VARCHAR(15) NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - mark_mark_desc VARCHAR(160) NULL, - mark_mark_state VARCHAR(80) NULL, - mark_crown_granted_ind VARCHAR(4) NULL, - mark_aac_partition VARCHAR(40) NULL, - mark_apportionment VARCHAR(40) NULL, - mark_endemic_percent NUMERIC(4, 2) NULL, - mark_species_type VARCHAR(8) NULL, - mark_all_log_grades_ind VARCHAR(4) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (mark_seq_nbr) -); - -ALTER TABLE lrm_replication.mark OWNER TO bcts_etl_user; diff --git a/bcts/lrm/forest/ddl/tenure_type.sql b/bcts/lrm/forest/ddl/tenure_type.sql deleted file mode 100644 index f658194..0000000 --- a/bcts/lrm/forest/ddl/tenure_type.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE TABLE lrm_replication.tenure_type ( - tent_seq_nbr NUMERIC(15) NOT NULL, - divi_div_nbr NUMERIC(2) NOT NULL, - tent_tenure_id VARCHAR(40) NOT NULL, - tent_tenure_name VARCHAR(160) NULL, - tety_tenure_type VARCHAR(40) NULL, - tent_active_ind VARCHAR(4) NULL, - modifiedby VARCHAR(120) NULL, - modifiedon TIMESTAMP NULL, - modifiedusing VARCHAR(120) NULL, - createdby VARCHAR(120) NULL, - createdon TIMESTAMP NULL, - createdusing VARCHAR(120) NULL, - PRIMARY KEY (tent_seq_nbr) -); - -ALTER TABLE lrm_replication.tenure_type OWNER TO bcts_etl_user; From 793c6611d20286bb25b825a8ae5f6aa28a5e7042 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 10:07:53 -0800 Subject: [PATCH 06/23] Added dockerfiles for executing adhoc sql queries in ODS from Airflow for BCTS --- shared/bcts_adhoc_sql/Dockerfile | 12 + shared/bcts_adhoc_sql/lrm_local_secrets.yaml | 17 + shared/bcts_adhoc_sql/run_sql.py | 76 +++ .../sql/active/bcts_lrm_forest_ddl.sql | 505 ++++++++++++++++++ .../sql/active/cdc_master_table_list.sql | 29 + .../sql/archived/placeholder.txt | 0 6 files changed, 639 insertions(+) create mode 100644 shared/bcts_adhoc_sql/Dockerfile create mode 100644 shared/bcts_adhoc_sql/lrm_local_secrets.yaml create mode 100644 shared/bcts_adhoc_sql/run_sql.py create mode 100644 shared/bcts_adhoc_sql/sql/active/bcts_lrm_forest_ddl.sql create mode 100644 shared/bcts_adhoc_sql/sql/active/cdc_master_table_list.sql create mode 100644 shared/bcts_adhoc_sql/sql/archived/placeholder.txt diff --git a/shared/bcts_adhoc_sql/Dockerfile b/shared/bcts_adhoc_sql/Dockerfile new file mode 100644 index 0000000..659932c --- /dev/null +++ b/shared/bcts_adhoc_sql/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11.4-slim-buster + +# PostgreSQL library + +RUN apt-get update \ + && apt-get -y install libpq-dev gcc \ + && pip install psycopg2 + +WORKDIR /app + +COPY . . + diff --git a/shared/bcts_adhoc_sql/lrm_local_secrets.yaml b/shared/bcts_adhoc_sql/lrm_local_secrets.yaml new file mode 100644 index 0000000..9f43bd1 --- /dev/null +++ b/shared/bcts_adhoc_sql/lrm_local_secrets.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Secret +metadata: + name: lrm-ods-database +type: Opaque +stringData: + ODS_USERNAME: bcts_etl_user + ODS_PASSWORD: + ODS_HOST: 172.28.0.10 + ODS_PORT: '5432' + ODS_DATABASE: ods + MSTR_SCHEMA: ods_data_management + APP_NAME: lrm + CONCUR_TASKS: '50' + + + \ No newline at end of file diff --git a/shared/bcts_adhoc_sql/run_sql.py b/shared/bcts_adhoc_sql/run_sql.py new file mode 100644 index 0000000..4ed661c --- /dev/null +++ b/shared/bcts_adhoc_sql/run_sql.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# coding: utf-8 + +#Imports +import os, time +import psycopg2 +import logging +import sys + +start = time.time() + +# Retrieve Postgres database configuration +postgres_username = os.environ['ODS_USERNAME'] +postgres_password = os.environ['ODS_PASSWORD'] +postgres_host = os.environ['ODS_HOST'] +postgres_port = os.environ['ODS_PORT'] +postgres_database = os.environ['ODS_DATABASE'] + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler() + ] +) + +def run_sql_script(script_path): + # Establish database connection + try: + connection = psycopg2.connect( + dbname=postgres_database, + user=postgres_username, + password=postgres_password, + host=postgres_host, + port=postgres_port + ) + cursor = connection.cursor() + logging.info("Database connection established.") + except psycopg2.Error as e: + logging.error(f"Error connecting to the database: {e}") + return + + # Read and execute the SQL script + try: + with open(script_path, 'r') as file: + sql_script = file.read() + + # Execute the SQL script + logging.info(sql_script) + cursor.execute(sql_script) + connection.commit() + logging.info(f"SQL script executed successfully from {script_path}.") + except psycopg2.Error as e: + logging.error(f"Error executing the SQL script: {e}") + connection.rollback() + except FileNotFoundError: + logging.error(f"SQL script not found: {script_path}") + finally: + # Clean up + cursor.close() + connection.close() + logging.info("Database connection closed.") + +if __name__ == "__main__": + if len(sys.argv) > 1: + filenames = sys.argv[1] + filenames = [file.strip() for file in filenames[2:-2].split(',')] + logging.info(filenames) + for file in filenames: + logging.info(f"Processing file: {file}") + file_path = f'./sql/active/{file}' + run_sql_script(file_path) + else: + logging.info("No filename provided.") + diff --git a/shared/bcts_adhoc_sql/sql/active/bcts_lrm_forest_ddl.sql b/shared/bcts_adhoc_sql/sql/active/bcts_lrm_forest_ddl.sql new file mode 100644 index 0000000..3d89f22 --- /dev/null +++ b/shared/bcts_adhoc_sql/sql/active/bcts_lrm_forest_ddl.sql @@ -0,0 +1,505 @@ +-- Create target tables +CREATE TABLE IF NOT EXISTS lrm_replication.division ( + divi_div_nbr NUMERIC(2) NOT NULL, + divi_country_code VARCHAR(40) NOT NULL, + divi_prov_state_code VARCHAR(40) NOT NULL, + divi_division_name VARCHAR(50) NOT NULL, + divi_startup_date TIMESTAMP NULL, + divi_end_date TIMESTAMP NULL, + divi_short_code VARCHAR(15) NULL, + divi_line1_addr VARCHAR(120) NULL, + divi_line2_addr VARCHAR(120) NULL, + divi_line3_addr VARCHAR(120) NULL, + divi_city_name VARCHAR(200) NULL, + divi_postal_code VARCHAR(40) NULL, + divi_intl_routing_code VARCHAR(60) NULL, + divi_phone_nbr VARCHAR(40) NULL, + divi_fax_nbr VARCHAR(40) NULL, + divi_stmpg_acctcd VARCHAR(200) NULL, + divi_stmpg_offset_acctcd VARCHAR(200) NULL, + divi_walker_entity_acctcd VARCHAR(60) NULL, + divi_gst_pay_acctcd VARCHAR(200) NULL, + divi_gst_recovery_acctd VARCHAR(200) NULL, + divi_operation_location_ind VARCHAR(4) NULL, + divi_client_location_code VARCHAR(40) NULL, + divi_abbreviation_code VARCHAR(16) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (divi_div_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.block_allocation ( + cutb_seq_nbr NUMERIC(15) NOT NULL, + blal_seq_nbr NUMERIC(15) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + licn_seq_nbr NUMERIC(15) NULL, + perm_seq_nbr NUMERIC(15) NULL, + mark_seq_nbr NUMERIC(15) NULL, + blal_gross_ha_area NUMERIC(11, 6) NULL, + blal_merch_ha_area NUMERIC(11, 6) NULL, + blal_cruise_m3_vol NUMERIC(15, 6) NULL, + blal_harvested_m3_vol NUMERIC(15, 6) NULL, + blal_harvested_ha_area NUMERIC(11, 6) NULL, + blal_firs_cutblock_id VARCHAR(80) NULL, + blal_firs_timbermark_id VARCHAR(80) NULL, + blal_estimated_area NUMERIC(7, 2) NULL, + manu_seq_nbr NUMERIC(15) NULL, + blal_digi_ind VARCHAR(1) NULL, + blal_primary_ind VARCHAR(4) NULL, + blal_actual_partition_vol NUMERIC(15, 2) NULL, + blal_usr_cruise_m3_vol NUMERIC(15, 6) NULL, + blal_rw_vol NUMERIC(15, 6) NULL, + blal_rw_ha_area NUMERIC(11, 6) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + blal_data_source VARCHAR(120) NULL, + PRIMARY KEY (blal_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.management_unit ( + manu_seq_nbr NUMERIC(15) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + manu_id VARCHAR(60) NOT NULL, + manu_name VARCHAR(200) NULL, + manu_comment VARCHAR(4000) NULL, + manu_type_id VARCHAR(40) NULL, + manu_operating_zone VARCHAR(40) NULL, + manu_mgr_seq_nbr NUMERIC(15) NULL, + manu_fst_seq_nbr NUMERIC(15) NULL, + manu_digitize_ind VARCHAR(20) NULL, + manu_area NUMERIC(11, 6) NULL, + manu_net_area NUMERIC(11, 6) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + nat_res_area VARCHAR(1) NULL, + nat_res_region VARCHAR(2) NULL, + manu_aac_effect_date TIMESTAMP NULL, + manu_aac NUMERIC(10, 2) NULL, + non_bcts_aac_ind NUMERIC(1) NULL, + PRIMARY KEY (manu_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.licence ( + licn_seq_nbr NUMERIC(15) NOT NULL, + licn_licence_id VARCHAR(15) NOT NULL, + licn_licence_desc VARCHAR(53) NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + licn_crown_land CHAR NULL, + licn_annual_allowable_cut NUMERIC(9) NULL, + lsee_licensee_id VARCHAR(10) NULL, + tent_seq_nbr NUMERIC(15) NULL, + licn_licence_state VARCHAR(20) NULL, + licn_permit_exists_ind VARCHAR(4) NULL, + licn_salvage_ind VARCHAR(1) NULL, + licn_category_id VARCHAR(10) NULL, + licn_field_team_id VARCHAR(10) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + blaz_admin_zone_id VARCHAR(10) NULL, + licn_digi_ind VARCHAR(1) NULL, + licl_licence_class VARCHAR(10) NULL, + licn_parent_licence NUMERIC(15) NULL, + licn_crown_granted_ind VARCHAR(4) NULL, + licn_client_loc_code VARCHAR(20) NULL, + licn_category_type VARCHAR(16) NULL, + licn_licence_to_cut_code VARCHAR(10) NULL, + linc_cert_level_id VARCHAR(10) NULL, + licn_mgr_seq_nbr NUMERIC(15) NULL, + licn_fst_seq_nbr NUMERIC(15) NULL, + licn_gross_area NUMERIC(11, 6) NULL, + licn_net_area NUMERIC(11, 6) NULL, + licn_comment VARCHAR(4000) NULL, + licn_apportion_tenure_type VARCHAR(30) NULL, + team_seq_nbr NUMERIC(15) NULL, + licn_hammermark VARCHAR(60) NULL, + licn_client_location_code VARCHAR(40) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + licn_archive_ind VARCHAR(3) NULL, + licn_archive_date TIMESTAMP NULL, + PRIMARY KEY (licn_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.block_admin_zone ( + blaz_admin_zone_id VARCHAR(40) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + blaz_admin_zone_desc VARCHAR(200) NULL, + blaz_active_ind VARCHAR(1) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (blaz_admin_zone_id, divi_div_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.division_code_lookup ( + colu_lookup_type VARCHAR(4) NOT NULL, + colu_lookup_id VARCHAR(120) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (colu_lookup_type, colu_lookup_id, divi_div_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.code_lookup ( + colu_lookup_type VARCHAR(4) NOT NULL, + colu_lookup_id VARCHAR(30) NOT NULL, + colu_lookup_desc VARCHAR(150) NOT NULL, + colu_user_defined_ind VARCHAR(4) NULL, + colu_display_ind VARCHAR(1) NULL, + colu_display_order NUMERIC(10) NULL, + colu_comment VARCHAR(4000) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + hq_display_ind VARCHAR(1) NULL, + PRIMARY KEY (colu_lookup_type, colu_lookup_id) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.tenure_type ( + tent_seq_nbr NUMERIC(15) NOT NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + tent_tenure_id VARCHAR(40) NOT NULL, + tent_tenure_name VARCHAR(160) NULL, + tety_tenure_type VARCHAR(40) NULL, + tent_active_ind VARCHAR(4) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (tent_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.cut_permit ( + perm_seq_nbr NUMERIC(15) NOT NULL, + regn_region_id VARCHAR(10) NULL, + perm_permit_id VARCHAR(40) NOT NULL, + perm_revision_date TIMESTAMP NULL, + papr_point_of_apprsl_id VARCHAR(16) NULL, + lsee_licensee_id VARCHAR(10) NULL, + admin_dsct_district_name VARCHAR(60) NULL, + geo_dsct_district_name VARCHAR(240) NULL, + perm_mgmt_unit_code VARCHAR(12) NULL, + perm_tsb_name VARCHAR(200) NULL, + perm_expiry_date TIMESTAMP NULL, + perm_remote_oper_ind VARCHAR(4) NULL, + perm_basic_silvi_ind VARCHAR(4) NULL, + perm_road_mtce_ind VARCHAR(4) NULL, + perm_road_land_use_chrg NUMERIC(15, 2) NULL, + perm_cost_eff_date TIMESTAMP NULL, + perm_rate_eff_date TIMESTAMP NULL, + perm_water_transport_ind VARCHAR(4) NULL, + perm_primary_block_seq_nbr NUMERIC(15) NULL, + perm_primary_mark_id VARCHAR(40) NULL, + perm_haul_ind VARCHAR(4) NULL, + tsar_tsa_nbr NUMERIC(3) NULL, + sblk_supply_block_id VARCHAR(40) NULL, + sptc_support_centre_name VARCHAR(50) NULL, + divi_div_nbr NUMERIC(2) NULL, + perm_damage_status VARCHAR(20) NULL, + perm_support_centre_dist NUMERIC(6, 2) NULL, + opty_opening_type_id VARCHAR(40) NULL, + pers_seq_nbr NUMERIC(15) NULL, + perm_permit_state VARCHAR(80) NULL, + perm_logprod_external_ind VARCHAR(4) NULL, + perm_logprod_customer_id VARCHAR(60) NULL, + perm_cruise_res_source VARCHAR(4) NULL, + perm_jurisdiction VARCHAR(40) NULL, + perm_application_description VARCHAR(4000) NULL, + perm_primary_species VARCHAR(40) NULL, + mkme_marking_method_code VARCHAR(40) NULL, + mkin_marking_instrument_code VARCHAR(40) NULL, + perm_prov_forest_conflict VARCHAR(40) NULL, + perm_cruise_based_ind VARCHAR(40) NULL, + perm_lsee_representative NUMERIC(15) NULL, + perm_traverse_start_point VARCHAR(160) NULL, + perm_traverse_end_point VARCHAR(160) NULL, + perm_digi_ind VARCHAR(4) NULL, + perm_bdt_conversion NUMERIC(15, 7) NULL, + cpcl_permit_class VARCHAR(40) NULL, + perm_parent_permit NUMERIC(15) NULL, + perm_spuc VARCHAR(40) NULL, + perm_salvage_ind VARCHAR(4) NULL, + perm_location VARCHAR(800) NULL, + perm_status VARCHAR(40) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + perm_fst_seq_nbr NUMERIC(15) NULL, + perm_bid_amount NUMERIC(10, 2) NULL, + perm_total_cost NUMERIC(10, 2) NULL, + perm_legal_description VARCHAR(400) NULL, + perm_gross_area NUMERIC(15, 7) NULL, + perm_high_lvl_pln VARCHAR(120) NULL, + perm_lcn_hvst_auth VARCHAR(120) NULL, + perm_lcn_reg CHAR NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (perm_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.mark ( + mark_seq_nbr NUMERIC(15) NOT NULL, + mark_mark_id VARCHAR(15) NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + mark_mark_desc VARCHAR(160) NULL, + mark_mark_state VARCHAR(80) NULL, + mark_crown_granted_ind VARCHAR(4) NULL, + mark_aac_partition VARCHAR(40) NULL, + mark_apportionment VARCHAR(40) NULL, + mark_endemic_percent NUMERIC(4, 2) NULL, + mark_species_type VARCHAR(8) NULL, + mark_all_log_grades_ind VARCHAR(4) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (mark_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.cut_block ( + cutb_seq_nbr NUMERIC(15) NOT NULL, + cutb_block_id VARCHAR(20) NOT NULL, + cutb_block_number VARCHAR(15) NULL, + divi_div_nbr NUMERIC(2) NULL, + cutb_devt_plan_status_code VARCHAR(8) NULL, + cutb_devt_plan_apprvl_date TIMESTAMP NULL, + cutb_silvi_plan_status_code VARCHAR(8) NULL, + cutb_silvi_plan_apprvl_date TIMESTAMP NULL, + cutb_block_memo VARCHAR(4000) NULL, + cutb_woods_run_km_nbr NUMERIC(15, 6) NULL, + cutb_hauling_km_nbr NUMERIC(15, 6) NULL, + cutb_hlcptr_drop_cruise_ind VARCHAR(4) NULL, + cutb_logging_plan_status_code VARCHAR(8) NULL, + cutb_logging_plan_apprvl_date TIMESTAMP NULL, + opar_operating_area_id VARCHAR(10) NULL, + bioz_zone_id VARCHAR(40) NULL, + cutb_gross_ha_area NUMERIC(11, 6) NULL, + cutb_bcgs VARCHAR(15) NULL, + cutb_opening VARCHAR(15) NULL, + cutb_nts VARCHAR(60) NULL, + cutb_ext_ha_area NUMERIC(11, 6) NULL, + cutb_photos VARCHAR(800) NULL, + cutb_field_work_by VARCHAR(200) NULL, + cutb_field_work_date TIMESTAMP NULL, + cutb_site_class VARCHAR(20) NULL, + fdps_status_id VARCHAR(10) NULL, + cutb_site_index NUMERIC(5) NULL, + cutb_cpi_slope_pct NUMERIC(7, 4) NULL, + cutb_cpi_road_access_km NUMERIC(13, 3) NULL, + cutb_cpi_road_block_km NUMERIC(13, 3) NULL, + cutb_cpi_road_num_landings NUMERIC(10) NULL, + cutb_cpi_road_haul_km NUMERIC(13, 3) NULL, + sblk_supply_block_id VARCHAR(10) NULL, + cutb_formc_harvest_print_date TIMESTAMP NULL, + cutb_formc_regen_print_date TIMESTAMP NULL, + cutb_formc_freegrow_print_date TIMESTAMP NULL, + fund_funding_code VARCHAR(15) NULL, + cutb_highway_ind VARCHAR(1) NULL, + cutb_winter_road_ind CHAR NULL, + finz_forest_inventory_zone_id VARCHAR(40) NULL, + cutb_user_mapsheet_id VARCHAR(120) NULL, + cutb_cell_number VARCHAR(60) NULL, + fjap_fdp_joint_approval VARCHAR(40) NULL, + pmod_modifier_id VARCHAR(40) NULL, + cutb_traverse_method_code VARCHAR(40) NULL, + cutb_critical_date_ind CHAR NULL, + cutb_greenup_date TIMESTAMP NULL, + grns_greenup_source VARCHAR(40) NULL, + pmpo_operating_zone VARCHAR(40) NULL, + lsun_landscape_unit VARCHAR(40) NULL, + plan_seq_nbr NUMERIC(15) NULL, + cutb_forma_print_date TIMESTAMP NULL, + cutb_block_state VARCHAR(20) NULL, + cutb_digi_ind VARCHAR(4) NULL, + sttp_stand_type VARCHAR(40) NULL, + ttac_timbertype_age_class VARCHAR(40) NULL, + ttsc_timbertype_stock_class VARCHAR(40) NULL, + tthc_timbertype_height_class VARCHAR(40) NULL, + cutb_site_plan_exempt_ind VARCHAR(4) NULL, + sssc_source_code VARCHAR(40) NULL, + cutb_vol_data_source VARCHAR(40) NULL, + cutb_vol_data_source_type VARCHAR(16) NULL, + cutb_heli_flight_dist NUMERIC(10) NULL, + cutb_harvest_sequence NUMERIC(6) NULL, + cutb_prov_forest_conflict VARCHAR(40) NULL, + treg_seq_nbr NUMERIC(15) NULL, + bcat_category_code VARCHAR(40) NULL, + cutb_traverse_start_point VARCHAR(160) NULL, + cutb_traverse_end_point VARCHAR(160) NULL, + cutb_forma_printed VARCHAR(4) NULL, + cutb_system_id VARCHAR(15) NULL, + cutb_block_status VARCHAR(40) NULL, + cutb_latitude_bak VARCHAR(40) NULL, + cutb_longitude_bak VARCHAR(40) NULL, + cutb_damage_type VARCHAR(40) NULL, + cutb_opening_id NUMERIC(10) NULL, + silp_use_block_num_ind VARCHAR(1) NOT NULL, + cutb_location VARCHAR(800) NULL, + cutb_silv_act_harv_link VARCHAR(120) NULL, + cutb_file_id VARCHAR(72) NULL, + suop_subop_area_id VARCHAR(40) NULL, + cutb_selling_price_period TIMESTAMP NULL, + cutb_synch_status VARCHAR(120) NULL, + cutb_block_grouping_id VARCHAR(80) NULL, + pers_seq_nbr NUMERIC(15) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + cutb_variant_id VARCHAR(60) NULL, + cutb_season_id VARCHAR(120) NULL, + cutb_volume NUMERIC(15, 6) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + cutb_parent_seq NUMERIC(15) NULL, + cutb_prev_forma_print_date TIMESTAMP NULL, + sila_seq_nbr NUMERIC(15) NULL, + cutb_cruise_lock_ind VARCHAR(4) NULL, + cutb_ref_dec CHAR NULL, + cutb_reg_code VARCHAR(12) NULL, + cutb_apr_eff_date TIMESTAMP NULL, + siph_seq_nbr NUMERIC(15) NULL, + cdat_seq_nbr INTEGER NULL, + cutb_access_restriction VARCHAR(4000) NULL, + cutb_planned_vol NUMERIC(15, 6) NULL, + cutb_row_ind VARCHAR(4) NOT NULL, + cutb_tim_dev_date TIMESTAMP NULL, + cutb_ebm_ind VARCHAR(4) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + documentkey NUMERIC(9) NULL, + log_start_cascade_date_ind VARCHAR(12) NULL, + pricing_date TIMESTAMP NULL, + cutb_latitude_dd NUMERIC(9, 6) NULL, + cutb_longitude_dd NUMERIC(9, 6) NULL, + cutb_latitude VARCHAR(40) NULL, + cutb_longitude VARCHAR(40) NULL, + marked_for_del_ind NUMERIC(1) NULL, + marked_for_del_by VARCHAR(120) NULL, + marked_for_del_on TIMESTAMP NULL, + cutb_archive_reason VARCHAR(4) NULL, + cutb_archive_date TIMESTAMP NULL, + cutb_cprp_protection_ind VARCHAR(1) NULL, + cutb_rc_risk_rating VARCHAR(1) NULL, + cutb_rc_risk_source VARCHAR(5) NULL, + cutb_rc_risk_comments VARCHAR(2000) NULL, + operational_site_factor VARCHAR(4000) NULL, + safety_concern VARCHAR(4000) NULL, + PRIMARY KEY (cutb_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.activity_class ( + accl_seq_nbr NUMERIC(15) NOT NULL, + accl_description VARCHAR(40) NULL, + accl_object_type VARCHAR(1) NULL, + accl_display_order NUMERIC(3) NULL, + divi_div_nbr NUMERIC(2) NULL, + accl_key_ind VARCHAR(12) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (accl_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.activity_type ( + actt_seq_nbr NUMERIC(15) NOT NULL, + accl_seq_nbr NUMERIC(15) NOT NULL, + actt_description VARCHAR(200) NULL, + divi_div_nbr NUMERIC(2) NOT NULL, + actt_default_ind VARCHAR(4) NULL, + actt_responsibility VARCHAR(160) NULL, + actt_display_order NUMERIC(2) NULL, + actt_key_ind VARCHAR(10) NULL, + actt_status_ind VARCHAR(4) NULL, + actt_date_relative VARCHAR(20) NULL, + ctor_contractor_id VARCHAR(40) NULL, + actt_view_level NUMERIC(2) NULL, + actt_system_ind VARCHAR(4) NULL, + ctor_seq_nbr NUMERIC(15) NULL, + actt_active_ind VARCHAR(4) NOT NULL, + actt_key_pair VARCHAR(40) NULL, + actt_harvs_ind VARCHAR(4) NULL, + actt_trvol_ind VARCHAR(4) NULL, + actt_indirect_cost_ind VARCHAR(8) NULL, + actt_default_cost NUMERIC(9, 2) NULL, + actt_cost_uom VARCHAR(120) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + PRIMARY KEY (actt_seq_nbr) +); + +CREATE TABLE IF NOT EXISTS lrm_replication.activity ( + acti_seq_nbr NUMERIC(15) NOT NULL, + cutb_seq_nbr NUMERIC(15) NULL, + perm_seq_nbr NUMERIC(15) NULL, + actt_seq_nbr NUMERIC(15) NOT NULL, + ctor_seq_nbr NUMERIC(15) NULL, + acti_status_ind VARCHAR(30) NULL, + acti_status_date TIMESTAMP NULL, + acti_comments VARCHAR(2000) NULL, + acti_cost NUMERIC(15, 2) NULL, + acti_harv_seas_id VARCHAR(40) NULL, + acti_responsibility VARCHAR(160) NULL, + acti_area NUMERIC(11, 6) NULL, + acti_harvest_vol NUMERIC(15, 6) NULL, + acti_int_reason VARCHAR(40) NULL, + acti_target_date TIMESTAMP NULL, + acti_fdp_reason VARCHAR(40) NULL, + acti_target_cost NUMERIC(15, 2) NULL, + plan_seq_nbr NUMERIC(15) NULL, + licn_seq_nbr NUMERIC(15) NULL, + mark_seq_nbr NUMERIC(15) NULL, + cloc_seq_nbr NUMERIC(15) NULL, + acti_digitized_ind VARCHAR(4) NULL, + acti_cost_uom VARCHAR(120) NULL, + modifiedby VARCHAR(120) NULL, + modifiedon TIMESTAMP NULL, + modifiedusing VARCHAR(120) NULL, + createdby VARCHAR(120) NULL, + createdon TIMESTAMP NULL, + createdusing VARCHAR(120) NULL, + documentkey NUMERIC(9) NULL, + accl_description VARCHAR(160) NULL, + PRIMARY KEY (acti_seq_nbr) +); + diff --git a/shared/bcts_adhoc_sql/sql/active/cdc_master_table_list.sql b/shared/bcts_adhoc_sql/sql/active/cdc_master_table_list.sql new file mode 100644 index 0000000..f8afb46 --- /dev/null +++ b/shared/bcts_adhoc_sql/sql/active/cdc_master_table_list.sql @@ -0,0 +1,29 @@ +DO $$ +DECLARE + tables text[] := ARRAY['DIVISION','BLOCK_ALLOCATION','MANAGEMENT_UNIT','LICENCE','BLOCK_ADMIN_ZONE','DIVISION_CODE_LOOKUP','CODE_LOOKUP','TENURE_TYPE','CUT_PERMIT','MARK','CUT_BLOCK','ACTIVITY_CLASS','ACTIVITY_TYPE','ACTIVITY']; + table_name text; +BEGIN + -- Loop through the list of table names + FOREACH table_name IN ARRAY tables + LOOP + INSERT INTO ods_data_management.cdc_master_table_list + VALUES ( + NULL, + 'lrm', + NULL, + 'forest', + table_name, + 'lrm_replication', + table_name, + 'Y', + NULL, + NULL, + NULL, + 'Y', + 1, + 'N', + NULL, + 'Oracle' + ); + END LOOP; +END $$; diff --git a/shared/bcts_adhoc_sql/sql/archived/placeholder.txt b/shared/bcts_adhoc_sql/sql/archived/placeholder.txt new file mode 100644 index 0000000..e69de29 From 781aa2d2e6fdb56db6df894f4b6f8d02bdb2f03e Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 10:42:44 -0800 Subject: [PATCH 07/23] Added github actions to build BCTS adhoc sql image --- .github/workflows/docker-bctsAdhocSql.yaml | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/docker-bctsAdhocSql.yaml diff --git a/.github/workflows/docker-bctsAdhocSql.yaml b/.github/workflows/docker-bctsAdhocSql.yaml new file mode 100644 index 0000000..1a1662a --- /dev/null +++ b/.github/workflows/docker-bctsAdhocSql.yaml @@ -0,0 +1,64 @@ +name: Push to GHCR + +on: + push: + branches: ["SD-128488-BCTS-ODS-GRANT-MANAGEMENT"] + +env: + REGISTRY: ghcr.io + DOCKERFILE_PATH: shared/bcts_adhoc_sql + IMAGE_NAME: ${{ github.repository }}-bctsAdhocSql + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@v3.3.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + # DF-NOTE: to help the action find the Dockerfile to build from + context: ${{ env.DOCKERFILE_PATH }}/ + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} From 6c643e95b3e8cb3eac2b6e019df7a4d3d3636a21 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 15:47:09 -0800 Subject: [PATCH 08/23] Added github actions to build bcts grant management --- .github/workflows/docker-bctsGrantMngmt.yaml | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/docker-bctsGrantMngmt.yaml diff --git a/.github/workflows/docker-bctsGrantMngmt.yaml b/.github/workflows/docker-bctsGrantMngmt.yaml new file mode 100644 index 0000000..1f2b70d --- /dev/null +++ b/.github/workflows/docker-bctsGrantMngmt.yaml @@ -0,0 +1,64 @@ +name: Push to GHCR + +on: + push: + branches: ["SD-128488-BCTS-ODS-GRANT-MANAGEMENT"] + +env: + REGISTRY: ghcr.io + DOCKERFILE_PATH: shared/bcts_access_management + IMAGE_NAME: ${{ github.repository }}-bctsGrantMngmt + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@v3.3.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + # DF-NOTE: to help the action find the Dockerfile to build from + context: ${{ env.DOCKERFILE_PATH }}/ + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} From a962fc9ec4a77b83ee85ce2a5f442b8aef6caa1b Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 16:05:44 -0800 Subject: [PATCH 09/23] Updated BCTS grant management file --- .../BCTS_ACCESS_CATALOG.csv | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv b/shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv index b6f6fb3..1561249 100644 --- a/shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv +++ b/shared/bcts_access_management/BCTS_ACCESS_CATALOG.csv @@ -1,31 +1,31 @@ -KEY,SCHEMA,TABLE_NAME,AUDIT_FOR_PI,BCTS_ETL_ROLE,BCTS_DEV_ROLE,BCTS_STAGE_ANALYST_ROLE,BCTS_STAGE_ANALYST_PI_ROLE,BCTS_ANALYST_ROLE,BCTS_ANALYST_PI_ROLE -1,LRM_REPLICATION,DIVISION,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -2,LRM_REPLICATION,BLOCK_ALLOCATION,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -3,LRM_REPLICATION,MANAGEMENT_UNIT,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -4,LRM_REPLICATION,LICENCE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -5,LRM_REPLICATION,BLOCK_ADMIN_ZONE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -6,LRM_REPLICATION,DIVISION_CODE_LOOKUP,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -7,LRM_REPLICATION,CODE_LOOKUP,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -8,LRM_REPLICATION,TENURE_TYPE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -9,LRM_REPLICATION,CUT_PERMIT,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -10,LRM_REPLICATION,MARK,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -11,LRM_REPLICATION,CUT_BLOCK,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -12,LRM_REPLICATION,ACTIVITY_CLASS,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -13,LRM_REPLICATION,ACTIVITY_TYPE,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -14,LRM_REPLICATION,ACTIVITY,No PI,Read/Write,Read/Write,Deny,Deny,Deny,Deny -15,BCTS_STAGING,DIVISION,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -16,BCTS_STAGING,BLOCK_ALLOCATION,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -17,BCTS_STAGING,MANAGEMENT_UNIT,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -18,BCTS_STAGING,LICENCE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -19,BCTS_STAGING,BLOCK_ADMIN_ZONE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -20,BCTS_STAGING,DIVISION_CODE_LOOKUP,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -21,BCTS_STAGING,CODE_LOOKUP,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -22,BCTS_STAGING,TENURE_TYPE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -23,BCTS_STAGING,CUT_PERMIT,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -24,BCTS_STAGING,MARK,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -25,BCTS_STAGING,CUT_BLOCK,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -26,BCTS_STAGING,ACTIVITY_CLASS,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -27,BCTS_STAGING,ACTIVITY_TYPE,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -28,BCTS_STAGING,ACTIVITY,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -29,BCTS_STAGING,ANNUAL_DEVELOPED_VOLUME,No PI,Read/Write,Read/Write,Read,Deny,Deny,Deny -30,BCTS_REPORTING,ANNUAL_DEVELOPED_VOLUME,No PI,Read/Write,Read/Write,Deny,Deny,Read,Deny +KEY,SCHEMA,TABLE_NAME,AUDIT_FOR_PI,BCTS_DEV_ROLE,BCTS_STAGE_ANALYST_ROLE,BCTS_STAGE_ANALYST_PI_ROLE,BCTS_ANALYST_ROLE,BCTS_ANALYST_PI_ROLE +1,LRM_REPLICATION,DIVISION,No PI,Read,Deny,Deny,Deny,Deny +2,LRM_REPLICATION,BLOCK_ALLOCATION,No PI,Read,Deny,Deny,Deny,Deny +3,LRM_REPLICATION,MANAGEMENT_UNIT,No PI,Read,Deny,Deny,Deny,Deny +4,LRM_REPLICATION,LICENCE,No PI,Read,Deny,Deny,Deny,Deny +5,LRM_REPLICATION,BLOCK_ADMIN_ZONE,No PI,Read,Deny,Deny,Deny,Deny +6,LRM_REPLICATION,DIVISION_CODE_LOOKUP,No PI,Read,Deny,Deny,Deny,Deny +7,LRM_REPLICATION,CODE_LOOKUP,No PI,Read,Deny,Deny,Deny,Deny +8,LRM_REPLICATION,TENURE_TYPE,No PI,Read,Deny,Deny,Deny,Deny +9,LRM_REPLICATION,CUT_PERMIT,No PI,Read,Deny,Deny,Deny,Deny +10,LRM_REPLICATION,MARK,No PI,Read,Deny,Deny,Deny,Deny +11,LRM_REPLICATION,CUT_BLOCK,No PI,Read,Deny,Deny,Deny,Deny +12,LRM_REPLICATION,ACTIVITY_CLASS,No PI,Read,Deny,Deny,Deny,Deny +13,LRM_REPLICATION,ACTIVITY_TYPE,No PI,Read,Deny,Deny,Deny,Deny +14,LRM_REPLICATION,ACTIVITY,No PI,Read,Deny,Deny,Deny,Deny +15,BCTS_STAGING,DIVISION,No PI,Read,Read,Deny,Deny,Deny +16,BCTS_STAGING,BLOCK_ALLOCATION,No PI,Read,Read,Deny,Deny,Deny +17,BCTS_STAGING,MANAGEMENT_UNIT,No PI,Read,Read,Deny,Deny,Deny +18,BCTS_STAGING,LICENCE,No PI,Read,Read,Deny,Deny,Deny +19,BCTS_STAGING,BLOCK_ADMIN_ZONE,No PI,Read,Read,Deny,Deny,Deny +20,BCTS_STAGING,DIVISION_CODE_LOOKUP,No PI,Read,Read,Deny,Deny,Deny +21,BCTS_STAGING,CODE_LOOKUP,No PI,Read,Read,Deny,Deny,Deny +22,BCTS_STAGING,TENURE_TYPE,No PI,Read,Read,Deny,Deny,Deny +23,BCTS_STAGING,CUT_PERMIT,No PI,Read,Read,Deny,Deny,Deny +24,BCTS_STAGING,MARK,No PI,Read,Read,Deny,Deny,Deny +25,BCTS_STAGING,CUT_BLOCK,No PI,Read,Read,Deny,Deny,Deny +26,BCTS_STAGING,ACTIVITY_CLASS,No PI,Read,Read,Deny,Deny,Deny +27,BCTS_STAGING,ACTIVITY_TYPE,No PI,Read,Read,Deny,Deny,Deny +28,BCTS_STAGING,ACTIVITY,No PI,Read,Read,Deny,Deny,Deny +29,BCTS_STAGING,ANNUAL_DEVELOPED_VOLUME,No PI,Read,Read,Deny,Deny,Deny +30,BCTS_REPORTING,ANNUAL_DEVELOPED_VOLUME,No PI,Read,Deny,Deny,Read,Deny From d7dadb7ae179dfe37b08bbdc969baf56a6ad5264 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 16:15:34 -0800 Subject: [PATCH 10/23] Updated BCTS grant management file --- shared/bcts_access_management/bcts_acces_apply_grants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shared/bcts_access_management/bcts_acces_apply_grants.py b/shared/bcts_access_management/bcts_acces_apply_grants.py index 5e3574b..3c22aed 100644 --- a/shared/bcts_access_management/bcts_acces_apply_grants.py +++ b/shared/bcts_access_management/bcts_acces_apply_grants.py @@ -40,11 +40,10 @@ def apply_grants(): all_statements = [] for _, row in df.iterrows(): all_statements.extend(generate_grant_statements(row)) - + logging.info(statement) # Execute the grant statements in PostgreSQL with engine.begin() as connection: for statement in all_statements: - logging.info(statement) connection.execute(text(statement)) logging.info("Grant statements executed successfully.") From 1de926993d521486a172a965dd4eda988a11e49b Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 16:20:06 -0800 Subject: [PATCH 11/23] Updated BCTS grant management file --- shared/bcts_access_management/bcts_acces_apply_grants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bcts_access_management/bcts_acces_apply_grants.py b/shared/bcts_access_management/bcts_acces_apply_grants.py index 3c22aed..aa90eeb 100644 --- a/shared/bcts_access_management/bcts_acces_apply_grants.py +++ b/shared/bcts_access_management/bcts_acces_apply_grants.py @@ -40,7 +40,7 @@ def apply_grants(): all_statements = [] for _, row in df.iterrows(): all_statements.extend(generate_grant_statements(row)) - logging.info(statement) + logging.info(all_statements) # Execute the grant statements in PostgreSQL with engine.begin() as connection: for statement in all_statements: From 4a182dfd40bef1fdddb205fe05b62ae3d33fce1d Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 16:24:51 -0800 Subject: [PATCH 12/23] Updated BCTS grant management file --- shared/bcts_access_management/bcts_acces_apply_grants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shared/bcts_access_management/bcts_acces_apply_grants.py b/shared/bcts_access_management/bcts_acces_apply_grants.py index aa90eeb..2854997 100644 --- a/shared/bcts_access_management/bcts_acces_apply_grants.py +++ b/shared/bcts_access_management/bcts_acces_apply_grants.py @@ -56,7 +56,6 @@ def generate_grant_statements(row): # List of roles and their permissions in the DataFrame role_permissions = { - 'BCTS_ETL_ROLE': row['BCTS_ETL_ROLE'], 'BCTS_DEV_ROLE': row['BCTS_DEV_ROLE'], 'BCTS_STAGE_ANALYST_ROLE': row['BCTS_STAGE_ANALYST_ROLE'], 'BCTS_STAGE_ANALYST_PI_ROLE': row['BCTS_STAGE_ANALYST_PI_ROLE'], From 344729cc648f9d51396b86205fc9c4671147aa40 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Fri, 15 Nov 2024 16:31:43 -0800 Subject: [PATCH 13/23] Added github actions for building BCTS transformations --- .../workflows/docker-bctsTransformations.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/docker-bctsTransformations.yaml diff --git a/.github/workflows/docker-bctsTransformations.yaml b/.github/workflows/docker-bctsTransformations.yaml new file mode 100644 index 0000000..fde994c --- /dev/null +++ b/.github/workflows/docker-bctsTransformations.yaml @@ -0,0 +1,64 @@ +name: Push to GHCR + +on: + push: + branches: ["SD-128488-BCTS-ODS-GRANT-MANAGEMENT"] + +env: + REGISTRY: ghcr.io + DOCKERFILE_PATH: shared/bcts_reports_etl + IMAGE_NAME: ${{ github.repository }}-bctsTransformations + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@v3.3.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + # DF-NOTE: to help the action find the Dockerfile to build from + context: ${{ env.DOCKERFILE_PATH }}/ + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} From afe6674a1475885ee3a775a784abedf77e8187e5 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Mon, 18 Nov 2024 09:28:45 -0800 Subject: [PATCH 14/23] Replaced staging tables to views to test transformation performance --- .../Annual_Developed_Volume_Query.sql | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql index ba6534b..a0f1e25 100644 --- a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql +++ b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql @@ -1,67 +1,67 @@ -DROP TABLE IF EXISTS bcts_staging.DIVISION; -CREATE TABLE bcts_staging.DIVISION AS +DROP VIEW IF EXISTS bcts_staging.DIVISION; +CREATE VIEW bcts_staging.DIVISION AS SELECT * FROM lrm_replication.DIVISION; -DROP TABLE IF EXISTS bcts_staging.BLOCK_ALLOCATION; -CREATE TABLE bcts_staging.BLOCK_ALLOCATION AS +DROP VIEW IF EXISTS bcts_staging.BLOCK_ALLOCATION; +CREATE VIEW bcts_staging.BLOCK_ALLOCATION AS SELECT * FROM lrm_replication.BLOCK_ALLOCATION; -DROP TABLE IF EXISTS bcts_staging.MANAGEMENT_UNIT; -CREATE TABLE bcts_staging.MANAGEMENT_UNIT AS +DROP VIEW IF EXISTS bcts_staging.MANAGEMENT_UNIT; +CREATE VIEW bcts_staging.MANAGEMENT_UNIT AS SELECT * FROM lrm_replication.MANAGEMENT_UNIT; -DROP TABLE IF EXISTS bcts_staging.LICENCE; -CREATE TABLE bcts_staging.LICENCE AS +DROP VIEW IF EXISTS bcts_staging.LICENCE; +CREATE VIEW bcts_staging.LICENCE AS SELECT * FROM lrm_replication.LICENCE; -DROP TABLE IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; -CREATE TABLE bcts_staging.BLOCK_ADMIN_ZONE AS +DROP VIEW IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; +CREATE VIEW bcts_staging.BLOCK_ADMIN_ZONE AS SELECT * FROM lrm_replication.BLOCK_ADMIN_ZONE; -DROP TABLE IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE TABLE bcts_staging.DIVISION_CODE_LOOKUP AS +DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; -DROP TABLE IF EXISTS bcts_staging.CODE_LOOKUP; -CREATE TABLE bcts_staging.CODE_LOOKUP AS +DROP VIEW IF EXISTS bcts_staging.CODE_LOOKUP; +CREATE VIEW bcts_staging.CODE_LOOKUP AS SELECT * FROM lrm_replication.CODE_LOOKUP; -DROP TABLE IF EXISTS bcts_staging.TENURE_TYPE; -CREATE TABLE bcts_staging.TENURE_TYPE AS +DROP VIEW IF EXISTS bcts_staging.TENURE_TYPE; +CREATE VIEW bcts_staging.TENURE_TYPE AS SELECT * FROM lrm_replication.TENURE_TYPE; -DROP TABLE IF EXISTS bcts_staging.CUT_PERMIT; -CREATE TABLE bcts_staging.CUT_PERMIT AS +DROP VIEW IF EXISTS bcts_staging.CUT_PERMIT; +CREATE VIEW bcts_staging.CUT_PERMIT AS SELECT * FROM lrm_replication.CUT_PERMIT; -DROP TABLE IF EXISTS bcts_staging.MARK; -CREATE TABLE bcts_staging.MARK AS +DROP VIEW IF EXISTS bcts_staging.MARK; +CREATE VIEW bcts_staging.MARK AS SELECT * FROM lrm_replication.MARK; -DROP TABLE IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE TABLE bcts_staging.DIVISION_CODE_LOOKUP AS +DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; -DROP TABLE IF EXISTS bcts_staging.CUT_BLOCK; -CREATE TABLE bcts_staging.CUT_BLOCK AS +DROP VIEW IF EXISTS bcts_staging.CUT_BLOCK; +CREATE VIEW bcts_staging.CUT_BLOCK AS SELECT * FROM lrm_replication.CUT_BLOCK; -DROP TABLE IF EXISTS bcts_staging.ACTIVITY_CLASS; -CREATE TABLE bcts_staging.ACTIVITY_CLASS AS +DROP VIEW IF EXISTS bcts_staging.ACTIVITY_CLASS; +CREATE VIEW bcts_staging.ACTIVITY_CLASS AS SELECT * FROM lrm_replication.ACTIVITY_CLASS; -DROP TABLE IF EXISTS bcts_staging.ACTIVITY_TYPE; -CREATE TABLE bcts_staging.ACTIVITY_TYPE AS +DROP VIEW IF EXISTS bcts_staging.ACTIVITY_TYPE; +CREATE VIEW bcts_staging.ACTIVITY_TYPE AS SELECT * FROM lrm_replication.ACTIVITY_TYPE; -DROP TABLE IF EXISTS bcts_staging.ACTIVITY; -CREATE TABLE bcts_staging.ACTIVITY AS +DROP VIEW IF EXISTS bcts_staging.ACTIVITY; +CREATE VIEW bcts_staging.ACTIVITY AS SELECT * FROM lrm_replication.ACTIVITY; -DROP TABLE IF EXISTS bcts_staging.annual_developed_volume; +DROP VIEW IF EXISTS bcts_staging.annual_developed_volume; -CREATE TABLE bcts_staging.annual_developed_volume AS +CREATE VIEW bcts_staging.annual_developed_volume AS WITH annual_developed_volume AS ( SELECT DISTINCT @@ -174,7 +174,7 @@ ORDER BY block ; -DROP TABLE IF EXISTS bcts_reporting.annual_developed_volume; +DROP VIEW IF EXISTS bcts_reporting.annual_developed_volume; CREATE TABLE bcts_reporting.annual_developed_volume AS - SELECT * FROM bcts_staging.annual_developed_volume; \ No newline at end of file + SELECT * FROM bcts_staging.annual_developed_volume; From e7a457efd608881ac1d2beca6efe0dfe118d7ec0 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Mon, 18 Nov 2024 09:39:48 -0800 Subject: [PATCH 15/23] Debugging LRM transformations --- .../Annual_Developed_Volume_Query copy.sql | 180 ++++++++++++++++++ .../Annual_Developed_Volume_Query.sql | 178 +---------------- 2 files changed, 182 insertions(+), 176 deletions(-) create mode 100644 shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql diff --git a/shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql b/shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql new file mode 100644 index 0000000..a0f1e25 --- /dev/null +++ b/shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql @@ -0,0 +1,180 @@ +DROP VIEW IF EXISTS bcts_staging.DIVISION; +CREATE VIEW bcts_staging.DIVISION AS + SELECT * FROM lrm_replication.DIVISION; + +DROP VIEW IF EXISTS bcts_staging.BLOCK_ALLOCATION; +CREATE VIEW bcts_staging.BLOCK_ALLOCATION AS + SELECT * FROM lrm_replication.BLOCK_ALLOCATION; + +DROP VIEW IF EXISTS bcts_staging.MANAGEMENT_UNIT; +CREATE VIEW bcts_staging.MANAGEMENT_UNIT AS + SELECT * FROM lrm_replication.MANAGEMENT_UNIT; + +DROP VIEW IF EXISTS bcts_staging.LICENCE; +CREATE VIEW bcts_staging.LICENCE AS + SELECT * FROM lrm_replication.LICENCE; + +DROP VIEW IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; +CREATE VIEW bcts_staging.BLOCK_ADMIN_ZONE AS + SELECT * FROM lrm_replication.BLOCK_ADMIN_ZONE; + +DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS + SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; + +DROP VIEW IF EXISTS bcts_staging.CODE_LOOKUP; +CREATE VIEW bcts_staging.CODE_LOOKUP AS + SELECT * FROM lrm_replication.CODE_LOOKUP; + +DROP VIEW IF EXISTS bcts_staging.TENURE_TYPE; +CREATE VIEW bcts_staging.TENURE_TYPE AS + SELECT * FROM lrm_replication.TENURE_TYPE; + +DROP VIEW IF EXISTS bcts_staging.CUT_PERMIT; +CREATE VIEW bcts_staging.CUT_PERMIT AS + SELECT * FROM lrm_replication.CUT_PERMIT; + +DROP VIEW IF EXISTS bcts_staging.MARK; +CREATE VIEW bcts_staging.MARK AS + SELECT * FROM lrm_replication.MARK; + +DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS + SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; + +DROP VIEW IF EXISTS bcts_staging.CUT_BLOCK; +CREATE VIEW bcts_staging.CUT_BLOCK AS + SELECT * FROM lrm_replication.CUT_BLOCK; + +DROP VIEW IF EXISTS bcts_staging.ACTIVITY_CLASS; +CREATE VIEW bcts_staging.ACTIVITY_CLASS AS + SELECT * FROM lrm_replication.ACTIVITY_CLASS; + +DROP VIEW IF EXISTS bcts_staging.ACTIVITY_TYPE; +CREATE VIEW bcts_staging.ACTIVITY_TYPE AS + SELECT * FROM lrm_replication.ACTIVITY_TYPE; + +DROP VIEW IF EXISTS bcts_staging.ACTIVITY; +CREATE VIEW bcts_staging.ACTIVITY AS + SELECT * FROM lrm_replication.ACTIVITY; + + +DROP VIEW IF EXISTS bcts_staging.annual_developed_volume; + +CREATE VIEW bcts_staging.annual_developed_volume AS +WITH annual_developed_volume AS +( + SELECT DISTINCT + CASE + WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN', + 'TCC', 'TKA', 'TKO', 'TOC' ) THEN + 'Interior' + WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN + 'Coast' + END AS business_area_region_category, + CASE + WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN' ) THEN + 'North Interior' + WHEN d.divi_short_code IN ( 'TCC', 'TKA', 'TKO', 'TOC' ) THEN + 'South Interior' + WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN + 'Coast' + END AS business_area_region, + CASE WHEN d.divi_division_name = 'Seaward' THEN 'Seaward-Tlasta' ELSE d.divi_division_name END + || ' (' + || d.divi_short_code + || ')' AS business_area, + d.divi_short_code AS business_area_code, + cl.colu_lookup_desc AS "Field Team", + mu.manu_id, + l.licn_licence_id AS licence, + tn.tent_tenure_id AS "File Type", + l.blaz_admin_zone_id AS agreement_type_code, + z.blaz_admin_zone_desc AS agreement_type, + cp.perm_permit_id AS permit, + m.mark_mark_id AS mark, + b.cutb_block_id AS block, + b.cutb_system_id AS ubi, + b.cutb_block_state AS block_state, + ba.blal_cruise_m3_vol AS cruise_volume, + ba.blal_rw_vol AS rw_volume, + actb.rc_done, + EXTRACT(YEAR FROM (actb.rc_done + INTERVAL '9 months')) AS rc_done_fiscal, + actb.dr_done, + EXTRACT(YEAR FROM (actb.dr_done + INTERVAL '9 months')) AS dr_done_fiscal, + actb.dvs_done, + EXTRACT(YEAR FROM (actb.dvs_done + INTERVAL '9 months')) AS dvs_done_fiscal, + actb.dvc_done, + EXTRACT(YEAR FROM (actb.dvc_done + INTERVAL '9 months')) AS dvc_done_fiscal, + b.cutb_seq_nbr + FROM + lrm_replication.division d + INNER JOIN bcts_staging.block_allocation ba + ON d.divi_div_nbr = ba.divi_div_nbr + INNER JOIN bcts_staging.management_unit mu + ON ba.manu_seq_nbr = mu.manu_seq_nbr + INNER JOIN bcts_staging.licence l + ON ba.licn_seq_nbr = l.licn_seq_nbr + LEFT OUTER JOIN bcts_staging.block_admin_zone z + ON l.divi_div_nbr = z.divi_div_nbr + AND l.blaz_admin_zone_id = z.blaz_admin_zone_id + AND ba.licn_seq_nbr = l.licn_seq_nbr + AND l.divi_div_nbr = z.divi_div_nbr + AND l.blaz_admin_zone_id = z.blaz_admin_zone_id + LEFT OUTER JOIN bcts_staging.division_code_lookup dcl + ON l.licn_field_team_id = dcl.colu_lookup_id + AND l.divi_div_nbr = dcl.divi_div_nbr + LEFT OUTER JOIN bcts_staging.code_lookup cl + ON dcl.colu_lookup_type = cl.colu_lookup_type + AND dcl.colu_lookup_id = cl.colu_lookup_id + LEFT JOIN bcts_staging.tenure_type tn + ON l.tent_seq_nbr = tn.tent_seq_nbr + LEFT OUTER JOIN bcts_staging.cut_permit cp + ON ba.perm_seq_nbr = cp.perm_seq_nbr + LEFT JOIN bcts_staging.mark m + ON ba.mark_seq_nbr = m.mark_seq_nbr + INNER JOIN bcts_staging.cut_block b + ON ba.cutb_seq_nbr = b.cutb_seq_nbr + INNER JOIN + ( + SELECT + A.CUTB_SEQ_NBR, + MAX(CASE WHEN ACTT_KEY_IND = 'RC' THEN DATE_TRUNC('DAY', ACTI_STATUS_DATE) END)::DATE AS RC_Done, + MAX(CASE WHEN ACTT_KEY_IND = 'DR' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DR_Done, + MAX(CASE WHEN ACTT_KEY_IND = 'DVS' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVS_Done, + MAX(CASE WHEN ACTT_KEY_IND = 'DVC' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVC_Done + FROM + bcts_staging.ACTIVITY_CLASS C, + bcts_staging.ACTIVITY_TYPE T, + bcts_staging.ACTIVITY A + WHERE + C.ACCL_SEQ_NBR = T.ACCL_SEQ_NBR + AND T.ACTT_SEQ_NBR = A.ACTT_SEQ_NBR + AND C.ACCL_DESCRIPTION = 'CMB' + AND T.ACTT_KEY_IND In ('RC', 'DR', 'DVS', 'DVC') + AND A.ACTI_STATUS_IND = 'D' + GROUP BY A.CUTB_SEQ_NBR, T.ACTT_KEY_IND + ) ACTB + ON ba.cutb_seq_nbr = actb.cutb_seq_nbr + AND actb.dvc_done BETWEEN TO_DATE('2024-04-01', 'YYYY-MM-DD') -- Date: beginning of current fiscal + AND TO_DATE('2024-09-30', 'YYYY-MM-DD') -- Date: end of reporting period +) + +SELECT * +FROM annual_developed_volume +ORDER BY + length(business_area_region) desc, + business_area_region, + business_area, + "Field Team", + MANU_ID, + licence, + permit, + mark, + block +; + +DROP VIEW IF EXISTS bcts_reporting.annual_developed_volume; + +CREATE TABLE bcts_reporting.annual_developed_volume AS + SELECT * FROM bcts_staging.annual_developed_volume; diff --git a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql index a0f1e25..e433a65 100644 --- a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql +++ b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql @@ -1,180 +1,6 @@ -DROP VIEW IF EXISTS bcts_staging.DIVISION; -CREATE VIEW bcts_staging.DIVISION AS - SELECT * FROM lrm_replication.DIVISION; -DROP VIEW IF EXISTS bcts_staging.BLOCK_ALLOCATION; -CREATE VIEW bcts_staging.BLOCK_ALLOCATION AS - SELECT * FROM lrm_replication.BLOCK_ALLOCATION; +SELECT 1 FROM lrm_replication.DIVISION; -DROP VIEW IF EXISTS bcts_staging.MANAGEMENT_UNIT; -CREATE VIEW bcts_staging.MANAGEMENT_UNIT AS - SELECT * FROM lrm_replication.MANAGEMENT_UNIT; -DROP VIEW IF EXISTS bcts_staging.LICENCE; -CREATE VIEW bcts_staging.LICENCE AS - SELECT * FROM lrm_replication.LICENCE; +SELECT 2 FROM lrm_replication.BLOCK_ALLOCATION; -DROP VIEW IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; -CREATE VIEW bcts_staging.BLOCK_ADMIN_ZONE AS - SELECT * FROM lrm_replication.BLOCK_ADMIN_ZONE; - -DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS - SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; - -DROP VIEW IF EXISTS bcts_staging.CODE_LOOKUP; -CREATE VIEW bcts_staging.CODE_LOOKUP AS - SELECT * FROM lrm_replication.CODE_LOOKUP; - -DROP VIEW IF EXISTS bcts_staging.TENURE_TYPE; -CREATE VIEW bcts_staging.TENURE_TYPE AS - SELECT * FROM lrm_replication.TENURE_TYPE; - -DROP VIEW IF EXISTS bcts_staging.CUT_PERMIT; -CREATE VIEW bcts_staging.CUT_PERMIT AS - SELECT * FROM lrm_replication.CUT_PERMIT; - -DROP VIEW IF EXISTS bcts_staging.MARK; -CREATE VIEW bcts_staging.MARK AS - SELECT * FROM lrm_replication.MARK; - -DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS - SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; - -DROP VIEW IF EXISTS bcts_staging.CUT_BLOCK; -CREATE VIEW bcts_staging.CUT_BLOCK AS - SELECT * FROM lrm_replication.CUT_BLOCK; - -DROP VIEW IF EXISTS bcts_staging.ACTIVITY_CLASS; -CREATE VIEW bcts_staging.ACTIVITY_CLASS AS - SELECT * FROM lrm_replication.ACTIVITY_CLASS; - -DROP VIEW IF EXISTS bcts_staging.ACTIVITY_TYPE; -CREATE VIEW bcts_staging.ACTIVITY_TYPE AS - SELECT * FROM lrm_replication.ACTIVITY_TYPE; - -DROP VIEW IF EXISTS bcts_staging.ACTIVITY; -CREATE VIEW bcts_staging.ACTIVITY AS - SELECT * FROM lrm_replication.ACTIVITY; - - -DROP VIEW IF EXISTS bcts_staging.annual_developed_volume; - -CREATE VIEW bcts_staging.annual_developed_volume AS -WITH annual_developed_volume AS -( - SELECT DISTINCT - CASE - WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN', - 'TCC', 'TKA', 'TKO', 'TOC' ) THEN - 'Interior' - WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN - 'Coast' - END AS business_area_region_category, - CASE - WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN' ) THEN - 'North Interior' - WHEN d.divi_short_code IN ( 'TCC', 'TKA', 'TKO', 'TOC' ) THEN - 'South Interior' - WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN - 'Coast' - END AS business_area_region, - CASE WHEN d.divi_division_name = 'Seaward' THEN 'Seaward-Tlasta' ELSE d.divi_division_name END - || ' (' - || d.divi_short_code - || ')' AS business_area, - d.divi_short_code AS business_area_code, - cl.colu_lookup_desc AS "Field Team", - mu.manu_id, - l.licn_licence_id AS licence, - tn.tent_tenure_id AS "File Type", - l.blaz_admin_zone_id AS agreement_type_code, - z.blaz_admin_zone_desc AS agreement_type, - cp.perm_permit_id AS permit, - m.mark_mark_id AS mark, - b.cutb_block_id AS block, - b.cutb_system_id AS ubi, - b.cutb_block_state AS block_state, - ba.blal_cruise_m3_vol AS cruise_volume, - ba.blal_rw_vol AS rw_volume, - actb.rc_done, - EXTRACT(YEAR FROM (actb.rc_done + INTERVAL '9 months')) AS rc_done_fiscal, - actb.dr_done, - EXTRACT(YEAR FROM (actb.dr_done + INTERVAL '9 months')) AS dr_done_fiscal, - actb.dvs_done, - EXTRACT(YEAR FROM (actb.dvs_done + INTERVAL '9 months')) AS dvs_done_fiscal, - actb.dvc_done, - EXTRACT(YEAR FROM (actb.dvc_done + INTERVAL '9 months')) AS dvc_done_fiscal, - b.cutb_seq_nbr - FROM - lrm_replication.division d - INNER JOIN bcts_staging.block_allocation ba - ON d.divi_div_nbr = ba.divi_div_nbr - INNER JOIN bcts_staging.management_unit mu - ON ba.manu_seq_nbr = mu.manu_seq_nbr - INNER JOIN bcts_staging.licence l - ON ba.licn_seq_nbr = l.licn_seq_nbr - LEFT OUTER JOIN bcts_staging.block_admin_zone z - ON l.divi_div_nbr = z.divi_div_nbr - AND l.blaz_admin_zone_id = z.blaz_admin_zone_id - AND ba.licn_seq_nbr = l.licn_seq_nbr - AND l.divi_div_nbr = z.divi_div_nbr - AND l.blaz_admin_zone_id = z.blaz_admin_zone_id - LEFT OUTER JOIN bcts_staging.division_code_lookup dcl - ON l.licn_field_team_id = dcl.colu_lookup_id - AND l.divi_div_nbr = dcl.divi_div_nbr - LEFT OUTER JOIN bcts_staging.code_lookup cl - ON dcl.colu_lookup_type = cl.colu_lookup_type - AND dcl.colu_lookup_id = cl.colu_lookup_id - LEFT JOIN bcts_staging.tenure_type tn - ON l.tent_seq_nbr = tn.tent_seq_nbr - LEFT OUTER JOIN bcts_staging.cut_permit cp - ON ba.perm_seq_nbr = cp.perm_seq_nbr - LEFT JOIN bcts_staging.mark m - ON ba.mark_seq_nbr = m.mark_seq_nbr - INNER JOIN bcts_staging.cut_block b - ON ba.cutb_seq_nbr = b.cutb_seq_nbr - INNER JOIN - ( - SELECT - A.CUTB_SEQ_NBR, - MAX(CASE WHEN ACTT_KEY_IND = 'RC' THEN DATE_TRUNC('DAY', ACTI_STATUS_DATE) END)::DATE AS RC_Done, - MAX(CASE WHEN ACTT_KEY_IND = 'DR' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DR_Done, - MAX(CASE WHEN ACTT_KEY_IND = 'DVS' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVS_Done, - MAX(CASE WHEN ACTT_KEY_IND = 'DVC' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVC_Done - FROM - bcts_staging.ACTIVITY_CLASS C, - bcts_staging.ACTIVITY_TYPE T, - bcts_staging.ACTIVITY A - WHERE - C.ACCL_SEQ_NBR = T.ACCL_SEQ_NBR - AND T.ACTT_SEQ_NBR = A.ACTT_SEQ_NBR - AND C.ACCL_DESCRIPTION = 'CMB' - AND T.ACTT_KEY_IND In ('RC', 'DR', 'DVS', 'DVC') - AND A.ACTI_STATUS_IND = 'D' - GROUP BY A.CUTB_SEQ_NBR, T.ACTT_KEY_IND - ) ACTB - ON ba.cutb_seq_nbr = actb.cutb_seq_nbr - AND actb.dvc_done BETWEEN TO_DATE('2024-04-01', 'YYYY-MM-DD') -- Date: beginning of current fiscal - AND TO_DATE('2024-09-30', 'YYYY-MM-DD') -- Date: end of reporting period -) - -SELECT * -FROM annual_developed_volume -ORDER BY - length(business_area_region) desc, - business_area_region, - business_area, - "Field Team", - MANU_ID, - licence, - permit, - mark, - block -; - -DROP VIEW IF EXISTS bcts_reporting.annual_developed_volume; - -CREATE TABLE bcts_reporting.annual_developed_volume AS - SELECT * FROM bcts_staging.annual_developed_volume; From 52975a6927392458430d01efbd3f9999ce16ba3b Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Mon, 18 Nov 2024 09:43:31 -0800 Subject: [PATCH 16/23] Debugging LRM transformations --- .../Annual_Developed_Volume_Query.sql | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql index e433a65..e6c95fd 100644 --- a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql +++ b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql @@ -1,6 +1,63 @@ +DROP VIEW IF EXISTS bcts_staging.DIVISION; +CREATE VIEW bcts_staging.DIVISION AS + SELECT 1 FROM lrm_replication.DIVISION; -SELECT 1 FROM lrm_replication.DIVISION; +DROP VIEW IF EXISTS bcts_staging.BLOCK_ALLOCATION; +CREATE VIEW bcts_staging.BLOCK_ALLOCATION AS + SELECT * FROM lrm_replication.BLOCK_ALLOCATION; +DROP VIEW IF EXISTS bcts_staging.MANAGEMENT_UNIT; +CREATE VIEW bcts_staging.MANAGEMENT_UNIT AS + SELECT * FROM lrm_replication.MANAGEMENT_UNIT; -SELECT 2 FROM lrm_replication.BLOCK_ALLOCATION; +DROP VIEW IF EXISTS bcts_staging.LICENCE; +CREATE VIEW bcts_staging.LICENCE AS + SELECT * FROM lrm_replication.LICENCE; + +DROP VIEW IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; +CREATE VIEW bcts_staging.BLOCK_ADMIN_ZONE AS + SELECT * FROM lrm_replication.BLOCK_ADMIN_ZONE; + +DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS + SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; + +DROP VIEW IF EXISTS bcts_staging.CODE_LOOKUP; +CREATE VIEW bcts_staging.CODE_LOOKUP AS + SELECT * FROM lrm_replication.CODE_LOOKUP; + +DROP VIEW IF EXISTS bcts_staging.TENURE_TYPE; +CREATE VIEW bcts_staging.TENURE_TYPE AS + SELECT * FROM lrm_replication.TENURE_TYPE; + +DROP VIEW IF EXISTS bcts_staging.CUT_PERMIT; +CREATE VIEW bcts_staging.CUT_PERMIT AS + SELECT * FROM lrm_replication.CUT_PERMIT; + +DROP VIEW IF EXISTS bcts_staging.MARK; +CREATE VIEW bcts_staging.MARK AS + SELECT * FROM lrm_replication.MARK; + +DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; +CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS + SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; + +DROP VIEW IF EXISTS bcts_staging.CUT_BLOCK; +CREATE VIEW bcts_staging.CUT_BLOCK AS + SELECT * FROM lrm_replication.CUT_BLOCK; + +DROP VIEW IF EXISTS bcts_staging.ACTIVITY_CLASS; +CREATE VIEW bcts_staging.ACTIVITY_CLASS AS + SELECT * FROM lrm_replication.ACTIVITY_CLASS; + +DROP VIEW IF EXISTS bcts_staging.ACTIVITY_TYPE; +CREATE VIEW bcts_staging.ACTIVITY_TYPE AS + SELECT * FROM lrm_replication.ACTIVITY_TYPE; + +DROP VIEW IF EXISTS bcts_staging.ACTIVITY; +CREATE VIEW bcts_staging.ACTIVITY AS + SELECT * FROM lrm_replication.ACTIVITY; + + +DROP VIEW IF EXISTS bcts_staging.annual_developed_volume; From fd5fe4eb348ce86dccc1e685f726347aa12cc8ba Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Mon, 18 Nov 2024 09:59:34 -0800 Subject: [PATCH 17/23] Debugging LRM transformations --- .../Annual_Developed_Volume_Query.sql | 166 ++++++++++++++---- 1 file changed, 133 insertions(+), 33 deletions(-) diff --git a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql index e6c95fd..75d444f 100644 --- a/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql +++ b/shared/bcts_reports_etl/Annual_Developed_Volume_Query.sql @@ -1,63 +1,163 @@ -DROP VIEW IF EXISTS bcts_staging.DIVISION; -CREATE VIEW bcts_staging.DIVISION AS - SELECT 1 FROM lrm_replication.DIVISION; +CREATE OR REPLACE VIEW bcts_staging.DIVISION AS + SELECT * FROM lrm_replication.DIVISION; -DROP VIEW IF EXISTS bcts_staging.BLOCK_ALLOCATION; -CREATE VIEW bcts_staging.BLOCK_ALLOCATION AS +CREATE OR REPLACE VIEW bcts_staging.BLOCK_ALLOCATION AS SELECT * FROM lrm_replication.BLOCK_ALLOCATION; -DROP VIEW IF EXISTS bcts_staging.MANAGEMENT_UNIT; -CREATE VIEW bcts_staging.MANAGEMENT_UNIT AS +CREATE OR REPLACE VIEW bcts_staging.MANAGEMENT_UNIT AS SELECT * FROM lrm_replication.MANAGEMENT_UNIT; -DROP VIEW IF EXISTS bcts_staging.LICENCE; -CREATE VIEW bcts_staging.LICENCE AS +CREATE OR REPLACE VIEW bcts_staging.LICENCE AS SELECT * FROM lrm_replication.LICENCE; -DROP VIEW IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; -CREATE VIEW bcts_staging.BLOCK_ADMIN_ZONE AS +CREATE OR REPLACE VIEW bcts_staging.BLOCK_ADMIN_ZONE AS SELECT * FROM lrm_replication.BLOCK_ADMIN_ZONE; -DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS +CREATE OR REPLACE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; -DROP VIEW IF EXISTS bcts_staging.CODE_LOOKUP; -CREATE VIEW bcts_staging.CODE_LOOKUP AS +CREATE OR REPLACE VIEW bcts_staging.CODE_LOOKUP AS SELECT * FROM lrm_replication.CODE_LOOKUP; -DROP VIEW IF EXISTS bcts_staging.TENURE_TYPE; -CREATE VIEW bcts_staging.TENURE_TYPE AS +CREATE OR REPLACE VIEW bcts_staging.TENURE_TYPE AS SELECT * FROM lrm_replication.TENURE_TYPE; -DROP VIEW IF EXISTS bcts_staging.CUT_PERMIT; -CREATE VIEW bcts_staging.CUT_PERMIT AS +CREATE OR REPLACE VIEW bcts_staging.CUT_PERMIT AS SELECT * FROM lrm_replication.CUT_PERMIT; -DROP VIEW IF EXISTS bcts_staging.MARK; -CREATE VIEW bcts_staging.MARK AS +CREATE OR REPLACE VIEW bcts_staging.MARK AS SELECT * FROM lrm_replication.MARK; -DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS +CREATE OR REPLACE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; -DROP VIEW IF EXISTS bcts_staging.CUT_BLOCK; -CREATE VIEW bcts_staging.CUT_BLOCK AS +CREATE OR REPLACE VIEW bcts_staging.CUT_BLOCK AS SELECT * FROM lrm_replication.CUT_BLOCK; -DROP VIEW IF EXISTS bcts_staging.ACTIVITY_CLASS; -CREATE VIEW bcts_staging.ACTIVITY_CLASS AS +CREATE OR REPLACE VIEW bcts_staging.ACTIVITY_CLASS AS SELECT * FROM lrm_replication.ACTIVITY_CLASS; -DROP VIEW IF EXISTS bcts_staging.ACTIVITY_TYPE; -CREATE VIEW bcts_staging.ACTIVITY_TYPE AS +CREATE OR REPLACE VIEW bcts_staging.ACTIVITY_TYPE AS SELECT * FROM lrm_replication.ACTIVITY_TYPE; -DROP VIEW IF EXISTS bcts_staging.ACTIVITY; -CREATE VIEW bcts_staging.ACTIVITY AS +CREATE OR REPLACE VIEW bcts_staging.ACTIVITY AS SELECT * FROM lrm_replication.ACTIVITY; -DROP VIEW IF EXISTS bcts_staging.annual_developed_volume; - +CREATE OR REPLACE VIEW bcts_staging.annual_developed_volume AS +WITH annual_developed_volume AS +( + SELECT DISTINCT + CASE + WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN', + 'TCC', 'TKA', 'TKO', 'TOC' ) THEN + 'Interior' + WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN + 'Coast' + END AS business_area_region_category, + CASE + WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN' ) THEN + 'North Interior' + WHEN d.divi_short_code IN ( 'TCC', 'TKA', 'TKO', 'TOC' ) THEN + 'South Interior' + WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN + 'Coast' + END AS business_area_region, + CASE WHEN d.divi_division_name = 'Seaward' THEN 'Seaward-Tlasta' ELSE d.divi_division_name END + || ' (' + || d.divi_short_code + || ')' AS business_area, + d.divi_short_code AS business_area_code, + cl.colu_lookup_desc AS "Field Team", + mu.manu_id, + l.licn_licence_id AS licence, + tn.tent_tenure_id AS "File Type", + l.blaz_admin_zone_id AS agreement_type_code, + z.blaz_admin_zone_desc AS agreement_type, + cp.perm_permit_id AS permit, + m.mark_mark_id AS mark, + b.cutb_block_id AS block, + b.cutb_system_id AS ubi, + b.cutb_block_state AS block_state, + ba.blal_cruise_m3_vol AS cruise_volume, + ba.blal_rw_vol AS rw_volume, + actb.rc_done, + EXTRACT(YEAR FROM (actb.rc_done + INTERVAL '9 months')) AS rc_done_fiscal, + actb.dr_done, + EXTRACT(YEAR FROM (actb.dr_done + INTERVAL '9 months')) AS dr_done_fiscal, + actb.dvs_done, + EXTRACT(YEAR FROM (actb.dvs_done + INTERVAL '9 months')) AS dvs_done_fiscal, + actb.dvc_done, + EXTRACT(YEAR FROM (actb.dvc_done + INTERVAL '9 months')) AS dvc_done_fiscal, + b.cutb_seq_nbr + FROM + lrm_replication.division d + INNER JOIN bcts_staging.block_allocation ba + ON d.divi_div_nbr = ba.divi_div_nbr + INNER JOIN bcts_staging.management_unit mu + ON ba.manu_seq_nbr = mu.manu_seq_nbr + INNER JOIN bcts_staging.licence l + ON ba.licn_seq_nbr = l.licn_seq_nbr + LEFT OUTER JOIN bcts_staging.block_admin_zone z + ON l.divi_div_nbr = z.divi_div_nbr + AND l.blaz_admin_zone_id = z.blaz_admin_zone_id + AND ba.licn_seq_nbr = l.licn_seq_nbr + AND l.divi_div_nbr = z.divi_div_nbr + AND l.blaz_admin_zone_id = z.blaz_admin_zone_id + LEFT OUTER JOIN bcts_staging.division_code_lookup dcl + ON l.licn_field_team_id = dcl.colu_lookup_id + AND l.divi_div_nbr = dcl.divi_div_nbr + LEFT OUTER JOIN bcts_staging.code_lookup cl + ON dcl.colu_lookup_type = cl.colu_lookup_type + AND dcl.colu_lookup_id = cl.colu_lookup_id + LEFT JOIN bcts_staging.tenure_type tn + ON l.tent_seq_nbr = tn.tent_seq_nbr + LEFT OUTER JOIN bcts_staging.cut_permit cp + ON ba.perm_seq_nbr = cp.perm_seq_nbr + LEFT JOIN bcts_staging.mark m + ON ba.mark_seq_nbr = m.mark_seq_nbr + INNER JOIN bcts_staging.cut_block b + ON ba.cutb_seq_nbr = b.cutb_seq_nbr + INNER JOIN + ( + SELECT + A.CUTB_SEQ_NBR, + MAX(CASE WHEN ACTT_KEY_IND = 'RC' THEN DATE_TRUNC('DAY', ACTI_STATUS_DATE) END)::DATE AS RC_Done, + MAX(CASE WHEN ACTT_KEY_IND = 'DR' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DR_Done, + MAX(CASE WHEN ACTT_KEY_IND = 'DVS' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVS_Done, + MAX(CASE WHEN ACTT_KEY_IND = 'DVC' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVC_Done + FROM + bcts_staging.ACTIVITY_CLASS C, + bcts_staging.ACTIVITY_TYPE T, + bcts_staging.ACTIVITY A + WHERE + C.ACCL_SEQ_NBR = T.ACCL_SEQ_NBR + AND T.ACTT_SEQ_NBR = A.ACTT_SEQ_NBR + AND C.ACCL_DESCRIPTION = 'CMB' + AND T.ACTT_KEY_IND In ('RC', 'DR', 'DVS', 'DVC') + AND A.ACTI_STATUS_IND = 'D' + GROUP BY A.CUTB_SEQ_NBR, T.ACTT_KEY_IND + ) ACTB + ON ba.cutb_seq_nbr = actb.cutb_seq_nbr + AND actb.dvc_done BETWEEN TO_DATE('2024-04-01', 'YYYY-MM-DD') -- Date: beginning of current fiscal + AND TO_DATE('2024-09-30', 'YYYY-MM-DD') -- Date: end of reporting period +) + +SELECT * +FROM annual_developed_volume +ORDER BY + length(business_area_region) desc, + business_area_region, + business_area, + "Field Team", + MANU_ID, + licence, + permit, + mark, + block +; + +DROP TABLE IF EXISTS bcts_reporting.annual_developed_volume; + +CREATE TABLE bcts_reporting.annual_developed_volume AS + SELECT * FROM bcts_staging.annual_developed_volume; From ad2bad5d565b6e59530133d419dad57503bb6659 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Mon, 18 Nov 2024 10:28:46 -0800 Subject: [PATCH 18/23] Debugging LRM transformations --- shared/bcts_reports_etl/bcts_etl.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shared/bcts_reports_etl/bcts_etl.py b/shared/bcts_reports_etl/bcts_etl.py index cb9a0c2..ab241ec 100644 --- a/shared/bcts_reports_etl/bcts_etl.py +++ b/shared/bcts_reports_etl/bcts_etl.py @@ -45,9 +45,10 @@ def run_sql_script(script_path): try: with open(script_path, 'r') as file: sql_script = file.read() - - # Execute the SQL script - cursor.execute(sql_script) + sql_statements = sql_script.split(';') + for statement in sql_statements: + if statement.strip(): + cursor.execute(statement.strip()) connection.commit() logging.info(f"SQL script executed successfully from {script_path}.") except psycopg2.Error as e: From ec7b132b82cf28bf37899d2f940463d52f151051 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Mon, 18 Nov 2024 10:34:53 -0800 Subject: [PATCH 19/23] Debugging LRM transformations --- .../Annual_Developed_Volume_Query copy.sql | 180 ------------------ 1 file changed, 180 deletions(-) delete mode 100644 shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql diff --git a/shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql b/shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql deleted file mode 100644 index a0f1e25..0000000 --- a/shared/bcts_reports_etl/Annual_Developed_Volume_Query copy.sql +++ /dev/null @@ -1,180 +0,0 @@ -DROP VIEW IF EXISTS bcts_staging.DIVISION; -CREATE VIEW bcts_staging.DIVISION AS - SELECT * FROM lrm_replication.DIVISION; - -DROP VIEW IF EXISTS bcts_staging.BLOCK_ALLOCATION; -CREATE VIEW bcts_staging.BLOCK_ALLOCATION AS - SELECT * FROM lrm_replication.BLOCK_ALLOCATION; - -DROP VIEW IF EXISTS bcts_staging.MANAGEMENT_UNIT; -CREATE VIEW bcts_staging.MANAGEMENT_UNIT AS - SELECT * FROM lrm_replication.MANAGEMENT_UNIT; - -DROP VIEW IF EXISTS bcts_staging.LICENCE; -CREATE VIEW bcts_staging.LICENCE AS - SELECT * FROM lrm_replication.LICENCE; - -DROP VIEW IF EXISTS bcts_staging.BLOCK_ADMIN_ZONE; -CREATE VIEW bcts_staging.BLOCK_ADMIN_ZONE AS - SELECT * FROM lrm_replication.BLOCK_ADMIN_ZONE; - -DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS - SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; - -DROP VIEW IF EXISTS bcts_staging.CODE_LOOKUP; -CREATE VIEW bcts_staging.CODE_LOOKUP AS - SELECT * FROM lrm_replication.CODE_LOOKUP; - -DROP VIEW IF EXISTS bcts_staging.TENURE_TYPE; -CREATE VIEW bcts_staging.TENURE_TYPE AS - SELECT * FROM lrm_replication.TENURE_TYPE; - -DROP VIEW IF EXISTS bcts_staging.CUT_PERMIT; -CREATE VIEW bcts_staging.CUT_PERMIT AS - SELECT * FROM lrm_replication.CUT_PERMIT; - -DROP VIEW IF EXISTS bcts_staging.MARK; -CREATE VIEW bcts_staging.MARK AS - SELECT * FROM lrm_replication.MARK; - -DROP VIEW IF EXISTS bcts_staging.DIVISION_CODE_LOOKUP; -CREATE VIEW bcts_staging.DIVISION_CODE_LOOKUP AS - SELECT * FROM lrm_replication.DIVISION_CODE_LOOKUP; - -DROP VIEW IF EXISTS bcts_staging.CUT_BLOCK; -CREATE VIEW bcts_staging.CUT_BLOCK AS - SELECT * FROM lrm_replication.CUT_BLOCK; - -DROP VIEW IF EXISTS bcts_staging.ACTIVITY_CLASS; -CREATE VIEW bcts_staging.ACTIVITY_CLASS AS - SELECT * FROM lrm_replication.ACTIVITY_CLASS; - -DROP VIEW IF EXISTS bcts_staging.ACTIVITY_TYPE; -CREATE VIEW bcts_staging.ACTIVITY_TYPE AS - SELECT * FROM lrm_replication.ACTIVITY_TYPE; - -DROP VIEW IF EXISTS bcts_staging.ACTIVITY; -CREATE VIEW bcts_staging.ACTIVITY AS - SELECT * FROM lrm_replication.ACTIVITY; - - -DROP VIEW IF EXISTS bcts_staging.annual_developed_volume; - -CREATE VIEW bcts_staging.annual_developed_volume AS -WITH annual_developed_volume AS -( - SELECT DISTINCT - CASE - WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN', - 'TCC', 'TKA', 'TKO', 'TOC' ) THEN - 'Interior' - WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN - 'Coast' - END AS business_area_region_category, - CASE - WHEN d.divi_short_code IN ( 'TBA', 'TPL', 'TPG', 'TSK', 'TSN' ) THEN - 'North Interior' - WHEN d.divi_short_code IN ( 'TCC', 'TKA', 'TKO', 'TOC' ) THEN - 'South Interior' - WHEN d.divi_short_code IN ( 'TCH', 'TST', 'TSG' ) THEN - 'Coast' - END AS business_area_region, - CASE WHEN d.divi_division_name = 'Seaward' THEN 'Seaward-Tlasta' ELSE d.divi_division_name END - || ' (' - || d.divi_short_code - || ')' AS business_area, - d.divi_short_code AS business_area_code, - cl.colu_lookup_desc AS "Field Team", - mu.manu_id, - l.licn_licence_id AS licence, - tn.tent_tenure_id AS "File Type", - l.blaz_admin_zone_id AS agreement_type_code, - z.blaz_admin_zone_desc AS agreement_type, - cp.perm_permit_id AS permit, - m.mark_mark_id AS mark, - b.cutb_block_id AS block, - b.cutb_system_id AS ubi, - b.cutb_block_state AS block_state, - ba.blal_cruise_m3_vol AS cruise_volume, - ba.blal_rw_vol AS rw_volume, - actb.rc_done, - EXTRACT(YEAR FROM (actb.rc_done + INTERVAL '9 months')) AS rc_done_fiscal, - actb.dr_done, - EXTRACT(YEAR FROM (actb.dr_done + INTERVAL '9 months')) AS dr_done_fiscal, - actb.dvs_done, - EXTRACT(YEAR FROM (actb.dvs_done + INTERVAL '9 months')) AS dvs_done_fiscal, - actb.dvc_done, - EXTRACT(YEAR FROM (actb.dvc_done + INTERVAL '9 months')) AS dvc_done_fiscal, - b.cutb_seq_nbr - FROM - lrm_replication.division d - INNER JOIN bcts_staging.block_allocation ba - ON d.divi_div_nbr = ba.divi_div_nbr - INNER JOIN bcts_staging.management_unit mu - ON ba.manu_seq_nbr = mu.manu_seq_nbr - INNER JOIN bcts_staging.licence l - ON ba.licn_seq_nbr = l.licn_seq_nbr - LEFT OUTER JOIN bcts_staging.block_admin_zone z - ON l.divi_div_nbr = z.divi_div_nbr - AND l.blaz_admin_zone_id = z.blaz_admin_zone_id - AND ba.licn_seq_nbr = l.licn_seq_nbr - AND l.divi_div_nbr = z.divi_div_nbr - AND l.blaz_admin_zone_id = z.blaz_admin_zone_id - LEFT OUTER JOIN bcts_staging.division_code_lookup dcl - ON l.licn_field_team_id = dcl.colu_lookup_id - AND l.divi_div_nbr = dcl.divi_div_nbr - LEFT OUTER JOIN bcts_staging.code_lookup cl - ON dcl.colu_lookup_type = cl.colu_lookup_type - AND dcl.colu_lookup_id = cl.colu_lookup_id - LEFT JOIN bcts_staging.tenure_type tn - ON l.tent_seq_nbr = tn.tent_seq_nbr - LEFT OUTER JOIN bcts_staging.cut_permit cp - ON ba.perm_seq_nbr = cp.perm_seq_nbr - LEFT JOIN bcts_staging.mark m - ON ba.mark_seq_nbr = m.mark_seq_nbr - INNER JOIN bcts_staging.cut_block b - ON ba.cutb_seq_nbr = b.cutb_seq_nbr - INNER JOIN - ( - SELECT - A.CUTB_SEQ_NBR, - MAX(CASE WHEN ACTT_KEY_IND = 'RC' THEN DATE_TRUNC('DAY', ACTI_STATUS_DATE) END)::DATE AS RC_Done, - MAX(CASE WHEN ACTT_KEY_IND = 'DR' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DR_Done, - MAX(CASE WHEN ACTT_KEY_IND = 'DVS' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVS_Done, - MAX(CASE WHEN ACTT_KEY_IND = 'DVC' THEN DATE_TRUNC('DAY',ACTI_STATUS_DATE) END)::DATE AS DVC_Done - FROM - bcts_staging.ACTIVITY_CLASS C, - bcts_staging.ACTIVITY_TYPE T, - bcts_staging.ACTIVITY A - WHERE - C.ACCL_SEQ_NBR = T.ACCL_SEQ_NBR - AND T.ACTT_SEQ_NBR = A.ACTT_SEQ_NBR - AND C.ACCL_DESCRIPTION = 'CMB' - AND T.ACTT_KEY_IND In ('RC', 'DR', 'DVS', 'DVC') - AND A.ACTI_STATUS_IND = 'D' - GROUP BY A.CUTB_SEQ_NBR, T.ACTT_KEY_IND - ) ACTB - ON ba.cutb_seq_nbr = actb.cutb_seq_nbr - AND actb.dvc_done BETWEEN TO_DATE('2024-04-01', 'YYYY-MM-DD') -- Date: beginning of current fiscal - AND TO_DATE('2024-09-30', 'YYYY-MM-DD') -- Date: end of reporting period -) - -SELECT * -FROM annual_developed_volume -ORDER BY - length(business_area_region) desc, - business_area_region, - business_area, - "Field Team", - MANU_ID, - licence, - permit, - mark, - block -; - -DROP VIEW IF EXISTS bcts_reporting.annual_developed_volume; - -CREATE TABLE bcts_reporting.annual_developed_volume AS - SELECT * FROM bcts_staging.annual_developed_volume; From 50e297113f274ad0619975b757dc7e6ff067742d Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Wed, 20 Nov 2024 11:08:23 -0800 Subject: [PATCH 20/23] Debugging BCTS access management --- shared/bcts_access_management/bcts_acces_apply_grants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/bcts_access_management/bcts_acces_apply_grants.py b/shared/bcts_access_management/bcts_acces_apply_grants.py index 2854997..fd32aa4 100644 --- a/shared/bcts_access_management/bcts_acces_apply_grants.py +++ b/shared/bcts_access_management/bcts_acces_apply_grants.py @@ -40,10 +40,10 @@ def apply_grants(): all_statements = [] for _, row in df.iterrows(): all_statements.extend(generate_grant_statements(row)) - logging.info(all_statements) - # Execute the grant statements in PostgreSQL + # Execute the grant statements in PostgreSQL with engine.begin() as connection: for statement in all_statements: + logging.info(statement) connection.execute(text(statement)) logging.info("Grant statements executed successfully.") From a2b657c7b02498f92426edf2300153115b29031c Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Wed, 20 Nov 2024 13:06:33 -0800 Subject: [PATCH 21/23] Added usage on schema statements to apply grants script --- shared/bcts_access_management/bcts_acces_apply_grants.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/shared/bcts_access_management/bcts_acces_apply_grants.py b/shared/bcts_access_management/bcts_acces_apply_grants.py index fd32aa4..186865d 100644 --- a/shared/bcts_access_management/bcts_acces_apply_grants.py +++ b/shared/bcts_access_management/bcts_acces_apply_grants.py @@ -40,7 +40,14 @@ def apply_grants(): all_statements = [] for _, row in df.iterrows(): all_statements.extend(generate_grant_statements(row)) - # Execute the grant statements in PostgreSQL + + # Execute the grant statements in PostgreSQL + schema_roles_list = list(set([(x.split(' ON ')[1].split('.')[0], x.split(' TO ')[1].split(';')[0]) for x in all_statements])) + usage_statements = [] + for (schema, role) in schema_roles_list: + usage_statement = f"GRANT USAGE ON SCHEMA {schema} TO {role};" + usage_statements.append(usage_statement) + all_statements.extend(usage_statements) with engine.begin() as connection: for statement in all_statements: logging.info(statement) From 4ab3fe4e73d2ec7b0cd91cdf9dbb39b6c4432b07 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Wed, 20 Nov 2024 13:27:18 -0800 Subject: [PATCH 22/23] Updated workflows to point to main branch for production deployment --- .github/workflows/docker-bctsAdhocSql.yaml | 2 +- .github/workflows/docker-bctsGrantMngmt.yaml | 2 +- .github/workflows/docker-bctsTransformations.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-bctsAdhocSql.yaml b/.github/workflows/docker-bctsAdhocSql.yaml index 1a1662a..3f29c8d 100644 --- a/.github/workflows/docker-bctsAdhocSql.yaml +++ b/.github/workflows/docker-bctsAdhocSql.yaml @@ -2,7 +2,7 @@ name: Push to GHCR on: push: - branches: ["SD-128488-BCTS-ODS-GRANT-MANAGEMENT"] + branches: ["main"] env: REGISTRY: ghcr.io diff --git a/.github/workflows/docker-bctsGrantMngmt.yaml b/.github/workflows/docker-bctsGrantMngmt.yaml index 1f2b70d..31f0d84 100644 --- a/.github/workflows/docker-bctsGrantMngmt.yaml +++ b/.github/workflows/docker-bctsGrantMngmt.yaml @@ -2,7 +2,7 @@ name: Push to GHCR on: push: - branches: ["SD-128488-BCTS-ODS-GRANT-MANAGEMENT"] + branches: ["main"] env: REGISTRY: ghcr.io diff --git a/.github/workflows/docker-bctsTransformations.yaml b/.github/workflows/docker-bctsTransformations.yaml index fde994c..8efa6ac 100644 --- a/.github/workflows/docker-bctsTransformations.yaml +++ b/.github/workflows/docker-bctsTransformations.yaml @@ -2,7 +2,7 @@ name: Push to GHCR on: push: - branches: ["SD-128488-BCTS-ODS-GRANT-MANAGEMENT"] + branches: ["main"] env: REGISTRY: ghcr.io From efa21630a296845bcd6ba7a470637bc344371bc3 Mon Sep 17 00:00:00 2001 From: Munthikodu Date: Mon, 25 Nov 2024 09:43:04 -0800 Subject: [PATCH 23/23] Removed duplicate ddl statement --- bcts/access_management/ddl/bcts_roles.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/bcts/access_management/ddl/bcts_roles.sql b/bcts/access_management/ddl/bcts_roles.sql index deb7fd2..c7b5843 100644 --- a/bcts/access_management/ddl/bcts_roles.sql +++ b/bcts/access_management/ddl/bcts_roles.sql @@ -9,7 +9,6 @@ CREATE SCHEMA bcts_reporting; -- Create roles for ETL and data consumption CREATE ROLE BCTS_ETL_ROLE; CREATE ROLE BCTS_DEV_ROLE; -CREATE ROLE BCTS_DEV_ROLE; CREATE ROLE BCTS_STAGE_ANALYST_ROLE; CREATE ROLE BCTS_STAGE_ANALYST_PI_ROLE; CREATE ROLE BCTS_ANALYST_ROLE;