Pep srh

scripts/us_census/pep/pep_by_srh/README.md

@@ -5,9 +5,9 @@
 This import covers US Census Population estimate by SRH (Sex, Race and Hispanic / Origin).
 Data is covered for 
 
-    1990 - 2020     County
-    1980 - 2020     State
-    1980 - 2020     National
+    1990 - 2023     County
+    1980 - 2023     State
+    1980 - 2023     National
 
 Properties Covered - 
 
@@ -36,22 +36,26 @@ Origin
 
 
 ## Scripts
-```
-download.sh
-```
-Above scipt creates necessary folders required for processing files and download's US 
-Census PEP files to folder download_files.
-
-
 ```
 process.py
 ```
-Reads downloaded files from download_files folder, prepares consolidated 
+Above scipt creates necessary folders required for processing files and download's US 
+Census PEP files to folder input_files. And reads downloaded files from input_files folder, prepares consolidated 
 (National, State, County) output files for as-is (population_estimate_by_srh.csv) and aggregated (population_estimate_by_srh_agg.csv) in output_files folder
 
+### Import Procedure
+The below command will run process.py and generate output csv, mcf and tmcf. 
+`python3 script/us_census/pep/pep_by_srh/process.py`
+
 
 ```
 generate_mcf.py
 generate_tmcf.py
 ```
 Generates mcf and tmcf files for both 'as-is' data processing and 'aggregated' data processing
+
+### Additional Notes
+
+We have modified the script for Automation setup, now it's not required to add the future URL's in input_url.jason file. Code will take care of future URL formation automatically.
+
+

scripts/us_census/pep/pep_by_srh/constants.py

@@ -21,7 +21,7 @@
 DC import
 """
 
-DOWNLOAD_DIR = '/download_files/'
+DOWNLOAD_DIR = '/input_files/'
 PROCESS_AS_IS_DIR = '/process_files/as_is/'
 PROCESS_AGG_DIR = '/process_files/agg/'
 OUTPUT_DIR = '/output_files/'
@@ -238,25 +238,39 @@
     '/process_files/agg/1990_2000/county/',
     '/process_files/agg/2000_2010/county/',
     '/process_files/agg/2010_2020/county/',
+    '/process_files/agg/2020_2023/county/',
     '/process_files/as_is/1990_2000/county/',
     '/process_files/as_is/2000_2010/county/',
     '/process_files/as_is/2010_2020/county/',
+    '/process_files/as_is/2020_2023/county/',
     '/process_files/as_is/1980_1990/state/',
     '/process_files/as_is/1990_2000/state/',
     '/process_files/as_is/2000_2010/state/',
     '/process_files/as_is/2010_2020/state/',
+    '/process_files/as_is/2020_2023/state/',
     '/process_files/agg/1980_1990/state/',
     '/process_files/agg/1990_2000/state/',
     '/process_files/agg/2000_2010/state/',
     '/process_files/agg/2010_2020/state/',
+    '/process_files/agg/2020_2023/state/',
     '/process_files/as_is/1980_1990/national/',
     '/process_files/as_is/1990_2000/national/',
     '/process_files/as_is/2000_2010/national/',
     '/process_files/as_is/2010_2020/national/',
+    '/process_files/as_is/2020_2023/national/',
     '/process_files/agg/1980_1990/national/',
     '/process_files/agg/1990_2000/national/',
     '/process_files/agg/2000_2010/national/',
-    '/process_files/agg/2010_2020/national/', '/output_files/'
+    '/process_files/agg/2020_2023/national/',
+    '/process_files/agg/2010_2020/national/',
+    '/output_files/',
+    '/input_files/',
+    '/input_files/1990_2000/county/',
+    '/input_files/2000_2010/county/',
+    '/input_files/2010_2020/county/',
+    '/input_files/2020_2023/county/',
+    '/input_files/1980_1990/state/',
+    '/input_files/1990_2000/state/'
 ]
 
 MCF_SRH = '''Node: dcid:{node_name}

scripts/us_census/pep/pep_by_srh/download.sh

@@ -2,76 +2,89 @@
 
 set -x
 
-rm -rf ./download_files/
+rm -rf ./input_files/
 rm -rf ./process_files/
 rm -rf ./output_files/
 
-mkdir -p ./download_files/1990_2000/county/
-mkdir -p ./download_files/2000_2010/county/
-mkdir -p ./download_files/2010_2020/county/
+mkdir -p ./input_files/1990_2000/county/
+mkdir -p ./input_files/2000_2010/county/
+mkdir -p ./input_files/2010_2020/county/
+mkdir -p ./input_files/2020_2023/county/
 
 mkdir -p ./process_files/agg/1990_2000/county/
 mkdir -p ./process_files/agg/2000_2010/county/
 mkdir -p ./process_files/agg/2010_2020/county/
+mkdir -p ./process_files/agg/2020_2023/county/
 
 mkdir -p ./process_files/as_is/1990_2000/county/
 mkdir -p ./process_files/as_is/2000_2010/county/
 mkdir -p ./process_files/as_is/2010_2020/county/
+mkdir -p ./process_files/as_is/2020_2023/county/
 
-mkdir -p ./download_files/1980_1990/state/
-mkdir -p ./download_files/1990_2000/state/
+mkdir -p ./input_files/1980_1990/state/
+mkdir -p ./input_files/1990_2000/state/
 
 mkdir -p ./process_files/as_is/1980_1990/state/
 mkdir -p ./process_files/as_is/1990_2000/state/
 mkdir -p ./process_files/as_is/2000_2010/state/
 mkdir -p ./process_files/as_is/2010_2020/state/
+mkdir -p ./process_files/as_is/2020_2023/state/
 
 mkdir -p ./process_files/agg/1980_1990/state/
 mkdir -p ./process_files/agg/1990_2000/state/
 mkdir -p ./process_files/agg/2000_2010/state/
 mkdir -p ./process_files/agg/2010_2020/state/
+mkdir -p ./process_files/agg/2020_2023/state/
 
 mkdir -p ./process_files/as_is/1980_1990/national/
 mkdir -p ./process_files/as_is/1990_2000/national/
 mkdir -p ./process_files/as_is/2000_2010/national/
 mkdir -p ./process_files/as_is/2010_2020/national/
+mkdir -p ./process_files/as_is/2020_2023/national/
 
 mkdir -p ./process_files/agg/1980_1990/national/
 mkdir -p ./process_files/agg/1990_2000/national/
 mkdir -p ./process_files/agg/2000_2010/national/
 mkdir -p ./process_files/agg/2010_2020/national/
+mkdir -p ./process_files/agg/2020_2023/national/
 
 mkdir -p ./output_files/
 
 for file in $(cat download_county_files_1990_2000.txt)
 do
     $(curl https://www2.census.gov/programs-surveys/popest/datasets/1990-2000/counties/asrh/${file} \
-          -o ./download_files/1990_2000/county/${file})
+          -o ./input_files/1990_2000/county/${file})
+done
+
+for file in $(cat download_county_files_2020_2023.txt)
+do
+    $(curl https://www2.census.gov/programs-surveys/popest/datasets/2020-2023/counties/asrh/${file} \
+        -o ./input_files/2020_2023/county/${file})
 done
 
 for file in $(cat download_county_files_2000_2010.txt)
 do
     $(curl https://www2.census.gov/programs-surveys/popest/datasets/2000-2010/intercensal/county/${file} \
-          -o ./download_files/2000_2010/county/${file})
+          -o ./input_files/2000_2010/county/${file})
 done
 
 
 for file in $(cat download_county_files_2010_2020.txt)
 do
     $(curl https://www2.census.gov/programs-surveys/popest/datasets/2010-2020/counties/asrh/${file} \
-          -o ./download_files/2010_2020/county/${file})
+          -o ./input_files/2010_2020/county/${file})
 done
 
 
 for file in $(cat download_state_files_1980_1990.txt)
 do
     $(curl https://www2.census.gov/programs-surveys/popest/datasets/1980-1990/state/asrh/${file} \
-          -o ./download_files/1980_1990/state/${file})
+          -o ./input_files/1980_1990/state/${file})
 done
 
 
 for file in $(cat download_state_files_1990_2000.txt)
 do
     $(curl https://www2.census.gov/programs-surveys/popest/tables/1990-2000/state/asrh/${file} \
-          -o ./download_files/1990_2000/state/${file})
+          -o ./input_files/1990_2000/state/${file})
 done

scripts/us_census/pep/pep_by_srh/download_county_files_2020_2023.txt

@@ -0,0 +1 @@
+cc-est2023-alldata.csv