From 17a41abf3ad130ce8a6b7cf3c22d56a45146aaea Mon Sep 17 00:00:00 2001 From: NickSwainston Date: Thu, 14 Dec 2023 19:05:38 +1100 Subject: [PATCH] Completed molonglo upload scripts --- dev_scripts/generate_all_jsons_molonglo.sh | 16 +++++ psrdb/load_data.py | 2 +- psrdb/scripts/generate_meerkat_json.py | 27 +------- psrdb/scripts/generate_molonglo_json.py | 74 ++-------------------- psrdb/utils/upload.py | 32 ++++++++++ pyproject.toml | 1 + 6 files changed, 58 insertions(+), 94 deletions(-) create mode 100644 dev_scripts/generate_all_jsons_molonglo.sh create mode 100644 psrdb/utils/upload.py diff --git a/dev_scripts/generate_all_jsons_molonglo.sh b/dev_scripts/generate_all_jsons_molonglo.sh new file mode 100644 index 0000000..bed92af --- /dev/null +++ b/dev_scripts/generate_all_jsons_molonglo.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +for path in $(find /fred/oz002/ldunn/meertime_dataportal/data/post -type f -name "obs.header"); do + if [ -e "${path%%obs.header}/meertime.json" ]; then + echo "Skipping $path" + else + echo "Making meetime.json for $path" + EXIT_CODE=0 + generate_meerkat_json $path -o ${path%%obs.header} || EXIT_CODE=$? + if [ "$EXIT_CODE" -ne 42 ]; then + ingest_obs ${path%%obs.header}/meertime.json + fi + fi +done \ No newline at end of file diff --git a/psrdb/load_data.py b/psrdb/load_data.py index 821be11..fe75b6b 100644 --- a/psrdb/load_data.py +++ b/psrdb/load_data.py @@ -1,4 +1,4 @@ import os DATA_DIR = os.path.join(os.path.dirname(__file__), "data") -MOLONGLO_CALIBRATIONS = os.path.join(DATA_DIR, "molonglo_calibrations.txt") +MOLONGLO_CALIBRATIONS = os.path.join(DATA_DIR, "molonglo_phasing.txt") diff --git a/psrdb/scripts/generate_meerkat_json.py b/psrdb/scripts/generate_meerkat_json.py index 9e54e62..ffddaaa 100644 --- a/psrdb/scripts/generate_meerkat_json.py +++ b/psrdb/scripts/generate_meerkat_json.py @@ -10,9 +10,8 @@ from decouple import config from datetime import datetime -import psrchive as psr - from psrdb.utils import header +from psrdb.utils.upload import generate_obs_length, get_archive_ephemeris CALIBRATIONS_DIR = config("CALIBRATIONS_DIR", "/fred/oz005/users/aparthas/reprocessing_MK/poln_calibration") @@ -21,15 +20,6 @@ SEARCH_DIR = config("SEARCH_DIR", "/fred/oz005/search") -def generate_obs_length(archive): - """ - Determine the length of the observation from the input archive file - """ - - ar = psr.Archive_load(archive) - ar = ar.total() - return ar.get_first_Integration().get_duration() - def get_sf_length(sf_files): """ Determine the length of input sf files with the vap command @@ -80,21 +70,6 @@ def get_calibration(utc_start): raise RuntimeError(f"Could not find calibration file for utc_start={utc_start}") -def get_archive_ephemeris(freq_summed_archive): - """ - Get the ephemeris from the archive file using the vap command. - """ - comm = "vap -E {0}".format(freq_summed_archive) - args = shlex.split(comm) - proc = subprocess.Popen(args,stdout=subprocess.PIPE) - proc.wait() - ephemeris_text = proc.stdout.read().decode("utf-8") - - if ephemeris_text.startswith('\n'): - # Remove newline character at start of output - ephemeris_text = ephemeris_text.lstrip('\n') - return ephemeris_text - def main(): import argparse diff --git a/psrdb/scripts/generate_molonglo_json.py b/psrdb/scripts/generate_molonglo_json.py index 5498194..7b5ea8d 100644 --- a/psrdb/scripts/generate_molonglo_json.py +++ b/psrdb/scripts/generate_molonglo_json.py @@ -1,66 +1,21 @@ #!/usr/bin/env python import os -import sys -import glob import json -import shlex import logging -import subprocess from decouple import config from datetime import datetime, timezone -import psrchive as psr - from psrdb.utils import header +from psrdb.utils.upload import generate_obs_length, get_archive_ephemeris from psrdb.load_data import MOLONGLO_CALIBRATIONS -RESULTS_DIR = config("RESULTS_DIR", "/fred/oz002/ldunn/meertime_dataportal/data/post") - - -def generate_obs_length(archive): - """ - Determine the length of the observation from the input archive file - """ - - ar = psr.Archive_load(archive) - ar = ar.total() - return ar.get_first_Integration().get_duration() - -def get_sf_length(sf_files): - """ - Determine the length of input sf files with the vap command - """ - comm = f"vap -c length {' '.join(sf_files)}" - args = shlex.split(comm) - proc = subprocess.Popen(args, stdout=subprocess.PIPE, text=True, bufsize=1) - vap_lines = [] - try: - # Read and process the output line by line in real-time - for line in iter(proc.stdout.readline, ''): - print(line, end='', flush=True) - vap_lines.append(line) - - # Handle Ctrl+C to gracefully terminate the subprocess - except KeyboardInterrupt: - logging.error("Process interrupted. Terminating...") - sys.exit(1) - - finally: - # Wait for the subprocess to complete - proc.wait() - - lengths = [] - for line in vap_lines[1:]: - if line == '': - continue - lengths.append(float(line.split()[1].strip())) - return sum(lengths) +MOLONGLO_RESULTS_DIR = config("MOLONGLO_RESULTS_DIR", "/fred/oz002/ldunn/meertime_dataportal/data/post") def get_calibration(utc_start): - utc_start_dt = datetime.strptime(utc_start, "%Y-%m-%d-%H:%M:%S") + utc_start_dt = datetime.strptime(utc_start, "%Y-%m-%d-%H:%M:%S").replace(tzinfo=timezone.utc) with open(MOLONGLO_CALIBRATIONS) as f: for line in f.readlines(): if line[0] == '#': @@ -76,21 +31,6 @@ def get_calibration(utc_start): return calibration -def get_archive_ephemeris(freq_summed_archive): - """ - Get the ephemeris from the archive file using the vap command. - """ - comm = "vap -E {0}".format(freq_summed_archive) - args = shlex.split(comm) - proc = subprocess.Popen(args,stdout=subprocess.PIPE) - proc.wait() - ephemeris_text = proc.stdout.read().decode("utf-8") - - if ephemeris_text.startswith('\n'): - # Remove newline character at start of output - ephemeris_text = ephemeris_text.lstrip('\n') - return ephemeris_text - def main(): import argparse @@ -131,7 +71,7 @@ def main(): obs_data.parse() # Find raw archive and frequency summed files - freq_summed_archive = f"{RESULTS_DIR}/{obs_data.source}/{obs_data.utc_start}/{obs_data.source}_{obs_data.utc_start}.FT" + freq_summed_archive = f"{MOLONGLO_RESULTS_DIR}/{obs_data.source}/{obs_data.utc_start}/{obs_data.source}_{obs_data.utc_start}.FT" # Check if there are freq.sum and archive files obs_length = generate_obs_length(freq_summed_archive) @@ -141,8 +81,8 @@ def main(): meertime_dict = { "pulsarName": obs_data.source, - "telescopeName": obs_data.telescope, - "projectCode": obs_data.proposal_id, + "telescopeName": "MONS", + "projectCode": "MONSPSR_TIMING", "schedule_block_id": obs_data.schedule_block_id, "cal_type": "pre", "cal_location": cal_location, @@ -151,7 +91,7 @@ def main(): "nchan": obs_data.nchan, "beam": obs_data.beam, "nant": obs_data.nant, - "nantEff": obs_data.nant_eff, + "nantEff": obs_data.nant, "npol": obs_data.npol, "obsType": obs_data.obs_type, "utcStart": obs_data.utc_start, diff --git a/psrdb/utils/upload.py b/psrdb/utils/upload.py new file mode 100644 index 0000000..fb92971 --- /dev/null +++ b/psrdb/utils/upload.py @@ -0,0 +1,32 @@ + +import shlex + +import subprocess + +import psrchive as psr + + +def generate_obs_length(archive): + """ + Determine the length of the observation from the input archive file + """ + + ar = psr.Archive_load(archive) + ar = ar.total() + return ar.get_first_Integration().get_duration() + + +def get_archive_ephemeris(freq_summed_archive): + """ + Get the ephemeris from the archive file using the vap command. + """ + comm = "vap -E {0}".format(freq_summed_archive) + args = shlex.split(comm) + proc = subprocess.Popen(args,stdout=subprocess.PIPE) + proc.wait() + ephemeris_text = proc.stdout.read().decode("utf-8") + + if ephemeris_text.startswith('\n'): + # Remove newline character at start of output + ephemeris_text = ephemeris_text.lstrip('\n') + return ephemeris_text \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f00acdd..1c649e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ description = "CLI for pulsars.org.au" authors = ["GWDC"] license = "MIT" packages = [{include = "psrdb"}] +include = ["psrdb/data/molonglo_phasing.txt"] [tool.poetry.scripts] psrdb = "psrdb.scripts.psrdb:main"