Merge pull request #3 from OZGrav/new_models

New models v3.0.0
OZGrav · Oct 28, 2023 · e6e7b2d · e6e7b2d
2 parents d0f3080 + 42ef78f
commit e6e7b2d
Show file tree

Hide file tree

Showing 77 changed files with 3,910 additions and 5,903 deletions.
diff --git a/dev_scripts/REAME.md b/dev_scripts/REAME.md
@@ -0,0 +1,4 @@
+# Dev Scripts
+
+Scripts in this directory are scripts that are used by the developers for admin tasks.
+These scripts will not be installed.
diff --git a/dev_scripts/chunks_of_pulsars.py b/dev_scripts/chunks_of_pulsars.py
@@ -0,0 +1,66 @@
+import os
+import glob
+
+from psrqpy import QueryATNF
+
+from psrdb.tables.pulsar import Pulsar
+from psrdb.graphql_client import GraphQLClient
+from psrdb.utils.other import setup_logging
+
+def chunk_list(lst, chunk_size):
+    for i in range(0, len(lst), chunk_size):
+        yield lst[i:i + chunk_size]
+
+# PSRDB setup
+logger = setup_logging()
+client = GraphQLClient(os.environ.get("PSRDB_URL"), False, logger=logger)
+puslar_client = Pulsar(client, os.environ.get("PSRDB_TOKEN"))
+puslar_client.get_dicts = True
+puslar_client.set_use_pagination(True)
+
+pulsars = []
+# Query based on provided parameters
+pulsar_data = puslar_client.list()
+for pulsar in pulsar_data:
+    if "_" in pulsar['name']:
+        continue
+    pulsar_name = pulsar['name']
+
+    if pulsar_name in [
+            "J1939-6342", # J0437 mislabeled
+            "J1924-2914", # J0437 mislabeled
+            "J2052-3640", # J0437 mislabeled
+            "J2214-3835", # J0437 mislabeled
+            "J0710-1604", # Perhaps a candidate that wasn't detectable
+            "J2003-0934", # A craft candidate
+            "J1823-3022", # A trapum candidate
+        ]:
+        continue
+
+    if pulsar_name in [
+            "J1644-4657",
+            "J1444-6026",
+            "J1835-09242",
+            "J0024-7204F",
+            "J2140-2310B",
+            "J1402-5021", # Missing archive files
+        ]:
+        # Skipping because single TPA detections with no template
+        continue
+
+    pulsars.append(pulsar_name)
+
+print(len(pulsars))
+pulsars_with_templates = []
+for pulsar in pulsars:
+    template = glob.glob(f"/fred/oz005/users/meerpipe/templates/*/*/{pulsar}.std")
+    if len(template) > 0:
+        pulsars_with_templates.append(pulsar)
+print(len(pulsars_with_templates))
+
+query = QueryATNF(psrs=pulsars_with_templates).pandas
+
+print(len(query))
+
+for pulsar_chunk in chunk_list(list(query.sort_values("P0")["PSRJ"]), 100):
+    print(",".join(pulsar_chunk))
diff --git a/dev_scripts/generate_all_jsons.sh b/dev_scripts/generate_all_jsons.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+for path in $(find /fred/oz005/timing -type f -name "obs.header"); do
+    IFS="/"
+    # Split the path into an array
+    read -ra directories <<< "$path"
+    beam=${directories[-3]}
+    utc=${directories[-4]}
+    jname=${directories[-5]}
+    IFS=" "
+    echo "Making meetime.json for $path"
+    generate_meerkat_json $path $beam -o ${path%%obs.header}
+    ingest_obs ${path%%obs.header}/meertime.json
+done
diff --git a/dev_scripts/test_basic_database.sh b/dev_scripts/test_basic_database.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Telescopes
+psrdb telescope create MeerKAT
+psrdb telescope create MONS
+
+# Main projects
+psrdb mainproject create MeerKAT MeerTIME
+psrdb mainproject create MeerKAT Trapum
+psrdb mainproject create MeerKAT Commissioning
+psrdb mainproject create MeerKAT Unknown
+psrdb mainproject create MONS    MONSPSR
+
+# Projects
+psrdb project create Trapum          SCI-20180923-MK-04      None            548 ""
+psrdb project create MeerTIME        SCI-20180516-MB-02      TPA             548 "Thousand Pulsar Array"
+psrdb project create MeerTIME        SCI-20180516-MB-05      PTA             548 "Pulsar Timing Array"
+psrdb project create MeerTIME        SCI-20180516-MB-03      RelBin          548 "Relativistic Binaries"
+psrdb project create MeerTIME        SCI-20180516-MB-04      GC              548 "Globular Clusters"
+psrdb project create Commissioning   COM-20200429-SB-01      unknown         548 ""
+psrdb project create Commissioning   COM-20200402-SB-01      unknown         548 ""
+psrdb project create Commissioning   COM-20180801-SB-01      unknown         548 ""
+psrdb project create Trapum          SCI-20180923-MK-01      unknown         548 ""
+psrdb project create MeerTIME        SCI-20200222-MB-01      unknown         548 ""
+psrdb project create Commissioning   COM-PTUSE-TEST          unknown         548 ""
+psrdb project create Commissioning   COM-20200929-AF-01      unknown         548 ""
+psrdb project create Commissioning   COM-20190902-MS-01      unknown         548 ""
+psrdb project create Commissioning   COM-20191023-MS-01      unknown         548 ""
+psrdb project create MeerTIME        SCI-20180516-MB-01      unknown         548 ""
+psrdb project create Commissioning   COM-20191129-MG-01      unknown         548 ""
+psrdb project create Commissioning   COM-20201001-DH-01      unknown         548 ""
+psrdb project create Commissioning   COM-20180614-MG-01      unknown         548 ""
+psrdb project create MeerTIME        SCI-20180516-MB-06      ngc6440         548 "Measuring the Shapiro delay in PSR J1748-2021B"
+psrdb project create MeerTIME        SCI-20180516-MB-99      phaseups        548 "MeerTIME Phase Up observations"
+psrdb project create MeerTIME        SCI-20180516-MB-95      phaseups2       548 "MeerTIME Phase Up observations"
+psrdb project create MeerTIME        SCI-20180516-MB-93      phaseups3       548 "MeerTIME Phase Up observations"
+psrdb project create Commissioning   COM-20190912-MS-01      unknown         548 ""
+psrdb project create MeerTIME        SCI-2018-0516-MB-03     unknown         548 ""
+psrdb project create MeerTIME        SCI-2018-0516-MB-02     unknown         548 ""
+psrdb project create Trapum          DDT-20200506-BS-01      unknown         548 ""
+psrdb project create Unknown         DDT-20190905-MC-01      unknown         548 ""
+psrdb project create Commissioning   COM-20191019-MG-01      unknown         548 ""
+psrdb project create Commissioning   COM-20190917-MS-01      unknown         548 ""
+psrdb project create Unknown         EXT-20210822-OW-01      unknown         548 ""
+psrdb project create Trapum          SCI-20180923-MK-06      unknown         548 ""
+psrdb project create Unknown         EXT-20210822-OW-02      unknown         548 ""
+psrdb project create Unknown         EXT-20220104-SB-01      unknown         548 ""
+psrdb project create Unknown         SCI-20200703-MK-02      unknown         548 ""
+psrdb project create Unknown         EXT-20220714-SG-01      unknown         548 ""
+psrdb project create MONSPSR         MONSPSR_TIMING          MONSPSR_TIMING  548 ""
+psrdb project create Unknown         ENG-20221026-PK-01      unknown         548 ""
+psrdb project create Unknown         EXT-20230510-SB-01      unknown         548 ""
+
+# Pulsar
+psrdb pulsar create J1705-1903 ""
+
+# Ephemeris
+psrdb ephemeris create J1705-1903 /home/nick/code/meertime_dataportal/backend/dataportal/tests/test_data/ephem_J1705-1903_same_1.eph SCI-20180516-MB-05 ""
+psrdb ephemeris create J1705-1903 /home/nick/code/meertime_dataportal/backend/dataportal/tests/test_data/ephem_J1705-1903_same_2.eph SCI-20180516-MB-05 ""
+psrdb ephemeris create J1705-1903 /home/nick/code/meertime_dataportal/backend/dataportal/tests/test_data/ephem_J1705-1903_diff.eph SCI-20180516-MB-05 ""
+
diff --git a/dev_scripts/update_pulsar_descriptions.py b/dev_scripts/update_pulsar_descriptions.py
@@ -0,0 +1,70 @@
+
+import os
+
+import psrqpy
+from pulsar_paragraph.pulsar_classes import PulsarParagraph
+from pulsar_paragraph.pulsar_paragraph import create_pulsar_paragraph
+
+from psrdb.tables.pulsar import Pulsar
+from psrdb.graphql_client import GraphQLClient
+from psrdb.utils.other import setup_logging, decode_id
+
+# PSRDB setup
+logger = setup_logging()
+client = GraphQLClient(os.environ.get("PSRDB_URL"), False, logger=logger)
+puslar_client = Pulsar(client, os.environ.get("PSRDB_TOKEN"))
+puslar_client.get_dicts = True
+puslar_client.set_use_pagination(True)
+
+query = psrqpy.QueryATNF().pandas
+pulsar_paragraph = PulsarParagraph()
+
+# Query based on provided parameters
+pulsar_data = puslar_client.list()
+for pulsar in pulsar_data:
+    if "_" in pulsar['name']:
+        continue
+    # Extract data from obs_data
+    pulsar_id = decode_id(pulsar['id'])
+    pulsar_name = pulsar['name']
+    if pulsar_name == "J1402-5124":
+        pulsar_name = "J1402-5021"
+    elif pulsar_name == "J1803-3002":
+        pulsar_name = "J1803-3002A"
+    elif pulsar_name == "J1826-2413":
+        pulsar_name = "J1826-2415"
+    elif pulsar_name == "J1147-6608":
+        pulsar_name = "J1146-6610"
+    elif pulsar_name == "J0514-4407":
+        pulsar_name = "J0514-4408"
+    elif pulsar_name == "J1325-6256":
+        pulsar_name = "J1325-6253"
+    elif pulsar_name == "J0922-5202":
+        pulsar_name = "J0921-5202"
+    elif pulsar_name == "J1759-2402":
+        pulsar_name = "J1759-24"
+    elif pulsar_name == "J0024-7204AA":
+        pulsar_name = "J0024-7204aa" # I think this one has been given a wrong ephemeris so will neeed to be fixed
+    elif pulsar_name == "J1653-4518":
+        pulsar_name = "J1653-45"
+    elif pulsar_name == "J0837-2454":
+        pulsar_name = "J0837-24"
+
+    elif pulsar_name in [
+            "J1939-6342", # J0437 mislabeled
+            "J1924-2914", # J0437 mislabeled
+            "J2052-3640", # J0437 mislabeled
+            "J2214-3835", # J0437 mislabeled
+            "J0710-1604", # Perhaps a candidate that wasn't detectable
+            "J2003-0934", # A craft candidate
+            "J1823-3022", # A trapum candidate
+        ]:
+        continue
+    print(pulsar_name)
+    comment = create_pulsar_paragraph(
+        pulsar_names=[pulsar_name],
+        query=query,
+        pulsar_paragraph=pulsar_paragraph,
+    )[0]
+    # print(comment)
+    puslar_client.update(pulsar_id, pulsar_name, comment)
diff --git a/dev_scripts/update_pulsar_name.sh b/dev_scripts/update_pulsar_name.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+old_name="$1"
+new_name="$2"
+
+mkdir -p /fred/oz005/timing/${new_name}
+
+for path in $(find /fred/oz005/timing/${old_name} -type f -name "obs.header"); do
+    IFS="/"
+    # Split the path into an array
+    read -ra directories <<< "$path"
+    freq=${directories[-2]}
+    beam=${directories[-3]}
+    utc=${directories[-4]}
+    jname=${directories[-5]}
+    IFS=" "
+    echo "Working on UTC:${utc} beam:${beam}"
+    mv /fred/oz005/timing/${old_name}/${utc} /fred/oz005/timing/${new_name}
+    sed -i "s/SOURCE              ${old_name}/SOURCE              ${new_name}/" /fred/oz005/timing/${new_name}/${utc}/${beam}/${freq}/obs.header
+    mv /fred/oz005/kronos/${beam}/${utc}/${old_name} /fred/oz005/kronos/${beam}/${utc}/${new_name}
+    sed -i "s/SOURCE              ${old_name}/SOURCE              ${new_name}/" /fred/oz005/kronos/${beam}/${utc}/${new_name}/obs.header
+done
diff --git a/docs/index.rst b/docs/index.rst
@@ -8,6 +8,7 @@ Welcome to psrdb's documentation!
    installation
    how_to_use
    database_structure
+   observation_ingest
    psrdb_modules
 
 

diff --git a/docs/observation_ingest.md b/docs/observation_ingest.md
@@ -0,0 +1,75 @@
+# Observation Ingest
+
+
+This documentation will describe how observation data is uploaded to the MeerTime database.
+It will first describe telescope specific steps to create a meertime.json and then how to upload the json.
+
+
+## Telescope specific ingest
+
+
+
+### MeerKAT ingest
+
+The design of the pulsar backend of MeerKAT (PTUSE) is described [here](https://ui.adsabs.harvard.edu/abs/2020PASA...37...28B/abstract).
+
+For each MeerKAT observation, two directories are created `/fred/oz005/kronos/<beam_num>/<utc>/<jname>` and `/fred/oz005/timing/<jname>/<utc>/<beam_num>/<frequency>`.
+The directories are split by `<beam_num>` as sometimes MeerKAT observe multiple sources at the same time (UTC) with different beams.
+
+In `/fred/oz005/kronos/<beam_num>/<utc>/<jname>` there is a `obs.header` file which contains the configuration that the PTUSE uses to observe a source, most of the information we require is in this file.
+There is also a `obs.info` file which has information about how the PTUSE split the `dspsr` command into subbands, this info does not need to be uploaded.
+The `obs.results` has a signal to noise ratio result of the folded result and the length of the observation in seconds, if the backend has not had enough time to finish the process required for this calculation the file may not exist and will need to be regenerated.
+
+In `/fred/oz005/timing/<jname>/<utc>/<beam_num>/<frequency>` it will also have the `obs.header`, `obs.info` and `obs.results` file.
+The `metadata.json` is generated to upload the data to the MeeKAT data portal.
+This directory also contains all the archive files that output from the PTUSE in 8 second files.
+
+The `meertime.json` file required for ingest is created with the `psrdb/scripts/generate_meerkat_json.py` script.
+The inputs of the script are the `obs.header` file (this is where it gets most of the metadata) and the beam_num which is worked out from the directory structure.
+The script also works out the calibration type ("pre" or "post") from the UTC and finds and calibration files that are on disk.
+It uses the frequency summed archive in `/fred/oz005/kronos/<beam_num>/<utc>/<jname>/freq.sum` to work out the observation length and the ephemeris used for folding.
+
+
+## Uploading the observation
+
+To upload an observation we use the `metadata.json` and the script `psrdb/scripts/ingest_obs.py`.
+
+An example of the `metadata.json` file is shown below
+
+```
+{
+ "pulsarName": "J1705-1903",
+ "telescopeName": "MeerKAT",
+ "projectCode": "SCI-20180516-MB-05",
+ "delaycal_id": "20201022-0018",
+ "cal_type": "pre",
+ "cal_location": null,
+ "frequency": 1283.58203125,
+ "bandwidth": 856.0,
+ "nchan": 1024,
+ "beam": 3,
+ "nant": 61,
+ "nantEff": 61,
+ "npol": 2,
+ "obsType": "fold",
+ "utcStart": "2020-10-22-16:24:55",
+ "raj": "17:05:43.8502743",
+ "decj": "-19:03:41.32025",
+ "duration": 1999.5421942429905,
+ "nbit": 8,
+ "tsamp": 1.196261682242991,
+ "foldNbin": 1024,
+ "foldNchan": 1024,
+ "foldTsubint": 8,
+ "filterbankNbit": null,
+ "filterbankNpol": null,
+ "filterbankNchan": null,
+ "filterbankTsamp": null,
+ "filterbankDm": null,
+ "ephemerisText": "PSRJ \t J1705-1903\nRAJ  \t  17:05:43.8502743  \t  2.846e-4\nDECJ  \t  -19:03:41.32025  \t  3.999e-2\nF0  \t  403.17844370811329346  \t  1.289e-9\nDM  \t  57.50571096535851744  \t  5.052e-5\nF1  \t  -3.4428715379610950e-15  \t  6.438e-18\nPEPOCH  \t  56618  \t  \nPOSEPOCH  \t  56618  \t  \nDMEPOCH  \t  56618  \t  \nPMRA  \t  -4.3811121114291140e+0  \t  1.3305\nPMDEC  \t  -1.7888684789156335e+1  \t  13.2010\nPX  \t  0  \t  \nBINARY  \t  ELL1  \t  \nPB  \t  0.18395403344503906874  \t  3.598e-8\nA1  \t  0.10436244224177347356  \t  1.698e-6\nPBDOT  \t  -3.7178429899716451e-12  \t  1.539e-11\nTASC  \t  56582.21217308180756000000  \t  2.285e-4\nEPS1  \t  7.9300304759274633e-5  \t  1.779e-5\nEPS2  \t  -3.3596418853323134e-5  \t  7.178e-6\nTZRMJD  \t  58855.25408581254015800000  \t  \nTZRFRQ  \t  944.52099999999995816000  \t  \nTZRSITE  \t  meerkat  \t  \nEPHVER  \t  5  \t  \nCLK  \t  TT(TAI)  \t  \nUNITS  \t  TCB  \t  \nTIMEEPH  \t  IF99  \t  \nT2CMETHOD  \t  IAU2000B  \t  \nCORRECT_TROPOSPHERE  \t  N  \t  \nEPHEM  \t  DE421  \t  \n"
+}
+
+
+
+
+
diff --git a/docs/psrdb_modules.rst b/docs/psrdb_modules.rst
@@ -9,39 +9,6 @@ psrdb modules
 ================
 
 
-psrdb joins classes
-===================
-
-FoldedObservations
-------------------
-
-.. autoclass:: psrdb.joins.folded_observations.FoldedObservations
-   :members:
-
-GraphQLJoin
------------
-
-.. autoclass:: psrdb.joins.graphql_join.GraphQLJoin
-   :members:
-
-ProcessedObservations
----------------------
-
-.. autoclass:: psrdb.joins.processed_observations.ProcessedObservations
-   :members:
-
-PulsarsLaunches
----------------
-
-.. autoclass:: psrdb.joins.pulsar_launches.PulsarsLaunches
-   :members:
-
-ToaedObservations
------------------
-
-.. autoclass:: psrdb.joins.toaed_observations.ToaedObservations
-   :members:
-
 
 psrdb tables classes
 ===================
@@ -85,13 +52,13 @@ Foldings
 GraphQLClause
 -------------
 
-.. autoclass:: psrdb.tables.graphql_query.GraphQLClause
+.. autoclass:: psrdb.graphql_query.GraphQLClause
    :members:
 
 GraphQLTable
 ------------
 
-.. autoclass:: psrdb.tables.graphql_table.GraphQLTable
+.. autoclass:: psrdb.graphql_table.GraphQLTable
    :members:
 
 Instrumentconfigs