xml_maker-updated.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7dcf4d95",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import (absolute_import, division, print_function,\n",
    "                    unicode_literals)\n",
    "import os\n",
    "from datetime import datetime\n",
    "import sys\n",
    "\n",
    "from caom2 import SimpleObservation, Plane, Artifact, Part, Chunk,\\\n",
    "                  TypedOrderedDict, ObservationWriter, ProductType, \\\n",
    "                  ReleaseType, TypedList, Target, TargetPosition, \\\n",
    "                  TargetType, ObservationIntentType, Instrument, \\\n",
    "                  Telescope, Environment, DataProductType, Provenance, \\\n",
    "                  CalibrationLevel, Metrics, Proposal, Point, Slice, Axis,\\\n",
    "                  ObservableAxis, CoordAxis1D, CoordAxis2D, SpatialWCS,\\\n",
    "                  SpectralWCS, EnergyTransition, TemporalWCS, CoordFunction1D,\\\n",
    "                  RefCoord, PolarizationWCS, ChecksumURI, CoordBounds1D, CoordRange1D\n",
    "\n",
    "from checksumdir import dirhash\n",
    "import casatools\n",
    "import numpy\n",
    "\n",
    "msmd = casatools.msmetadata()\n",
    "ms = casatools.ms()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e7677644",
   "metadata": {},
   "outputs": [],
   "source": [
    "ms_file = '/home/h14471mj/e-merlin/casa6_docker/prod/TS8004_C_001_20190801/TS8004_C_001_20190801_avg.ms/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "67634341",
   "metadata": {},
   "outputs": [],
   "source": [
    "def basename(name):\n",
    "    base_name = os.path.dirname(name).split('/')[-1]\n",
    "    return base_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9340f8d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_mssources(ms_file):\n",
    "    # Get list of sources from measurement set\n",
    "    # To do: discern target and calibrators for CAOM Observation.targetName\n",
    "    msmd.open(ms_file)\n",
    "    mssources = ','.join(numpy.sort(msmd.fieldnames()))\n",
    "    #mssources = msmd.fieldnames()\n",
    "    msmd.done()\n",
    "    # logger.debug('Sources in MS {0}: {1}'.format(msfile, mssources))\n",
    "    return mssources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e8fb97fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_obs_name(ms_file):\n",
    "    msmd.open(ms_file)\n",
    "    obs_name = msmd.observatorynames()\n",
    "    msmd.done()\n",
    "    return obs_name[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0b2ab840",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_size(start_path = '.'):\n",
    "    total_size = 0\n",
    "    for dirpath, dirnames, filenames in os.walk(start_path):\n",
    "        for f in filenames:\n",
    "            fp = os.path.join(dirpath, f)\n",
    "            # skip if it is symbolic link\n",
    "            if not os.path.islink(fp):\n",
    "                total_size += os.path.getsize(fp)\n",
    "\n",
    "    return total_size\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bc3a4a5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FileInfo:\n",
    "    \"\"\"\n",
    "    Container for the metadata of a file:\n",
    "        - ID\n",
    "        - size\n",
    "        - name\n",
    "        - md5sum\n",
    "        - file_type\n",
    "        - encoding\n",
    "    \"\"\"\n",
    "    def __init__(self, id, size=None, name=None, md5sum=None, lastmod=None,\n",
    "                 file_type=None, encoding=None):\n",
    "        if not id:\n",
    "            raise AttributeError(\n",
    "                'ID of the file in Storage Inventory is required')\n",
    "        self.id = id\n",
    "        self.size = size\n",
    "        self.name = name\n",
    "        self.md5sum = md5sum\n",
    "        self.lastmod = lastmod\n",
    "        self.file_type = file_type\n",
    "        self.encoding = encoding\n",
    "\n",
    "    def __str__(self):\n",
    "        return (\n",
    "            'id={}, name={}, size={}, type={}, encoding={}, last modified={}, '\n",
    "            'md5sum={}'.format(self.id, self.name, self.size, self.file_type,\n",
    "                               self.encoding, date2ivoa(self.lastmod),\n",
    "                               self.md5sum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6396de9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_type(fqn):\n",
    "    \"\"\"Basic header extension to content_type lookup.\"\"\"\n",
    "    lower_fqn = fqn.lower()\n",
    "    if os.path.isdir(fqn):\n",
    "        return 'application/measurement-set'\n",
    "    elif lower_fqn.endswith('.fits') or lower_fqn.endswith('.fits.fz') or lower_fqn.endswith('.fits.bz2'):\n",
    "        return 'application/fits'\n",
    "    elif lower_fqn.endswith('.gif'):\n",
    "        return 'image/gif'\n",
    "    elif lower_fqn.endswith('.png'):\n",
    "        return 'image/png'\n",
    "    elif lower_fqn.endswith('.jpg'):\n",
    "        return 'image/jpeg'\n",
    "    elif lower_fqn.endswith('.tar.gz'):\n",
    "        return 'application/x-tar'\n",
    "    elif lower_fqn.endswith('.csv'):\n",
    "        return 'text/csv'\n",
    "    elif lower_fqn.endswith('.hdf5') or fqn.endswith('.h5'):\n",
    "        return 'application/x-hdf5'\n",
    "    else:\n",
    "        return 'text/plain'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d1b8e5d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_local_file_info(fqn):\n",
    "    \"\"\"\n",
    "    Gets descriptive metadata for a directory of measurement set files on disk.\n",
    "    :param fqn: Fully-qualified name of the file on disk.\n",
    "    :return: FileInfo, no scheme on the md5sum value.\n",
    "    \"\"\"\n",
    "    file_type_local = get_file_type(fqn)\n",
    "\n",
    "    if file_type_local == 'application/measurement-set':\n",
    "        file_size = get_size(fqn)\n",
    "        final_hash_val = dirhash(fqn)  # very slow, may need to remove in future\n",
    "        file_id = os.path.dirname(fqn).split('/')[-1]\n",
    "\n",
    "    else:\n",
    "        file_id = os.path.basename(fqn)\n",
    "        s = os.stat(fqn)\n",
    "        file_size = s.st_size\n",
    "        hash_md5 = md5()\n",
    "        with open(fqn, 'rb') as f:\n",
    "            for chunk in iter(lambda: f.read(4096), b''):\n",
    "                hash_md5.update(chunk)\n",
    "        final_hash_val = hash_md5.hexdigest()\n",
    "\n",
    "    meta = FileInfo(\n",
    "        id=file_id,\n",
    "        size=file_size,\n",
    "        md5sum=final_hash_val,\n",
    "        file_type=file_type_local,\n",
    "    )\n",
    "    return meta\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "21f5d1cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_scan_sum(ms_file):\n",
    "    ms.open(ms_file)\n",
    "    scan_sum = ms.getscansummary()\n",
    "    ms.close()\n",
    "    return scan_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "9fa4d221",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_xml_from_file(ms_file):\n",
    "    \n",
    "    obs_id = basename(ms_file)\n",
    "    observation = SimpleObservation('collection', obs_id)\n",
    "    observation.obs_type = 'science'\n",
    "    observation.intent = ObservationIntentType.SCIENCE\n",
    "    \n",
    "    observation.target = Target('TBD')\n",
    "    #observation.target_position = TargetPosition(str(find_mssources(ms_file)), 'J2000')\n",
    "    \n",
    "    observation.telescope = Telescope(get_obs_name(ms_file)[0])\n",
    "    \n",
    "    observation.planes = TypedOrderedDict(Plane)\n",
    "    plane = Plane(obs_id) # use for now\n",
    "    observation.planes[obs_id] = plane\n",
    "    \n",
    "    plane.artifacts = TypedOrderedDict(Artifact)\n",
    "    artifact = Artifact('uri:foo/bar', ProductType.SCIENCE, ReleaseType.META)\n",
    "    plane.artifacts['uri:foo/bar'] = artifact\n",
    "    \n",
    "    meta_data = get_local_file_info(ms_file)\n",
    "    \n",
    "    artifact.content_type = meta_data.file_type\n",
    "    artifact.content_length = meta_data.size\n",
    "    artifact.content_checksum = ChecksumURI('md5:{}'.format(meta_data.md5sum))\n",
    "    \n",
    "    artifact.parts = TypedOrderedDict(Part)\n",
    "    part = Part('name')\n",
    "    artifact.parts['name'] = part\n",
    "    part.product_type = ProductType.SCIENCE\n",
    "    \n",
    "    part.chunks = TypedList(Chunk)\n",
    "    \n",
    "    ms_summary_data = get_scan_sum(ms_file)\n",
    "    \n",
    "    for key, value in ms_summary_data.items():\n",
    "        chunk_dict = value['0']\n",
    "        chunk = Chunk()\n",
    "        part.chunks.append(chunk)\n",
    "        time_axis = CoordAxis1D(Axis('MJD', 's'))\n",
    "        chunk.time = TemporalWCS(time_axis)\n",
    "        chunk.time.exposure = chunk_dict['IntegrationTime']\n",
    "        chunk.time.range = CoordBounds1D()\n",
    "        chunk.time.range.start = RefCoord(0.5, value['0']['BeginTime']) # unsure what the first number refers to\n",
    "        chunk.time.range.end.val = RefCoord(1.5, value['0']['EndTime']) # unsure what the first number refers to\n",
    "    writer = ObservationWriter()\n",
    "    writer.write(observation, sys.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7952242",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_axis = CoordAxis1D(Axis('MJD', 's'))\n",
    "a = TemporalWCS(time_axis)\n",
    "a.range = CoordBounds1D()\n",
    "a.range.start = RefCoord(0.5, 1.)\n",
    "a.range.end = RefCoord(1.5, 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06956b91",
   "metadata": {},
   "outputs": [],
   "source": [
    "create_xml_from_file(ms_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7cb0070a",
   "metadata": {},
   "outputs": [],
   "source": [
    "observation = SimpleObservation('collection', 'observationID')\n",
    "observation.obs_type = 'science'\n",
    "observation.intent = ObservationIntentType.SCIENCE\n",
    "# observation.meta_release = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe1ed3f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "observation.target = Target(str(find_mssources(ms_file)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e46df04b",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = dirhash(ms_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dfa6a342",
   "metadata": {},
   "outputs": [],
   "source": [
    "b = get_scan_sum(ms_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "62d77744",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58696.972366898146"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b['1']['0']['BeginTime']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b7e0e223",
   "metadata": {},
   "outputs": [],
   "source": [
    "checky_boy = ChecksumURI('md5:{}'.format(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "12b78c25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "caom2.common.ChecksumURI(uri='md5:e56af5f3314f048648e19a82bf751f9a')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checky_boy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5038b7d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_axis = CoordAxis1D(Axis('MJD', 's'))\n",
    "a = TemporalWCS(time_axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd8d2b0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "a.mjdrefrange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29bd13a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def _build_time(start, end, tos):\n",
    "    bounds = CoordBounds1D()\n",
    "    if start is not None and end is not None:\n",
    "        start_date = ac.get_datetime_mjd(start)\n",
    "        end_date = ac.get_datetime_mjd(end)\n",
    "        start_ref_coord = RefCoord(0.5, start_date.value)\n",
    "        end_ref_coord = RefCoord(1.5, end_date.value)\n",
    "        bounds.samples.append(CoordRange1D(start_ref_coord, end_ref_coord))\n",
    "    exposure = None\n",
    "    if tos is not None:\n",
    "        exposure = float(ac.get_timedelta_in_s(tos))\n",
    "    return bounds, exposure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "0e27a32d",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_axis = CoordAxis1D(Axis('MJD', 's'))\n",
    "a = TemporalWCS(time_axis)\n",
    "a.range = CoordBounds1D()\n",
    "a.range.start = RefCoord(0.5, 1.)\n",
    "a.range.end = RefCoord(1.5, 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b70a0855",
   "metadata": {},
   "outputs": [],
   "source": [
    "a.range.start = RefCoord(0.5, 1.)\n",
    "a.range.end = RefCoord(1.5, 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3ee9a8a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "emerlin2caom_test",
   "language": "python",
   "name": "test"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}