Skip to content

Commit

Permalink
Refactor time names
Browse files Browse the repository at this point in the history
  • Loading branch information
@mraspaud
mraspaud committed Aug 29, 2024
1 parent 04206d9 commit 3d9188b
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 34 deletions.
6 changes: 3 additions & 3 deletions pygac/pod_reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ def _adjust_clock_drift(self):
error_utcs = np.array(error_utcs, dtype='datetime64[ms]')
# interpolate to get the clock offsets at the scan line utcs
# the clock_error is given in seconds, so offsets are in seconds, too.
offsets = np.interp(self._utcs.astype(np.uint64),
offsets = np.interp(self._times_as_np_datetime64.astype(np.uint64),
error_utcs.astype(np.uint64),
clock_error)
LOG.info("Adjusting for clock drift of %s to %s seconds.",
Expand All @@ -506,7 +506,7 @@ def _adjust_clock_drift(self):
num_lines = max_line - min_line + 1
missed_lines = np.setdiff1d(np.arange(min_line, max_line+1), scan_lines)
missed_utcs = ((missed_lines - scan_lines[0])*np.timedelta64(scan_rate, "ms")
+ self._utcs[0])
+ self._times_as_np_datetime64[0])
# calculate the missing geo locations
try:
missed_lons, missed_lats = self._compute_missing_lonlat(missed_utcs)
Expand Down Expand Up @@ -539,7 +539,7 @@ def _adjust_clock_drift(self):
# set corrected values
self.lons = slerp_res[:, :, 0]
self.lats = slerp_res[:, :, 1]
self._utcs -= (offsets * 1000).astype('timedelta64[ms]')
self._times_as_np_datetime64 -= (offsets * 1000).astype('timedelta64[ms]')

toc = datetime.datetime.now()
LOG.debug("clock drift adjustment took %s", str(toc - tic))
Expand Down
41 changes: 21 additions & 20 deletions pygac/reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ def __init__(self, interpolate_coords=True, adjust_clock_drift=True,
self.scans = None
self.spacecraft_name = None
self.spacecraft_id = None
self._utcs = None
self._times_as_np_datetime64 = None
self.lats = None
self.lons = None
self.tle_lines = None
Expand All @@ -154,11 +154,10 @@ def __init__(self, interpolate_coords=True, adjust_clock_drift=True,
self.calibration_parameters = dict(custom_coeffs=custom_calibration,
coeffs_file=calibration_file)


@property
def _times(self):
def times(self):
"""Get the UTCs as datetime.datetime."""
return self.to_datetime(self._utcs)
return self.to_datetime(self._times_as_np_datetime64)

@property
def filename(self):
Expand Down Expand Up @@ -378,7 +377,7 @@ def save(self, start_line, end_line, output_file_prefix="PyGAC", output_dir="./"
raise ValueError("All data is masked out.")
gac_io.save_gac(
self.spacecraft_name,
self._utcs, self.lats, self.lons,
self._times_as_np_datetime64, self.lats, self.lons,
channels[:, :, 0], channels[:, :, 1],
channels[:, :, 2], channels[:, :, 3],
channels[:, :, 4], channels[:, :, 5],
Expand Down Expand Up @@ -453,20 +452,20 @@ def get_times(self):
UTC timestamps
"""
if self._utcs is None:
if self._times_as_np_datetime64 is None:
# Read timestamps
year, jday, msec = self._get_times_from_file()

# Correct invalid values
year, jday, msec = self.correct_times_median(year=year, jday=jday,
msec=msec)
self._utcs = self.to_datetime64(year=year, jday=jday, msec=msec)
self._times_as_np_datetime64 = self.to_datetime64(year=year, jday=jday, msec=msec)
try:
self._utcs = self.correct_times_thresh()
self._times_as_np_datetime64 = self.correct_times_thresh()
except TimestampMismatch as err:
LOG.error(str(err))

return self._utcs
return self._times_as_np_datetime64

@staticmethod
def to_datetime64(year, jday, msec):
Expand Down Expand Up @@ -516,7 +515,9 @@ def lineno2msec(self, scan_line_number):
def get_sun_earth_distance_correction(self):
"""Get the julian day and the sun-earth distance correction."""
self.get_times()
jday = self._times[0].timetuple().tm_yday
# Shouldn't we use the start time from the header if available?
start_time = self._times_as_np_datetime64[0]
jday = (start_time - start_time.astype("datetime64[Y]")).astype("timedelta64[D]").astype(int) + 1
return calculate_sun_earth_distance_correction(jday)

def update_meta_data(self):
Expand Down Expand Up @@ -798,7 +799,7 @@ def get_tle_lines(self):
return self.tle_lines
self.get_times()
tle_data = self.read_tle_file(self.get_tle_file())
sdate = self._utcs[0]
sdate = self._times_as_np_datetime64[0]
dates = self.tle2datetime64(
np.array([float(line[18:32]) for line in tle_data[::2]]))

Expand Down Expand Up @@ -855,7 +856,7 @@ def _get_sat_angles_with_tle(self):
tle1, tle2 = self.get_tle_lines()
orb = Orbital(self.spacecrafts_orbital[self.spacecraft_id],
line1=tle1, line2=tle2)
sat_azi, sat_elev = orb.get_observer_look(self._times[:, np.newaxis],
sat_azi, sat_elev = orb.get_observer_look(self._times_as_np_datetime64[:, np.newaxis],
self.lons, self.lats, 0)
return sat_azi, sat_elev

Expand All @@ -869,7 +870,7 @@ def _get_sat_angles_without_tle(self):
self.lons[:, mid_column][:, np.newaxis],
self.lats[:, mid_column][:, np.newaxis], # approximate satellite position
sat_alt, # approximate satellite altitude
self._times[:, np.newaxis],
self._times_as_np_datetime64[:, np.newaxis],
self.lons, self.lats, 0)
# Sometimes (pyorbital <= 1.6.1) the get_observer_look_not_tle returns nodata instead of 90.
# Problem solved with https://github.com/pytroll/pyorbital/pull/77
Expand Down Expand Up @@ -901,7 +902,7 @@ def get_angles(self):
"""
self.get_times()
self.get_lonlat()
times = self._times
times = self._times_as_np_datetime64
sat_azi, sat_elev = self.get_sat_angles()

sat_zenith = 90 - sat_elev
Expand Down Expand Up @@ -1101,15 +1102,15 @@ def correct_times_thresh(self, max_diff_from_t0_head=6*60*1000,

# Check whether scanline number increases monotonically
nums = self.scans["scan_line_number"]
results.update({'t': self._utcs.copy(), 'n': nums})
results.update({'t': self._times_as_np_datetime64.copy(), 'n': nums})
if np.any(np.diff(nums) < 0):
LOG.error("Cannot perform timestamp correction. Scanline number "
"does not increase monotonically.")
results['fail_reason'] = "Scanline number jumps backwards"
return results

# Convert time to milliseconds since 1970-01-01
t = self._utcs.astype("i8")
t = self._times_as_np_datetime64.astype("i8")
try:
t0_head = np.array([self.get_header_timestamp().isoformat()],
dtype="datetime64[ms]").astype("i8")[0]
Expand Down Expand Up @@ -1151,10 +1152,10 @@ def correct_times_thresh(self, max_diff_from_t0_head=6*60*1000,
# Replace timestamps deviating more than a certain threshold from the
# ideal timestamp with the ideal timestamp.
corrupt_lines = np.where(np.fabs(t - tn) > max_diff_from_ideal_t)
self._utcs[corrupt_lines] = tn[corrupt_lines].astype(dt64_msec)
self._times_as_np_datetime64[corrupt_lines] = tn[corrupt_lines].astype(dt64_msec)
LOG.debug("Corrected %s timestamp(s)", str(len(corrupt_lines[0])))

return self._utcs
return self._times_as_np_datetime64

@property
@abstractmethod
Expand All @@ -1177,7 +1178,7 @@ def is_tsm_affected(self):
"""
self.get_times()
ts, te = self.to_datetime(self._utcs[[0, -1]])
ts, te = self.to_datetime(self._times_as_np_datetime64[[0, -1]])
try:
for interval in self.tsm_affected_intervals[self.spacecraft_id]:
if ts >= interval[0] and te <= interval[1]:
Expand All @@ -1200,7 +1201,7 @@ def get_midnight_scanline(self):
"""
self.get_times()
d0 = np.datetime64(datetime.date(1970, 1, 1), 'D')
days = (self._utcs.astype('datetime64[D]') - d0).astype(int)
days = (self._times_as_np_datetime64.astype('datetime64[D]') - d0).astype(int)
incr = np.where(np.diff(days) == 1)[0]
if len(incr) != 1:
if len(incr) > 1:
Expand Down
6 changes: 3 additions & 3 deletions pygac/tests/test_pod.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ def test__adjust_clock_drift(self, avhrr_gac, get_tle_lines,

# prepare the reader
reader.scans = {"scan_line_number": scan_lines}
reader._utcs = scan_utcs
reader._times_as_np_datetime64 = scan_utcs
reader.lons = scan_lons
reader.lats = scan_lats
reader.spacecraft_name = sat_name
Expand Down Expand Up @@ -276,7 +276,7 @@ def test__adjust_clock_drift(self, avhrr_gac, get_tle_lines,
# check output
# use allclose for geolocations, because the slerp interpolation
# includes a transormation to cartesian coordinates and back to lon, lats.
numpy.testing.assert_array_equal(reader._utcs, expected_utcs)
numpy.testing.assert_array_equal(reader._times_as_np_datetime64, expected_utcs)
numpy.testing.assert_allclose(reader.lons, expected_lons)
numpy.testing.assert_allclose(reader.lats, expected_lats)

Expand All @@ -288,7 +288,7 @@ def test__adjust_clock_drift(self, avhrr_gac, get_tle_lines,
def test__adjust_clock_drift_without_tle(self, get_tle_lines, get_offsets):
"""Test that clockdrift adjustment can handle missing TLE data."""
reader = self.reader
reader._utcs = np.zeros(10, dtype='datetime64[ms]')
reader._times_as_np_datetime64 = np.zeros(10, dtype='datetime64[ms]')
reader.scans = {"scan_line_number": np.arange(10)}
get_offsets.return_value = np.zeros(10), np.zeros(10)
get_tle_lines.side_effect = NoTLEData('No TLE data available')
Expand Down
16 changes: 8 additions & 8 deletions pygac/tests/test_reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ def test_midnight_scanline(self):
scanline2 = None

for utcs, scan_line in zip((utcs1, utcs2), (scanline1, scanline2)):
self.reader._utcs = utcs
self.reader._times_as_np_datetime64 = utcs
self.assertEqual(self.reader.get_midnight_scanline(), scan_line,
msg='Incorrect midnight scanline')

Expand Down Expand Up @@ -422,7 +422,7 @@ def test_get_tle_lines(self, read_tle_file, *mocks):

read_tle_file.return_value = tle_data
for time, tle_idx in expected.items():
self.reader._utcs = np.array([time], dtype='datetime64[ms]')
self.reader._times_as_np_datetime64 = np.array([time], dtype='datetime64[ms]')
self.reader.tle_lines = None
if tle_idx is None:
self.assertRaises(NoTLEData, self.reader.get_tle_lines)
Expand All @@ -438,7 +438,7 @@ def test_get_sat_angles_without_tle_at_nadir(self, get_tle_lines):
rng = np.random.RandomState(125)
self.reader.lons = rng.rand(100, 409) * 90
self.reader.lats = rng.rand(100, 409) * 90
self.reader._utcs = np.array(
self.reader._times_as_np_datetime64 = np.array(
[numpy.datetime64(datetime.datetime(1980, 1, 3, 11, 47, 15, 469000)) for date in range(100)])
sat_azi, sat_elev = self.reader.get_sat_angles()
self.assertEqual(np.sum(np.isnan(sat_elev)), 0)
Expand Down Expand Up @@ -481,7 +481,7 @@ def test_get_sat_angles_without_tle(self, get_tle_lines):
[71.7390608, 71.78108857, 70.01126173, 69.99449392, 69.97770882,
69.9609375, 69.9442054, 69.92751555, 69.91086544, 61.67769951,
61.52802445]])
self.reader._utcs = np.array([numpy.datetime64(datetime.datetime(1980, 1, 3, 11, 47, 15, 469000)),
self.reader._times_as_np_datetime64 = np.array([numpy.datetime64(datetime.datetime(1980, 1, 3, 11, 47, 15, 469000)),
numpy.datetime64(datetime.datetime(1980, 1, 3, 11, 47, 15, 969000)),
numpy.datetime64(datetime.datetime(1980, 1, 3, 11, 47, 16, 469000)),
numpy.datetime64(datetime.datetime(1980, 1, 3, 11, 47, 16, 969000)),
Expand Down Expand Up @@ -512,7 +512,7 @@ def test_get_angles(self, get_corrupt_mask):
self.reader.tle_lines = [
'1 11060U 78096A 80003.54792075 .00000937 00000-0 52481-3 0 2588\r\n', # noqa
'2 11060 98.9783 332.1605 0012789 88.8047 271.4583 14.11682873 63073\r\n'] # noqa
self.reader._utcs = np.array(
self.reader._times_as_np_datetime64 = np.array(
[315748035469, 315748359969,
315751135469, 315754371969,
315754371969]).astype('datetime64[ms]')
Expand Down Expand Up @@ -592,17 +592,17 @@ def test_correct_times_thresh(self, get_header_timestamp):

# Mock reader
get_header_timestamp.return_value = header_time
self.reader._utcs = msecs.astype(">M8[ms]")
self.reader._times_as_np_datetime64 = msecs.astype(">M8[ms]")
self.reader.scans = np.array(scan_line_numbers,
dtype=[("scan_line_number", ">u2")])

# Test correction
self.reader.correct_times_thresh()
numpy.testing.assert_array_equal(self.reader._utcs, utcs_expected)
numpy.testing.assert_array_equal(self.reader._times_as_np_datetime64, utcs_expected)

def test_calculate_sun_earth_distance_correction(self):
"""Test the calculate sun earth distance correction method."""
self.reader._utcs = np.array([315748035469, 315748359969,
self.reader._times_as_np_datetime64 = np.array([315748035469, 315748359969,
315751135469, 315754371969,
315754371969]).astype('datetime64[ms]')
corr = self.reader.get_sun_earth_distance_correction()
Expand Down

0 comments on commit 3d9188b

Please sign in to comment.