Write patch results for RR, DR, etc. when write_patch_results=True

CHANGELOG.rst

@@ -37,6 +37,7 @@ API Changes
 - Changed estimate_cov with method='shot' to only return the diagonal, rather than gratuitously
   making a full, mostly empty diagonal matrix. (#166)
 - Changed name of Catalog.write kwarg from cat_precision to just precision. (#169)
+- Added additionaly information in the header of output files to enable ``from_file``. (#172)
 
 
 Performance improvements
@@ -77,6 +78,14 @@ New features
   three-point method, for which it is the default. (#169)
 - Added ``angle_slop`` option to separately tune the allowed angular slop from using cells,
   irrespective of the binning. (#170)
+- Added ``algo`` option to 3-point ``process`` functions to conrol whether to use new
+  multipole algorithm or the old triangle algorithm. (#171)
+- Added serialization of rr, dr, etc. when writing with write_patch_results=True option,
+  so you no longer have to separately write files for them to recover the covariance. (#172)
+- Added :ref:`from_file <GGCorrlation.from_file>` class methods to construct a Correlation
+  object from a file without needing to know the correct configuration parameters. (#172)
+- Added ``write_cov`` option to write functions to include the covariance in the output file.
+  (#172)
 
 
 Bug fixes

tests/Tutorial.ipynb

@@ -252,7 +252,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "## {'coords': 'spherical', 'metric': 'Euclidean', 'sep_units': 'arcmin', 'bin_type': 'Log'}\n",
+      "## {'min_sep': 1.0, 'max_sep': 400.0, 'nbins': 100, 'sep_units': 'arcmin', 'verbose': 2, 'output_dots': True, 'coords': 'spherical', 'metric': 'Euclidean'}\n",
       "#   r_nom       meanr       meanlogr       xip          xim         xip_im       xim_im     sigma_xip    sigma_xim      weight       npairs   \n",
       "  1.0304e+00   1.0311e+00   3.0480e-02   7.5773e-06   1.6730e-07   7.0287e-08  -3.5157e-08   1.8983e-08   1.8983e-08   2.0181e+05   2.0181e+05\n",
       "  1.0940e+00   1.0948e+00   9.0389e-02   7.3854e-06   3.5900e-07  -2.0456e-08   1.5109e-08   1.7939e-08   1.7939e-08   2.2597e+05   2.2597e+05\n",

tests/mpi_test.py

@@ -344,14 +344,10 @@ def do_mpi_cov(comm, method, output=True):
 
     # Finally, read back in from the save file and redo the covariance.
     rng = np.random.default_rng(31415)
-    gg = treecorr.GGCorrelation(bin_size=0.3, min_sep=10., max_sep=50., rng=rng)
-    ng = treecorr.NGCorrelation(bin_size=0.3, min_sep=10., max_sep=50., rng=rng)
-    nn = treecorr.NNCorrelation(bin_size=0.3, min_sep=10., max_sep=50., rng=rng)
-    rr = treecorr.NNCorrelation(bin_size=0.3, min_sep=10., max_sep=50., rng=rng)
-    gg.read(gg_save_file)
-    ng.read(ng_save_file)
-    nn.read(nn_save_file)
-    rr.read(rr_save_file)
+    gg = treecorr.GGCorrelation.from_file(gg_save_file, rng=rng)
+    ng = treecorr.NGCorrelation.from_file(ng_save_file, rng=rng)
+    nn = treecorr.NNCorrelation.from_file(nn_save_file, rng=rng)
+    rr = treecorr.NNCorrelation.from_file(rr_save_file, rng=rng)
 
     ng.calculateXi()
     nn.calculateXi(rr=rr)

tests/test_gg.py

@@ -154,7 +154,7 @@ def test_direct():
 
     ascii_name = 'output/gg_ascii.txt'
     gg.write(ascii_name, precision=16)
-    gg3 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins)
+    gg3 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, bin_type='Log')
     gg3.read(ascii_name)
     np.testing.assert_allclose(gg3.npairs, gg.npairs)
     np.testing.assert_allclose(gg3.weight, gg.weight)
@@ -165,6 +165,22 @@ def test_direct():
     np.testing.assert_allclose(gg3.xim, gg.xim)
     np.testing.assert_allclose(gg3.xim_im, gg.xim_im)
 
+    # Check that the repr is minimal
+    assert repr(gg3) == f'GGCorrelation(min_sep={min_sep}, max_sep={max_sep}, nbins={nbins})'
+
+    # New in version 5.0 is a simpler API for reading
+    with CaptureLog() as cl:
+        gg3b = treecorr.GGCorrelation.from_file(ascii_name, logger=cl.logger)
+    assert ascii_name in cl.output
+    np.testing.assert_allclose(gg3b.npairs, gg.npairs)
+    np.testing.assert_allclose(gg3b.weight, gg.weight)
+    np.testing.assert_allclose(gg3b.meanr, gg.meanr)
+    np.testing.assert_allclose(gg3b.meanlogr, gg.meanlogr)
+    np.testing.assert_allclose(gg3b.xip, gg.xip)
+    np.testing.assert_allclose(gg3b.xip_im, gg.xip_im)
+    np.testing.assert_allclose(gg3b.xim, gg.xim)
+    np.testing.assert_allclose(gg3b.xim_im, gg.xim_im)
+
     try:
         import fitsio
     except ImportError:
@@ -183,6 +199,16 @@ def test_direct():
         np.testing.assert_allclose(gg4.xim, gg.xim)
         np.testing.assert_allclose(gg4.xim_im, gg.xim_im)
 
+        gg4b = treecorr.GGCorrelation.from_file(fits_name)
+        np.testing.assert_allclose(gg4b.npairs, gg.npairs)
+        np.testing.assert_allclose(gg4b.weight, gg.weight)
+        np.testing.assert_allclose(gg4b.meanr, gg.meanr)
+        np.testing.assert_allclose(gg4b.meanlogr, gg.meanlogr)
+        np.testing.assert_allclose(gg4b.xip, gg.xip)
+        np.testing.assert_allclose(gg4b.xip_im, gg.xip_im)
+        np.testing.assert_allclose(gg4b.xim, gg.xim)
+        np.testing.assert_allclose(gg4b.xim_im, gg.xim_im)
+
     with assert_raises(TypeError):
         gg2 += config
     gg4 = treecorr.GGCorrelation(min_sep=min_sep/2, max_sep=max_sep, nbins=nbins)
@@ -566,8 +592,7 @@ def test_gg():
     np.testing.assert_allclose(data['npairs'], gg.npairs)
 
     # Check the read function
-    gg2 = treecorr.GGCorrelation(bin_size=0.1, min_sep=1., max_sep=100., sep_units='arcmin')
-    gg2.read(out_file_name)
+    gg2 = treecorr.GGCorrelation.from_file(out_file_name)
     np.testing.assert_allclose(gg2.logr, gg.logr)
     np.testing.assert_allclose(gg2.meanr, gg.meanr)
     np.testing.assert_allclose(gg2.meanlogr, gg.meanlogr)

tests/test_ggg.py

@@ -18,7 +18,7 @@
 import time
 
 from test_helper import do_pickle, assert_raises, timer, is_ccw, is_ccw_3d
-from test_helper import get_from_wiki
+from test_helper import get_from_wiki, CaptureLog
 
 @timer
 def test_direct_logruv():
@@ -336,6 +336,28 @@ def test_direct_logruv():
     np.testing.assert_allclose(ggg3.gam3r, ggg.gam3r)
     np.testing.assert_allclose(ggg3.gam3i, ggg.gam3i)
 
+    # New in version 5.0 is a simpler API for reading
+    with CaptureLog() as cl:
+        ggg3b = treecorr.GGGCorrelation.from_file(ascii_name, logger=cl.logger)
+    assert ascii_name in cl.output
+    np.testing.assert_allclose(ggg3b.weight, ggg.weight)
+    np.testing.assert_allclose(ggg3b.meand1, ggg.meand1)
+    np.testing.assert_allclose(ggg3b.meand2, ggg.meand2)
+    np.testing.assert_allclose(ggg3b.meand3, ggg.meand3)
+    np.testing.assert_allclose(ggg3b.meanlogd1, ggg.meanlogd1)
+    np.testing.assert_allclose(ggg3b.meanlogd2, ggg.meanlogd2)
+    np.testing.assert_allclose(ggg3b.meanlogd3, ggg.meanlogd3)
+    np.testing.assert_allclose(ggg3b.meanu, ggg.meanu)
+    np.testing.assert_allclose(ggg3b.meanv, ggg.meanv)
+    np.testing.assert_allclose(ggg3b.gam0r, ggg.gam0r)
+    np.testing.assert_allclose(ggg3b.gam0i, ggg.gam0i)
+    np.testing.assert_allclose(ggg3b.gam1r, ggg.gam1r)
+    np.testing.assert_allclose(ggg3b.gam1i, ggg.gam1i)
+    np.testing.assert_allclose(ggg3b.gam2r, ggg.gam2r)
+    np.testing.assert_allclose(ggg3b.gam2i, ggg.gam2i)
+    np.testing.assert_allclose(ggg3b.gam3r, ggg.gam3r)
+    np.testing.assert_allclose(ggg3b.gam3i, ggg.gam3i)
+
     try:
         import fitsio
     except ImportError:
@@ -365,6 +387,25 @@ def test_direct_logruv():
         np.testing.assert_allclose(ggg4.gam3r, ggg.gam3r)
         np.testing.assert_allclose(ggg4.gam3i, ggg.gam3i)
 
+        ggg4b = treecorr.GGGCorrelation.from_file(fits_name)
+        np.testing.assert_allclose(ggg4b.weight, ggg.weight)
+        np.testing.assert_allclose(ggg4b.meand1, ggg.meand1)
+        np.testing.assert_allclose(ggg4b.meand2, ggg.meand2)
+        np.testing.assert_allclose(ggg4b.meand3, ggg.meand3)
+        np.testing.assert_allclose(ggg4b.meanlogd1, ggg.meanlogd1)
+        np.testing.assert_allclose(ggg4b.meanlogd2, ggg.meanlogd2)
+        np.testing.assert_allclose(ggg4b.meanlogd3, ggg.meanlogd3)
+        np.testing.assert_allclose(ggg4b.meanu, ggg.meanu)
+        np.testing.assert_allclose(ggg4b.meanv, ggg.meanv)
+        np.testing.assert_allclose(ggg4b.gam0r, ggg.gam0r)
+        np.testing.assert_allclose(ggg4b.gam0i, ggg.gam0i)
+        np.testing.assert_allclose(ggg4b.gam1r, ggg.gam1r)
+        np.testing.assert_allclose(ggg4b.gam1i, ggg.gam1i)
+        np.testing.assert_allclose(ggg4b.gam2r, ggg.gam2r)
+        np.testing.assert_allclose(ggg4b.gam2i, ggg.gam2i)
+        np.testing.assert_allclose(ggg4b.gam3r, ggg.gam3r)
+        np.testing.assert_allclose(ggg4b.gam3i, ggg.gam3i)
+
     assert ggg.var_method == 'shot'  # Only option currently.
 
     with assert_raises(TypeError):
@@ -1602,11 +1643,7 @@ def test_ggg_logruv():
         # Check the read function
         # Note: These don't need the flatten.
         # The read function should reshape them to the right shape.
-        ggg2 = treecorr.GGGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins,
-                                   min_u=min_u, max_u=max_u, min_v=min_v, max_v=max_v,
-                                   nubins=nubins, nvbins=nvbins,
-                                   sep_units='arcmin', verbose=1, bin_type='LogRUV')
-        ggg2.read(out_file_name1)
+        ggg2 = treecorr.GGGCorrelation.from_file(out_file_name1)
         np.testing.assert_allclose(ggg2.logr, ggg.logr)
         np.testing.assert_allclose(ggg2.u, ggg.u)
         np.testing.assert_allclose(ggg2.v, ggg.v)
@@ -2534,6 +2571,23 @@ def test_direct_logsas():
     np.testing.assert_allclose(ggg3.gam2, ggg.gam2)
     np.testing.assert_allclose(ggg3.gam3, ggg.gam3)
 
+    # Check that the repr is minimal
+    assert repr(ggg3) == f"GGGCorrelation(min_sep={min_sep}, bin_size={bin_size}, nbins={nbins}, nphi_bins={nphi_bins})"
+
+    # New in version 5.0 is a simpler API for reading
+    ggg3b = treecorr.GGGCorrelation.from_file(ascii_name)
+    np.testing.assert_allclose(ggg3b.weight, ggg.weight)
+    np.testing.assert_allclose(ggg3b.meand1, ggg.meand1)
+    np.testing.assert_allclose(ggg3b.meand2, ggg.meand2)
+    np.testing.assert_allclose(ggg3b.meand3, ggg.meand3)
+    np.testing.assert_allclose(ggg3b.meanlogd1, ggg.meanlogd1)
+    np.testing.assert_allclose(ggg3b.meanlogd2, ggg.meanlogd2)
+    np.testing.assert_allclose(ggg3b.meanlogd3, ggg.meanlogd3)
+    np.testing.assert_allclose(ggg3b.gam0, ggg.gam0)
+    np.testing.assert_allclose(ggg3b.gam1, ggg.gam1)
+    np.testing.assert_allclose(ggg3b.gam2, ggg.gam2)
+    np.testing.assert_allclose(ggg3b.gam3, ggg.gam3)
+
     try:
         import fitsio
     except ImportError:
@@ -2558,6 +2612,19 @@ def test_direct_logsas():
         np.testing.assert_allclose(ggg4.gam2, ggg.gam2)
         np.testing.assert_allclose(ggg4.gam3, ggg.gam3)
 
+        ggg4b = treecorr.GGGCorrelation.from_file(fits_name)
+        np.testing.assert_allclose(ggg4b.weight, ggg.weight)
+        np.testing.assert_allclose(ggg4b.meand1, ggg.meand1)
+        np.testing.assert_allclose(ggg4b.meand2, ggg.meand2)
+        np.testing.assert_allclose(ggg4b.meand3, ggg.meand3)
+        np.testing.assert_allclose(ggg4b.meanlogd1, ggg.meanlogd1)
+        np.testing.assert_allclose(ggg4b.meanlogd2, ggg.meanlogd2)
+        np.testing.assert_allclose(ggg4b.meanlogd3, ggg.meanlogd3)
+        np.testing.assert_allclose(ggg4b.gam0, ggg.gam0)
+        np.testing.assert_allclose(ggg4b.gam1, ggg.gam1)
+        np.testing.assert_allclose(ggg4b.gam2, ggg.gam2)
+        np.testing.assert_allclose(ggg4b.gam3, ggg.gam3)
+
     # Split into patches to test the list-based version of the code.
     catp = treecorr.Catalog(x=x, y=y, w=w, g1=g1, g2=g2, npatch=3, rng=rng)
 
@@ -3724,9 +3791,9 @@ def test_ggg_logsas():
     except ImportError:
         pass
     else:
-        out_file_name1 = os.path.join('output','ggg_out1_logsas.fits')
-        ggg.write(out_file_name1)
-        data = fitsio.read(out_file_name1)
+        out_file_name = os.path.join('output','ggg_out_logsas.fits')
+        ggg.write(out_file_name)
+        data = fitsio.read(out_file_name)
         np.testing.assert_allclose(data['d2_nom'], np.exp(ggg.logd2).flatten())
         np.testing.assert_allclose(data['d3_nom'], np.exp(ggg.logd3).flatten())
         np.testing.assert_allclose(data['phi_nom'], ggg.phi.flatten())
@@ -3755,10 +3822,7 @@ def test_ggg_logsas():
         # Check the read function
         # Note: These don't need the flatten.
         # The read function should reshape them to the right shape.
-        ggg2 = treecorr.GGGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins,
-                                       min_phi=min_phi, max_phi=max_phi, nphi_bins=nphi_bins,
-                                       sep_units='arcmin', phi_units='degrees', bin_type='LogSAS')
-        ggg2.read(out_file_name1)
+        ggg2 = treecorr.GGGCorrelation.from_file(out_file_name)
         np.testing.assert_allclose(ggg2.logd2, ggg.logd2)
         np.testing.assert_allclose(ggg2.logd3, ggg.logd3)
         np.testing.assert_allclose(ggg2.phi, ggg.phi)
@@ -4334,6 +4398,20 @@ def test_direct_logmultipole_auto():
     np.testing.assert_allclose(ggg3.meanlogd2, ggg.meanlogd2)
     np.testing.assert_allclose(ggg3.meanlogd3, ggg.meanlogd3)
 
+    ggg3b = treecorr.GGGCorrelation.from_file(ascii_name)
+    np.testing.assert_allclose(ggg3b.ntri, ggg.ntri)
+    np.testing.assert_allclose(ggg3b.weight, ggg.weight)
+    np.testing.assert_allclose(ggg3b.gam0, ggg.gam0)
+    np.testing.assert_allclose(ggg3b.gam1, ggg.gam1)
+    np.testing.assert_allclose(ggg3b.gam2, ggg.gam2)
+    np.testing.assert_allclose(ggg3b.gam3, ggg.gam3)
+    np.testing.assert_allclose(ggg3b.meand1, ggg.meand1)
+    np.testing.assert_allclose(ggg3b.meand2, ggg.meand2)
+    np.testing.assert_allclose(ggg3b.meand3, ggg.meand3)
+    np.testing.assert_allclose(ggg3b.meanlogd1, ggg.meanlogd1)
+    np.testing.assert_allclose(ggg3b.meanlogd2, ggg.meanlogd2)
+    np.testing.assert_allclose(ggg3b.meanlogd3, ggg.meanlogd3)
+
     try:
         import fitsio
     except ImportError:
@@ -4357,6 +4435,20 @@ def test_direct_logmultipole_auto():
         np.testing.assert_allclose(ggg4.meanlogd2, ggg.meanlogd2)
         np.testing.assert_allclose(ggg4.meanlogd3, ggg.meanlogd3)
 
+        ggg4b = treecorr.GGGCorrelation.from_file(fits_name)
+        np.testing.assert_allclose(ggg4b.ntri, ggg.ntri)
+        np.testing.assert_allclose(ggg4b.weight, ggg.weight)
+        np.testing.assert_allclose(ggg3b.gam0, ggg.gam0)
+        np.testing.assert_allclose(ggg3b.gam1, ggg.gam1)
+        np.testing.assert_allclose(ggg3b.gam2, ggg.gam2)
+        np.testing.assert_allclose(ggg3b.gam3, ggg.gam3)
+        np.testing.assert_allclose(ggg4b.meand1, ggg.meand1)
+        np.testing.assert_allclose(ggg4b.meand2, ggg.meand2)
+        np.testing.assert_allclose(ggg4b.meand3, ggg.meand3)
+        np.testing.assert_allclose(ggg4b.meanlogd1, ggg.meanlogd1)
+        np.testing.assert_allclose(ggg4b.meanlogd2, ggg.meanlogd2)
+        np.testing.assert_allclose(ggg4b.meanlogd3, ggg.meanlogd3)
+
 @timer
 def test_direct_logmultipole_spherical():
     # Repeat in spherical coords
@@ -4867,7 +4959,6 @@ def test_map3_logmultipole():
     np.testing.assert_allclose(map3, true_map3[mask], rtol=0.1)
 
     # The default is to use max_n phi bins, which should also work fine here.
-    gggm.read(os.path.join('data',out_name))
     ggg = gggm.toSAS()
     map3 = ggg.calculateMap3(R=R)[0]
     np.testing.assert_allclose(map3, true_map3[mask], rtol=0.1)

tests/test_kg.py

@@ -147,7 +147,7 @@ def test_direct():
 
     ascii_name = 'output/kg_ascii.txt'
     kg.write(ascii_name, precision=16)
-    kg3 = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins)
+    kg3 = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, bin_type='Log')
     kg3.read(ascii_name)
     np.testing.assert_allclose(kg3.npairs, kg.npairs)
     np.testing.assert_allclose(kg3.weight, kg.weight)
@@ -156,6 +156,20 @@ def test_direct():
     np.testing.assert_allclose(kg3.xi, kg.xi)
     np.testing.assert_allclose(kg3.xi_im, kg.xi_im)
 
+    # Check that the repr is minimal
+    assert repr(kg3) == f'KGCorrelation(min_sep={min_sep}, max_sep={max_sep}, nbins={nbins})'
+
+    # New in version 5.0 is a simpler API for reading
+    with CaptureLog() as cl:
+        kg3b = treecorr.KGCorrelation.from_file(ascii_name, logger=cl.logger)
+    assert ascii_name in cl.output
+    np.testing.assert_allclose(kg3b.npairs, kg.npairs)
+    np.testing.assert_allclose(kg3b.weight, kg.weight)
+    np.testing.assert_allclose(kg3b.meanr, kg.meanr)
+    np.testing.assert_allclose(kg3b.meanlogr, kg.meanlogr)
+    np.testing.assert_allclose(kg3b.xi, kg.xi)
+    np.testing.assert_allclose(kg3b.xi_im, kg.xi_im)
+
     try:
         import fitsio
     except ImportError:
@@ -172,6 +186,14 @@ def test_direct():
         np.testing.assert_allclose(kg4.xi, kg.xi)
         np.testing.assert_allclose(kg4.xi_im, kg.xi_im)
 
+        kg4b = treecorr.KGCorrelation.from_file(fits_name)
+        np.testing.assert_allclose(kg4b.npairs, kg.npairs)
+        np.testing.assert_allclose(kg4b.weight, kg.weight)
+        np.testing.assert_allclose(kg4b.meanr, kg.meanr)
+        np.testing.assert_allclose(kg4b.meanlogr, kg.meanlogr)
+        np.testing.assert_allclose(kg4b.xi, kg.xi)
+        np.testing.assert_allclose(kg4b.xi_im, kg.xi_im)
+
     with assert_raises(TypeError):
         kg2 += config
     kg4 = treecorr.KGCorrelation(min_sep=min_sep/2, max_sep=max_sep, nbins=nbins)
@@ -446,9 +468,9 @@ def test_kg():
     except ImportError:
         pass
     else:
-        out_file_name1 = os.path.join('output','kg_out1.fits')
-        kg.write(out_file_name1)
-        data = fitsio.read(out_file_name1)
+        out_file_name = os.path.join('output','kg_out.fits')
+        kg.write(out_file_name)
+        data = fitsio.read(out_file_name)
         np.testing.assert_almost_equal(data['r_nom'], np.exp(kg.logr))
         np.testing.assert_almost_equal(data['meanr'], kg.meanr)
         np.testing.assert_almost_equal(data['meanlogr'], kg.meanlogr)
@@ -459,8 +481,7 @@ def test_kg():
         np.testing.assert_almost_equal(data['npairs'], kg.npairs)
 
         # Check the read function
-        kg2 = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin')
-        kg2.read(out_file_name1)
+        kg2 = treecorr.KGCorrelation.from_file(out_file_name)
         np.testing.assert_almost_equal(kg2.logr, kg.logr)
         np.testing.assert_almost_equal(kg2.meanr, kg.meanr)
         np.testing.assert_almost_equal(kg2.meanlogr, kg.meanlogr)