Added test for sparse checkout and updated documentation

README.md

@@ -202,6 +202,21 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines begining with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/checkout.py

@@ -227,6 +227,21 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines beginning with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/repository_git.py

@@ -369,7 +369,8 @@ def _sparse_checkout(self, repo_dir, verbosity):
         """Use git read-tree to thin the working tree."""
         cwd = os.getcwd()
 
-        cmd = ['cp', self._sparse, os.path.join(repo_dir, '.git/info/sparse-checkout')]
+        cmd = ['cp', self._sparse, os.path.join(repo_dir,
+                                                '.git/info/sparse-checkout')]
         if verbosity >= VERBOSITY_VERBOSE:
             printlog('    {0}'.format(' '.join(cmd)))
         execute_subprocess(cmd)

test/repos/simple-ext.git/refs/heads/master

@@ -1 +1 @@
-9b75494003deca69527bb64bcaa352e801611dd2
+607ec299c17dd285c029edc41a0109e49d441380

test/repos/simple-ext.git/refs/tags/tag2

@@ -0,0 +1 @@
+b7692b6d391899680da7b9b6fd8af4c413f06fe7

test/test_sys_checkout.py

@@ -47,7 +47,7 @@
 from manic.repository_git import GitRepository
 from manic.utils import printlog, execute_subprocess
 from manic.global_constants import LOCAL_PATH_INDICATOR, VERBOSITY_DEFAULT
-from manic.global_constants import LOG_FILE_NAME
+from manic.global_constants import LOG_FILE_NAME, EMPTY_STR
 from manic import checkout
 
 # ConfigParser was renamed in python2 to configparser. In python2,
@@ -183,6 +183,25 @@ def container_simple_svn(self, dest_dir):
 
         self.write_config(dest_dir)
 
+    def container_sparse(self, dest_dir):
+        """Create a container with a full external and a sparse external
+
+        """
+        # Create a file for a sparse pattern match
+        sparse_filename = 'sparse_checkout'
+        with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile:
+            sfile.write('readme.txt')
+
+        self.create_config()
+        self.create_section(SIMPLE_REPO_NAME, 'simp_tag',
+                            tag='tag2')
+
+        sparse_relpath = '../../{}'.format(sparse_filename)
+        self.create_section(SIMPLE_REPO_NAME, 'simp_sparse',
+                            tag='tag2', sparse=sparse_relpath)
+
+        self.write_config(dest_dir)
+
     def mixed_simple_base(self, dest_dir):
         """Create a mixed-use base externals file with only simple externals.
 
@@ -239,7 +258,8 @@ def create_metadata(self):
 
     def create_section(self, repo_type, name, tag='', branch='',
                        ref_hash='', required=True, path=EXTERNALS_NAME,
-                       externals='', repo_path=None, from_submodule=False):
+                       externals='', repo_path=None, from_submodule=False,
+                       sparse=''):
         # pylint: disable=too-many-branches
         """Create a config section with autofilling some items and handling
         optional items.
@@ -287,6 +307,9 @@ def create_section(self, repo_type, name, tag='', branch='',
         if externals:
             self._config.set(name, ExternalsDescription.EXTERNALS, externals)
 
+        if sparse:
+            self._config.set(name, ExternalsDescription.SPARSE, sparse)
+
         if from_submodule:
             self._config.set(name, ExternalsDescription.SUBMODULE, "True")
 
@@ -710,6 +733,14 @@ def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/mixed_req'.format(directory)
         self._check_generic_modified_ok_required(tree, name)
 
+    def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_empty_default_required(tree, name)
+
+    def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_ok_clean_required(tree, name)
+
     # ----------------------------------------------------------------
     #
     # Check results for groups of externals under specific conditions
@@ -870,6 +901,23 @@ def _check_mixed_cont_simple_required_post_checkout(self, overall, tree):
         self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME)
         self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH)
 
+    def _check_container_sparse_pre_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_empty(tree)
+        self._check_simple_sparse_empty(tree)
+
+    def _check_container_sparse_post_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_ok(tree)
+        self._check_simple_sparse_ok(tree)
+
+    def _check_file_exists(self, repo_dir, pathname):
+        "Check that <pathname> exists in <repo_dir>"
+        self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname)))
+
+    def _check_file_absent(self, repo_dir, pathname):
+        "Check that <pathname> does not exist in <repo_dir>"
+        self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname)))
 
 class TestSysCheckout(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals
@@ -1234,6 +1282,14 @@ def test_container_full(self):
                                                 self.status_args)
         self._check_container_full_post_checkout(overall, tree)
 
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
         # update the mixed-use repo to point to different branch
         self._generator.update_branch(under_test_dir, 'mixed_req',
                                       'new-feature', MIXED_REPO_NAME)
@@ -1314,6 +1370,40 @@ def test_mixed_simple(self):
                                                 self.status_args)
         self._check_mixed_cont_simple_required_post_checkout(overall, tree)
 
+    def test_container_sparse(self):
+        """Verify that 'full' container with simple subrepo
+        can run a sparse checkout and generate the correct initial status.
+
+        """
+        # create the test repository
+        under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME)
+
+        # create the top level externals file
+        self._generator.container_sparse(under_test_dir)
+
+        # inital checkout
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.checkout_args)
+        self._check_container_sparse_pre_checkout(overall, tree)
+
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.status_args)
+        self._check_container_sparse_post_checkout(overall, tree)
+
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_exists(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+        subrepo_path = os.path.join('externals', 'simp_sparse')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
 
 class TestSysCheckoutSVN(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals accessing svn repositories