Merge pull request #6 from Tecnativa/master-fetch-only-required

Fetch only required remotes. Saves disk space and bandwidth by default.

README.rst

@@ -3,7 +3,7 @@
     :alt: License: AGPL-3
 .. image:: https://travis-ci.org/acsone/git-aggregator.svg?branch=master
     :target: https://travis-ci.org/acsone/git-aggregator
-.. image:: https://coveralls.io/repos/acsone/git-aggregator/badge.png?branch=master 
+.. image:: https://coveralls.io/repos/acsone/git-aggregator/badge.png?branch=master
     :target: https://coveralls.io/r/acsone/git-aggregator?branch=master
 .. image:: https://img.shields.io/badge/python-2.7%2C%203.3%2C%203.4%2C%203.5-blue.svg
     :alt: Python support: 2.7, 3.3, 3.4, 3.5
@@ -14,93 +14,124 @@ git-aggregator
 
 Manage the aggregation of git branches from different remotes to build a consolidated one.
 
-
-Command line Usage:
-===================
+Configuration file
+==================
 
 Create a ``repos.yaml`` file:
 
-  .. code-block:: yaml
-
-	./product_attribute:
-	    remotes:
-		oca: https://github.com/OCA/product-attribute.git
-		acsone: git+ssh://git@github.com/acsone/product-attribute.git
-	    merges:
-		- oca 8.0
-		- oca refs/pull/105/head
-		- oca refs/pull/106/head
-	    target: acsone aggregated_branch_name
-
-	./connector-interfaces:
-	    remotes:
-		oca:  https://github.com/OCA/connector-interfaces.git
-		acsone:  https://github.com/acsone/connector-interfaces.git
-	    merges:
-		- oca 6054de2c4e669f85cec380da90d746061967dc83
-		- acsone 8.0-connector_flow
-		- acsone 80_connector_flow_ir_cron_able-lmi
-		- acsone 8.0_connector_flow_improve_eval_config
-	    target: acsone aggregated_branch_name
-
-Aggregate you repositories at any time:
-
-  .. code-block:: bash
+.. code-block:: yaml
 
-    $ gitaggregate -c repos.yaml
+    ./product_attribute:
+        remotes:
+            oca: https://github.com/OCA/product-attribute.git
+            acsone: git+ssh://git@github.com/acsone/product-attribute.git
+        merges:
+            - oca 8.0
+            - oca refs/pull/105/head
+            - oca refs/pull/106/head
+        target: acsone aggregated_branch_name
 
-You can also aggregate and automatically push the result to the target:
+    ./connector-interfaces:
+        remotes:
+            oca:  https://github.com/OCA/connector-interfaces.git
+            acsone:  https://github.com/acsone/connector-interfaces.git
+        merges:
+            - oca 6054de2c4e669f85cec380da90d746061967dc83
+            - acsone 8.0-connector_flow
+            - acsone 80_connector_flow_ir_cron_able-lmi
+            - acsone 8.0_connector_flow_improve_eval_config
+        target: acsone aggregated_branch_name
+        fetch_all:
+            - oca
 
-  .. code-block:: bash
+Fetching only required branches
+-------------------------------
 
-    $ gitaggregate -c repos.yaml -p
+If any of your merges refer to a specific commit, you will probably need to
+fetch all remotes from the corresponding remote or `use any other strategy to
+get that fetch working <http://stackoverflow.com/a/30701724/1468388>`_, but we
+recommend to simply add this like in the example above:
 
-Only aggregate a specific repository using `fnmatch`_:
+.. code-block:: yaml
 
-  .. code-block:: bash
+    fetch_all:
+        - oca
+        - other-remote
 
-    $ gitaggregate -c repos.yaml -p -d connector-interfaces
+You can specify that you want to fetch all references from all remotes you have defined with:
 
-.. _fnmatch: https://docs.python.org/2/library/fnmatch.html
+.. code-block:: yaml
 
-It's also possible to specify a command or a list of shell commands to execute 
+    fetch_all: true
+
+Triggers
+--------
+
+It's also possible to specify a command or a list of shell commands to execute
 after the aggregation (and before the push). The commands are executed into
 the aggregated directory.
 
-  .. code-block:: yaml
+.. code-block:: yaml
 
-	./product_attribute:
-	    remotes:
-		oca: https://github.com/OCA/product-attribute.git
-		acsone: git+ssh://git@github.com/acsone/product-attribute.git
-	    merges:
-		- oca 8.0
-	    target: acsone aggregated_branch_name
+    ./product_attribute:
+        remotes:
+        oca: https://github.com/OCA/product-attribute.git
+        acsone: git+ssh://git@github.com/acsone/product-attribute.git
+        merges:
+            - oca 8.0
+        target: acsone aggregated_branch_name
             shell_command_after: echo 'my command'
 
-	./connector-interfaces:
-	    remotes:
-		oca:  https://github.com/OCA/connector-interfaces.git
-		acsone:  https://github.com/acsone/connector-interfaces.git
-	    merges:
-		- oca 9.0
-	    target: acsone aggregated_branch_name
+    ./connector-interfaces:
+        remotes:
+        oca:  https://github.com/OCA/connector-interfaces.git
+        acsone:  https://github.com/acsone/connector-interfaces.git
+        merges:
+            - oca 9.0
+        target: acsone aggregated_branch_name
             shell_command_after:
-		- echo 'a first command'
-		- echo 'a second command'
+                - echo 'a first command'
+                - echo 'a second command'
+
+Command line Usage
+==================
+
+Following the example ``repos.yaml`` file from above, aggregate your
+repositories at any time:
+
+.. code-block:: bash
+
+    $ gitaggregate -c repos.yaml
+
+You can also aggregate and automatically push the result to the target:
+
+.. code-block:: bash
+
+    $ gitaggregate -c repos.yaml -p
+
+Only aggregate a specific repository using `fnmatch`_:
+
+.. code-block:: bash
+
+    $ gitaggregate -c repos.yaml -p -d connector-interfaces
+
+.. _fnmatch: https://docs.python.org/2/library/fnmatch.html
 
 Credits
 =======
 
 Author
 ------
 
-  * Laurent Mignon (ACSONE)
-  
+* Laurent Mignon (ACSONE)
+
 Contributors
 ------------
 
-  * Cyril Gaudin (camptocamp)
+* Cyril Gaudin (camptocamp)
+* Jairo Llopis (Tecnativa_)
+
+.. _Tecnativa: https://www.tecnativa.com
 
 Maintainer
 ----------
@@ -109,4 +140,4 @@ Maintainer
    :alt: ACSONE SA/NV
    :target: http://www.acsone.eu
 
-This module is maintained by ACSONE SA/NV.
+This project is maintained by ACSONE SA/NV.

git_aggregator/config.py

@@ -72,6 +72,12 @@ def get_repos(config):
         else:
             raise ConfigException(
                 '%s: merges is not defined.' % directory)
+        # Only fetch required remotes by default
+        repo_dict["fetch_all"] = repo_data.get("fetch_all", False)
+        if isinstance(repo_dict["fetch_all"], string_types):
+            repo_dict["fetch_all"] = frozenset((repo_dict["fetch_all"],))
+        elif isinstance(repo_dict["fetch_all"], list):
+            repo_dict["fetch_all"] = frozenset(repo_dict["fetch_all"])
         if 'target' not in repo_data:
             raise ConfigException('%s: No target defined.' % directory)
         parts = (repo_data.get('target') or "") .split(' ')

git_aggregator/repo.py

@@ -34,7 +34,7 @@ class Repo(object):
     _git_version = None
 
     def __init__(self, cwd, remotes, merges, target,
-                 shell_command_after=None):
+                 shell_command_after=None, fetch_all=False):
         """Initialize a git repository aggregator
 
         :param cwd: path to the directory where to initialize the repository
@@ -43,11 +43,19 @@ def __init__(self, cwd, remotes, merges, target,
         :param: merges list of merge to apply to build the aggregated
         repository. A merge is a dict {'remote': '', 'ref': ''}
         :param target:
-        :patam shell_command_after: an optional list of shell command to
+        :param fetch_all:
+            Can be an iterable (recommended: ``frozenset``) that yields names
+            of remotes where all refs should be fetched, or ``True`` to do it
+            for every configured remote.
+        :param shell_command_after: an optional list of shell command to
         execute after the aggregation
         """
         self.cwd = cwd
         self.remotes = remotes
+        if fetch_all is True:
+            self.fetch_all = frozenset(r["name"] for r in remotes)
+        else:
+            self.fetch_all = fetch_all or frozenset()
         self.merges = merges
         self.target = target
         self.shell_command_after = shell_command_after or []
@@ -160,7 +168,7 @@ def aggregate(self):
             self._switch_to_branch(self.target['branch'])
             for r in self.remotes:
                 self._set_remote(**r)
-            self.fetch_all()
+            self.fetch()
             merges = self.merges
             if not is_new:
                 # reset to the first merge
@@ -176,9 +184,14 @@ def init_repository(self, target_dir):
         logger.info('Init empty git repository in %s', target_dir)
         self.log_call(['git', 'init', target_dir])
 
-    def fetch_all(self):
-        logger.info('Fetching all remotes')
-        self.log_call(['git', 'fetch',  '--all'], cwd=self.cwd)
+    def fetch(self):
+        basecmd = ("git", "fetch")
+        logger.info("Fetching required remotes")
+        for merge in self.merges:
+            cmd = basecmd + (merge["remote"],)
+            if merge["remote"] not in self.fetch_all:
+                cmd += (merge["ref"],)
+            self.log_call(cmd, cwd=self.cwd)
 
     def push(self):
         remote = self.target['remote']

git_aggregator/utils.py

@@ -25,4 +25,5 @@ def __exit__(self, *exc_args):
         os.chdir(self.wd)
         self.active = False
 
+
 working_directory_keeper = WorkingDirectoryKeeper()

tests/test_config.py

@@ -5,6 +5,7 @@
 import tempfile
 import unittest
 import kaptan
+from textwrap import dedent
 
 from git_aggregator import config
 from git_aggregator.exception import ConfigException
@@ -39,6 +40,7 @@ def test_load(self):
         self.assertDictEqual(
             repos[0],
             {'cwd': '/product_attribute',
+             'fetch_all': False,
              'merges': [{'ref': '8.0', 'remote': 'oca'},
                         {'ref': 'refs/pull/105/head', 'remote': 'oca'},
                         {'ref': 'refs/pull/106/head', 'remote': 'oca'}],
@@ -249,3 +251,46 @@ def test_import_config(self):
         finally:
             if os.path.exists(config_path):
                 os.remove(config_path)
+
+    def test_fetch_all_string(self):
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                fetch_all: oca
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertSetEqual(repos[0]["fetch_all"], {"oca"})
+
+    def test_fetch_all_list(self):
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                fetch_all:
+                    - oca
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertSetEqual(repos[0]["fetch_all"], {"oca"})
+
+    def test_fetch_all_true(self):
+        config_yaml = """
+            ./test:
+                remotes:
+                    oca: https://github.com/test/test.git
+                merges:
+                    - oca 8.0
+                target: oca aggregated_branch_name
+                fetch_all: yes
+            """
+        config_yaml = dedent(config_yaml)
+        repos = config.get_repos(self._parse_config(config_yaml))
+        self.assertIs(repos[0]["fetch_all"], True)

tests/test_repo.py

@@ -136,7 +136,7 @@ def test_simple_merge(self):
             'remote': 'r1',
             'branch': 'agg'
         }
-        repo = Repo(self.cwd, remotes, merges, target)
+        repo = Repo(self.cwd, remotes, merges, target, fetch_all=True)
         repo.aggregate()
         last_rev = git_get_last_rev(self.cwd)
         self.assertEqual(last_rev, self.commit_3_sha)
@@ -171,7 +171,7 @@ def test_update_aggregate(self):
             'remote': 'r1',
             'branch': 'agg'
         }
-        repo = Repo(self.cwd, remotes, merges, target)
+        repo = Repo(self.cwd, remotes, merges, target, fetch_all=True)
         repo.aggregate()
         self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked')))
         self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked2')))
@@ -205,7 +205,7 @@ def test_update_aggregate_2(self):
             'remote': 'r1',
             'branch': 'agg'
         }
-        repo = Repo(self.cwd, remotes, merges, target)
+        repo = Repo(self.cwd, remotes, merges, target, fetch_all=True)
         repo.aggregate()
         self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked')))
         self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked2')))