sync: superproject - support for switching hosts and switching branches.
+ superproject will be fetched into a directory with the name
“<remote name>-superproject.git” instead of the current
“superproject.git” folder.
+ Deleted _Clone method and added _Init method.
+ _Init method will do “git init --bare <remote>-superproject.git”.
It will create the folder and set up a bare repository in
<remote>-superproject.git folder.
+ _Fetch method, will pass <remote url>, <branch> arguments.
Moved the --filter argument from “git clone” to “git fetch”.
_Fetch method will execute the following command to fetch
superproject. Added --no-tags argument.
master: git fetch <remote url> --force --no-tags --filter blob:none
branch: git fetch <remote url> --force --no-tags --filter blob:none \
<branch>:<branch>
+ Performance improvements for aosp-master
++ repo init performance improved from 35 seconds to 17 seconds.
++ repo init --use-superproject is around 5 to 7 secsonds slower.
++ repo sync --use-superproject is around 3 to 4 minutes faster.
Tested the code with the following commands.
$ ./run_tests -v
Tested the sync code by using repo_dev alias and pointing to this CL.
$ time repo_dev init -u sso://android.git.corp.google.com/platform/manifest -b master --partial-clone --clone-filter=blob:limit=10M --repo-rev=main --use-superproject
...
real 0m20.648s
user 0m8.046s
sys 0m3.271s
+ Without superproject
$ time repo init -u sso://android.git.corp.google.com/platform/manifest -b master --partial-clone --clone-filter=blob:limit=10M --repo-rev=main
real 0m13.078s
user 0m9.783s
sys 0m2.528s
$ time repo_dev sync -c -j32 --use-superproject
...
real 15m7.072s
user 110m7.216s
sys 20m17.559s
+ Without superproject
$ time repo sync -c -j32
...
real 19m25.644s
user 91m56.331s
sys 20m59.170s
Bug: [google internal] b/180492484
Bug: [google internal] b/179470886
Bug: [google internal] b/180124069
Bug: https://crbug.com/gerrit/13709
Bug: https://crbug.com/gerrit/13707
Change-Id: Ib04bd7f1e25ceb75532643e58ad0129300ba3299
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/297702
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Raman Tenneti <rtenneti@google.com>
diff --git a/git_superproject.py b/git_superproject.py
index 471dadc..a09edc1 100644
--- a/git_superproject.py
+++ b/git_superproject.py
@@ -22,13 +22,13 @@
project_commit_ids = superproject.UpdateProjectsRevisionId(projects)
"""
+import hashlib
import os
import sys
from error import BUG_REPORT_URL
from git_command import GitCommand
from git_refs import R_HEADS
-import platform_utils
_SUPERPROJECT_GIT_NAME = 'superproject.git'
_SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
@@ -37,9 +37,9 @@
class Superproject(object):
"""Get commit ids from superproject.
- It does a 'git clone' of superproject and 'git ls-tree' to get list of commit ids
- for all projects. It contains project_commit_ids which is a dictionary with
- project/commit id entries.
+ Initializes a local copy of a superproject for the manifest. This allows
+ lookup of commit ids for all projects. It contains _project_commit_ids which
+ is a dictionary with project/commit id entries.
"""
def __init__(self, manifest, repodir, superproject_dir='exp-superproject'):
"""Initializes superproject.
@@ -58,8 +58,12 @@
self._superproject_path = os.path.join(self._repodir, superproject_dir)
self._manifest_path = os.path.join(self._superproject_path,
_SUPERPROJECT_MANIFEST_NAME)
- self._work_git = os.path.join(self._superproject_path,
- _SUPERPROJECT_GIT_NAME)
+ git_name = ''
+ if self._manifest.superproject:
+ remote_name = self._manifest.superproject['remote'].name
+ git_name = hashlib.md5(remote_name.encode('utf8')).hexdigest() + '-'
+ self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
+ self._work_git = os.path.join(self._superproject_path, self._work_git_name)
@property
def project_commit_ids(self):
@@ -77,20 +81,15 @@
branch = branch[len(R_HEADS):]
return branch
- def _Clone(self, url):
- """Do a 'git clone' for the given url.
-
- Args:
- url: superproject's url to be passed to git clone.
+ def _Init(self):
+ """Sets up a local Git repository to get a copy of a superproject.
Returns:
- True if git clone is successful, or False.
+ True if initialization is successful, or False.
"""
if not os.path.exists(self._superproject_path):
os.mkdir(self._superproject_path)
- cmd = ['clone', url, '--filter', 'blob:none', '--bare']
- if self._branch:
- cmd += ['--branch', self._branch]
+ cmd = ['init', '--bare', self._work_git_name]
p = GitCommand(None,
cmd,
cwd=self._superproject_path,
@@ -98,24 +97,27 @@
capture_stderr=True)
retval = p.Wait()
if retval:
- # `git clone` is documented to produce an exit status of `128` if
- # the requested url or branch are not present in the configuration.
- print('repo: error: git clone call failed with return code: %r, stderr: %r' %
+ print('repo: error: git init call failed with return code: %r, stderr: %r' %
(retval, p.stderr), file=sys.stderr)
return False
return True
- def _Fetch(self):
- """Do a 'git fetch' to to fetch the latest content.
+ def _Fetch(self, url):
+ """Fetches a local copy of a superproject for the manifest based on url.
+
+ Args:
+ url: superproject's url.
Returns:
- True if 'git fetch' is successful, or False.
+ True if fetch is successful, or False.
"""
if not os.path.exists(self._work_git):
print('git fetch missing drectory: %s' % self._work_git,
file=sys.stderr)
return False
- cmd = ['fetch', 'origin', '+refs/heads/*:refs/heads/*', '--prune']
+ cmd = ['fetch', url, '--force', '--no-tags', '--filter', 'blob:none']
+ if self._branch:
+ cmd += [self._branch + ':' + self._branch]
p = GitCommand(None,
cmd,
cwd=self._work_git,
@@ -129,7 +131,7 @@
return True
def _LsTree(self):
- """Returns the data from 'git ls-tree ...'.
+ """Gets the commit ids for all projects.
Works only in git repositories.
@@ -153,14 +155,12 @@
if retval == 0:
data = p.stdout
else:
- # `git clone` is documented to produce an exit status of `128` if
- # the requested url or branch are not present in the configuration.
print('repo: error: git ls-tree call failed with return code: %r, stderr: %r' % (
retval, p.stderr), file=sys.stderr)
return data
def Sync(self):
- """Sync superproject either by git clone/fetch.
+ """Gets a local copy of a superproject for the manifest.
Returns:
True if sync of superproject is successful, or False.
@@ -179,17 +179,10 @@
file=sys.stderr)
return False
- do_clone = True
- if os.path.exists(self._superproject_path):
- if not self._Fetch():
- # If fetch fails due to a corrupted git directory, then do a git clone.
- platform_utils.rmtree(self._superproject_path)
- else:
- do_clone = False
- if do_clone:
- if not self._Clone(url):
- print('error: git clone failed for url: %s' % url, file=sys.stderr)
- return False
+ if not self._Init():
+ return False
+ if not self._Fetch(url):
+ return False
return True
def _GetAllProjectsCommitIds(self):
@@ -203,7 +196,8 @@
data = self._LsTree()
if not data:
- print('error: git ls-tree failed for superproject', file=sys.stderr)
+ print('error: git ls-tree failed to return data for superproject',
+ file=sys.stderr)
return None
# Parse lines like the following to select lines starting with '160000' and