repo_sync_manifest: Rework repo_sync_manifest.
This script now supports syncing based on a variety of manifest
sources (branch, buildspecs (based on manifest-verions), or a local
manifest file). It also supports repo preloading, and git cache
optimization hints.
This switches the script from the repo_util library to the older
repository library.
It also enforces some cleanup of existing checkouts. Source will be
detached, and any existing local branches will be deleted. Files not
under verssion control will NOT be cleaned up, and other expensive
cleaning operations (deleting git locks) will not be performed.
It is NOT safe to assume that the checkout can be further updated with
a simple "repo sync", instead you should call this script again with
the same options.
Additional CLs are coming which will add support for cherry-picking
changes into the checkout during sync.
BUG=chromium:864816
TEST=run_tests
Change-Id: If8cd16c60f417e07d16d099d948b8ce53e4afdf6
Reviewed-on: https://chromium-review.googlesource.com/c/1316817
Tested-by: Don Garrett <dgarrett@chromium.org>
Reviewed-by: Alec Thilenius <athilenius@google.com>
diff --git a/scripts/repo_sync_manifest.py b/scripts/repo_sync_manifest.py
index f19e938..dfdd0aa 100644
--- a/scripts/repo_sync_manifest.py
+++ b/scripts/repo_sync_manifest.py
@@ -3,42 +3,188 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-"""Sync a repo checkout to the given manifest file."""
+"""Sync a git repository to a given manifest.
+
+This script is intended to define all of the ways that a source checkout can be
+defined for a Chrome OS builder.
+
+If a sync completes successfully, the checked out code will exactly match
+whatever manifest is defined, and no local git branches will remain. Extraneous
+files not under version management will be ignored.
+
+It is NOT safe to assume that the checkout can be further updated with
+a simple "repo sync", instead you should call this script again with
+the same options.
+"""
from __future__ import print_function
-import os
-
+from chromite.cbuildbot import manifest_version
+from chromite.cbuildbot import repository
from chromite.lib import commandline
-from chromite.lib import repo_util
+from chromite.lib import cros_logging as logging
+from chromite.lib import config_lib
+from chromite.lib import osutils
def GetParser():
"""Creates the argparse parser."""
parser = commandline.ArgumentParser(description=__doc__)
- parser.add_argument('manifest', type='path',
- help='The manifest file to sync to.')
+
parser.add_argument('--repo-root', type='path', default='.',
help='Path to the repo root to sync.')
- parser.add_argument('--copy-repo', type='path',
- help='Path to an existing repo root; the .repo dir '
- 'there will be copied to the target repo root, '
- 'which must not be within an existing root.')
+
+ manifest_group = parser.add_argument_group(
+ 'Manifest',
+ description='What manifest do we sync?')
+
+ manifest_ex = manifest_group.add_mutually_exclusive_group()
+ manifest_ex.add_argument(
+ '--branch',
+ default='master',
+ help='Sync to top of given branch.')
+ manifest_ex.add_argument(
+ '--buildspec',
+ help='Path to manifest, relative to manifest-versions root.')
+ manifest_ex.add_argument(
+ '--version',
+ help='Shorthand for an official release buildspec. e.g. 9799.0.0')
+ manifest_ex.add_argument(
+ '--manifest-file', type='path',
+ help='Sync to an existing local manifest file.')
+
+ manifest_group.add_argument(
+ '--external', action='store_true', default=False,
+ help='Sync to the external version of a manifest. Switch from '
+ 'manifest-versions-internal to manifest-versions for buildspecs. '
+ 'Not usable with --manifest.')
+
+ resources_group = parser.add_argument_group(
+ 'Resources',
+ description='External resources that might be needed.')
+
+ resources_group.add_argument(
+ '--manifest-versions-int', type='path',
+ help='Directory for internal manifest versions checkout. '
+ 'May be refreshed.')
+
+ resources_group.add_argument(
+ '--manifest-versions-ext', type='path',
+ help='Directory for internal manifest versions checkout. '
+ 'May be refreshed.')
+
+ optimization_group = parser.add_argument_group(
+ 'Optimization',
+ description='Hints provided to possibly speed up initial sync.')
+
+ optimization_group.add_argument(
+ '--copy-repo', type='path',
+ help='Path to an existing repo root. Used to preload the local '
+ "checkout if the local checkout doesn't exist.")
+ optimization_group.add_argument(
+ '--git-cache-dir', type='path',
+ help='Git cache directory to use.')
+
return parser
+def PrepareManifestVersions(options):
+ """Select manifest-versions checkout to use, and update it.
+
+ Looks at command line options to decide which manifest-versions checkout to
+ use, and updates (or creates) it as needed.
+
+ Args:
+ options: Parsed command line options.
+
+ Returns:
+ Full path to manifest-versions directory to use for this sync.
+
+ Raises:
+ AssertionError: If the needed manifest-versions path wasn't con the
+ command line.
+ """
+ site_params = config_lib.GetSiteParams()
+
+ if options.external:
+ assert options.manifest_versions_ext, '--manifest-versions-ext required.'
+ manifest_versions_url = site_params.MANIFEST_VERSIONS_GOB_URL
+ manifest_versions_path = options.manifest_versions_ext
+ else:
+ assert options.manifest_versions_int, '--manifest-versions-int required.'
+ manifest_versions_url = site_params.MANIFEST_VERSIONS_INT_GOB_URL
+ manifest_versions_path = options.manifest_versions_int
+
+ # Resolve buildspecs against a current manifest versions value.
+ manifest_version.RefreshManifestCheckout(
+ manifest_versions_path, manifest_versions_url)
+
+ return manifest_versions_path
+
+
+def ResolveLocalManifestPath(options):
+ """Based on command line options, decide what local manifest file to use.
+
+ Args:
+ options: Our parsed command line options.
+
+ Returns:
+ Path to local manifest file to use, or None for no file.
+ """
+ if options.manifest_file:
+ # If the user gives us an explicit local manifest file, use it.
+ return options.manifest_file
+
+ elif options.buildspec:
+ # Buildspec builds use a manifest file from manifest_versions. We do NOT
+ # use manifest_versions as the manifest git repo, because it's so large that
+ # sync time would be a major performance problem.
+ manifest_versions_path = PrepareManifestVersions(options)
+ return manifest_version.ResolveBuildspec(
+ manifest_versions_path, options.buildspec)
+
+ elif options.version:
+ # Versions are a short hand version of a buildspec.
+ manifest_versions_path = PrepareManifestVersions(options)
+ return manifest_version.ResolveBuildspecVersion(
+ manifest_versions_path, options.version)
+
+ elif options.branch:
+ # Branch checkouts use our normal manifest repos, not a local manifest file.
+ return None
+
+ else:
+ assert False, 'No sync options specified. Should not be possible.'
+
+
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
options.Freeze()
- # Assert manifest exists.
- os.stat(options.manifest)
+ local_manifest = ResolveLocalManifestPath(options)
+
+ if local_manifest:
+ logging.info('Using local_manifest: %s', local_manifest)
+
+ if options.external:
+ manifest_url = config_lib.GetSiteParams().MANIFEST_URL
+ else:
+ manifest_url = config_lib.GetSiteParams().MANIFEST_INT_URL
+
+ osutils.SafeMakedirs(options.repo_root)
+ repo = repository.RepoRepository(
+ manifest_repo_url=manifest_url,
+ directory=options.repo_root,
+ branch=options.branch,
+ git_cache_dir=options.git_cache_dir)
if options.copy_repo:
- copy_repo = repo_util.Repository(options.copy_repo)
- copy_repo.Copy(options.repo_root)
+ repo.PreLoad(options.copy_repo)
- repo = repo_util.Repository(options.repo_root)
+ if repository.IsARepoRoot(options.repo_root):
+ repo.BuildRootGitCleanup(prune_all=True)
- repo.Sync(manifest_path=options.manifest)
+ repo.Sync(local_manifest=local_manifest, detach=True)
+
+ # TODO: Cherry-pick in changes.