Rework loman to use manifest groups.
This is done to fold minilayout.xml into the full manifest, rather
than maintaining it separately (or having to use manifest inheritance).
The new flow works via using manifest-groups to turn on the specific
project that is being worked on; via this, it's easy to programmatically
turn off that project, and we no longer have to endure the design flaw
where the local_manifest addition enabling that project becomes stale,
using the old revision rather than the current setting.
Finally, note that this auto-upgrades any minilayout.xml user if
a `loman add ...` invocation occurs, or `loman --upgrade-minilayout`.
The upgrade pathway strips out the duplicate projects from local_manifest.xml
(required, else it would be a project level conflict against full.xml),
turns on the groups that the user already had cros_workon'd. If the
local_manifest is rendered empty via this, it's wiped; else it's written
w/ just the remainder (this is required else we could wipe a local_manifest
addition a user has for local development of a project not yet in our
tree). Once all of that is done, it swaps the sym to full.xml and continues
with the users request.
This should be a pretty much seemless upgrade via this. In the process,
drop unused options, drop the unittest (this script isn't exactly supported,
thus frankly I'm not interested in spending time rewriting the tests in
entirety- the way the machinery works now, loman is just a dumb wrapper
around git config manipulation).
BUG=chromium-os:32247,chromium-os:31867,chromium-os:9914
TEST=manual upgrading of a minilayout checkout.
TEST=loman add --workon <target> followed by repo sync to verify
the new source is brought in.
Change-Id: I21549cc73b776bbf0b22b2d3d9a38dd175fd797b
Reviewed-on: https://gerrit.chromium.org/gerrit/29250
Commit-Ready: Brian Harring <ferringb@chromium.org>
Reviewed-by: Brian Harring <ferringb@chromium.org>
Tested-by: Brian Harring <ferringb@chromium.org>
diff --git a/scripts/loman.py b/scripts/loman.py
index 3764a40..024ea4f 100644
--- a/scripts/loman.py
+++ b/scripts/loman.py
@@ -1,150 +1,231 @@
-#!/usr/bin/python
-
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This module allows adding and deleting of projects to the local manifest."""
-import sys
+import logging
+import platform
import optparse
import os
+import sys
import xml.etree.ElementTree as ElementTree
-
-from cros_build_lib import Die, FindRepoDir
+from chromite.lib import cros_build_lib
-def _ReadManifest(manifest, err_not_found=False):
- if os.path.isfile(manifest):
- ptree = LocalManifest(open(manifest).read())
- elif err_not_found:
- Die('Manifest file, %s, not found' % manifest)
- else:
- ptree = LocalManifest()
- ptree.Parse()
- return ptree
-
-
-class LocalManifest:
+class Manifest(object):
"""Class which provides an abstraction for manipulating the local manifest."""
+ @classmethod
+ def FromPath(cls, path, empty_if_missing=False):
+ if os.path.isfile(path):
+ with open(path) as f:
+ return cls(f.read())
+ elif empty_if_missing:
+ cros_build_lib.Die('Manifest file, %r, not found' % path)
+ return cls()
+
def __init__(self, text=None):
self._text = text or '<manifest>\n</manifest>'
-
- def Parse(self):
- """Parse the manifest."""
- self._root = ElementTree.fromstring(self._text)
-
- def AddProjectElement(self, element, workon='False', remote=None):
- """Add a new project element to the manifest tree.
-
- Returns:
- True on success.
- """
- name = element.attrib['name']
- path = element.attrib['path']
- for project in self._root.findall('project'):
- if project.attrib['path'] == path or project.attrib['name'] == name:
- if project.attrib['path'] == path and project.attrib['name'] == name:
- return True
- else:
- return False
- element.attrib['workon'] = workon
- if remote is not None:
- element.attrib['remote'] = remote
- element.tail = '\n'
- self._root.append(element)
- return True
-
- def AddProject(self, name, path, workon='False', remote=None):
- """Add a workon project if it is not already in the manifest.
-
- Returns:
- True on success.
- """
- element = ElementTree.Element('project', name=name, path=path)
- return self.AddProjectElement(element, workon=workon, remote=remote)
-
- def AddWorkonProjectElement(self, element):
- return self.AddProjectElement(element, workon='True')
-
- def AddWorkonProject(self, name, path):
- return self.AddProject(name, path, workon='True')
-
- def AddNonWorkonProjectElement(self, element, remote):
- return self.AddProjectElement(element, workon='False', remote=remote)
+ self.nodes = ElementTree.fromstring(self._text)
def AddNonWorkonProject(self, name, path, remote):
- return self.AddProject(name, path, workon='False', remote=remote)
+ """Add a new nonworkon project element to the manifest tree."""
+ element = ElementTree.Element('project', name=name, path=path,
+ remote=remote)
+ element.attrib['workon'] = 'False'
+ self.nodes.append(element)
+ return element
- def GetProject(self, name):
+ def GetProject(self, name, path=None):
"""Accessor method for getting a project node from the manifest tree.
Returns:
project element node from ElementTree, otherwise, None
"""
-
- for project in self._root.findall('project'):
- if project.attrib['name'] == name:
+ if path is None:
+ # Use a unique value that can't ever match.
+ path = object()
+ for project in self.nodes.findall('project'):
+ if project.attrib['name'] == name or project.attrib['path'] == path:
return project
return None
def ToString(self):
- return ElementTree.tostring(self._root, encoding='UTF-8')
+ # Reset the tail for each node, then just do a hacky replace.
+ project = None
+ for project in self.nodes.findall('project'):
+ project.tail = '\n '
+ if project is not None:
+ # Tweak the last project to not have the trailing space.
+ project.tail = '\n'
+ # Fix manifest tag text and tail.
+ self.nodes.text = '\n '
+ self.nodes.tail = '\n'
+ return ElementTree.tostring(self.nodes)
+
+ def GetProjects(self):
+ return list(self.nodes.findall('project'))
+
+
+def _AddProjectsToManifestGroups(options, *projects):
+ """Enable the given manifest groups for the configured repository."""
+
+ groups_to_enable = ['name:%s' % x for x in projects]
+
+ git_config = options.git_config
+
+ enabled_groups = cros_build_lib.RunCommandCaptureOutput(
+ ['git', 'config', '-f', git_config, '--get', 'manifest.groups'],
+ error_code_ok=True, print_cmd=False).output.split(',')
+
+ # Note that ordering actually matters, thus why the following code
+ # is written this way.
+ # Per repo behaviour, enforce an appropriate platform group if
+ # we're converting from a default manifest group to a limited one.
+ # Finally, note we reprocess the existing groups; this is to allow
+ # us to cleanup any user screwups, or our own screwups.
+ requested_groups = (
+ ['minilayout', 'platform-%s' % (platform.system().lower(),)] +
+ enabled_groups + list(groups_to_enable))
+
+ processed_groups = set()
+ finalized_groups = []
+
+ for group in requested_groups:
+ if group not in processed_groups:
+ finalized_groups.append(group)
+ processed_groups.add(group)
+
+ cros_build_lib.RunCommandCaptureOutput(
+ ['git', 'config', '-f', git_config, 'manifest.groups',
+ ','.join(finalized_groups)], print_cmd=False)
+
+
+def _UpgradeMinilayout(options):
+ """Convert a repo checkout away from minilayout.xml to default.xml."""
+
+ full_tree = Manifest.FromPath(options.default_manifest_path)
+ local_manifest_exists = os.path.exists(options.local_manifest_path)
+
+ new_groups = []
+ if local_manifest_exists:
+ local_tree = Manifest.FromPath(options.local_manifest_path)
+ # Identify which projects need to be transferred across.
+ projects = local_tree.GetProjects()
+ new_groups = [x.attrib['name'] for x in projects]
+ allowed = set(x.attrib['name'] for x in full_tree.GetProjects())
+ transferred = [x for x in projects if x.attrib['name'] in allowed]
+ for project in transferred:
+ # Mangle local_manifest object, removing those projects;
+ # note we'll still be adding those projects to the default groups,
+ # including those that didn't intersect the main manifest.
+ local_tree.nodes.remove(project)
+
+ _AddProjectsToManifestGroups(options, *new_groups)
+
+ if local_manifest_exists:
+ # Rewrite the local_manifest now; if there is no settings left in
+ # the local_manifest, wipe it.
+ if local_tree.nodes.getchildren():
+ with open(options.local_manifest_path, 'w') as f:
+ f.write(local_tree.ToString())
+ else:
+ os.unlink(options.local_manifest_path)
+
+ # Finally, move the symlink.
+ os.unlink(options.manifest_sym_path)
+ os.symlink('manifests/default.xml', options.manifest_sym_path)
+ logging.info("Converted the checkout to manifest groups based minilayout.")
def main(argv):
- repo_dir = FindRepoDir()
- if not repo_dir:
- Die("Unable to find repo dir.")
-
- usage = 'usage: %prog add [options] <name> <path>'
- parser = optparse.OptionParser(usage=usage)
+ parser = optparse.OptionParser(usage='usage: %prog add [options] <name> '
+ '<--workon | <path> --remote <remote> >')
parser.add_option('-w', '--workon', action='store_true', dest='workon',
default=False, help='Is this a workon package?')
- parser.add_option('-f', '--file', dest='local_manifest',
- default='%s/local_manifest.xml' % repo_dir,
- help='Non-default manifest file to read.')
- parser.add_option('-m', '--main', dest='main_manifest',
- default='%s/manifest.xml' % repo_dir,
- help='Main manifest file to read.')
- parser.add_option('-d', '--default', dest='full_manifest',
- default='%s/manifests/full.xml' % repo_dir,
- help='Default manifest file to read.')
parser.add_option('-r', '--remote', dest='remote',
default=None)
- (options, args) = parser.parse_args(argv[2:])
- if len(args) < 1:
- parser.error('Not enough arguments')
- if argv[1] not in ['add']:
- parser.error('Unsupported command: %s.' % argv[1])
- if not options.workon and options.remote is None:
- parser.error('Adding non-workon projects requires a remote.')
- name = args[0]
+ parser.add_option('--upgrade-minilayout', default=False, action='store_true',
+ help="Upgrade a minilayout checkout into a full.xml "
+ "checkout utilizing manifest groups.")
+ (options, args) = parser.parse_args(argv)
- local_tree = _ReadManifest(options.local_manifest)
- main_tree = _ReadManifest(options.main_manifest)
- full_tree = _ReadManifest(options.full_manifest)
+ repo_dir = cros_build_lib.FindRepoDir()
+ if not repo_dir:
+ parser.error("This script must be invoked from within a repository "
+ "checkout.")
- # Only add this project to local_manifest.xml if not in manifest.xml
- if options.workon:
- if main_tree.GetProject(name) is None:
- project_element = full_tree.GetProject(name)
- if project_element is None:
- Die('No project named %s, in the default manifest.' % name)
- success = local_tree.AddWorkonProjectElement(project_element)
- if not success:
- Die('Name "%s" already exists with a different path.' % name)
+ options.git_config = os.path.join(repo_dir, 'manifests.git', 'config')
+ options.repo_dir = repo_dir
+ options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml')
+ # This constant is used only when we're doing an upgrade away from
+ # minilayout.xml to default.xml.
+ options.default_manifest_path = os.path.join(repo_dir, 'manifests',
+ 'default.xml')
+ options.manifest_sym_path = os.path.join(repo_dir, 'manifest.xml')
+
+ active_manifest = os.path.basename(os.readlink(options.manifest_sym_path))
+ upgrade_required = active_manifest == 'minilayout.xml'
+
+ if options.upgrade_minilayout:
+ if args:
+ parser.error("--upgrade-minilayout takes no arguments.")
+ if not upgrade_required:
+ print "This repository checkout isn't using minilayout.xml; nothing to do"
+ else:
+ _UpgradeMinilayout(options)
+ return 0
+ elif upgrade_required:
+ logging.warn(
+ "Your repository checkout is using the old minilayout.xml workflow; "
+ "auto-upgrading it.")
+ cros_build_lib.RunCommand(
+ [sys.argv[0], '--upgrade-minilayout'], cwd=os.getcwd(),
+ print_cmd=False)
+
+ if not args:
+ parser.error("No command specified.")
+ elif args[0] != 'add':
+ parser.error("Only supported subcommand is add right now.")
+ elif options.workon:
+ if len(args) != 2:
+ parser.error(
+ "Argument count is wrong for --workon; must be add <project>")
+ name, path = args[1], None
else:
- success = local_tree.AddNonWorkonProject(name, args[1], options.remote)
- if not success:
- Die('Name "%s" already exists with a different path.' % name)
+ if options.remote is None:
+ parser.error('Adding non-workon projects requires a remote.')
+ elif len(args) != 3:
+ parser.error(
+ "Argument count is wrong for non-workon mode; "
+ "must be add <project> <path> --remote <remote-arg>")
+ name, path = args[1:]
- try:
- print >> open(options.local_manifest, 'w'), local_tree.ToString()
- except Exception, e:
- Die('Error writing to manifest: %s' % e)
+ main_manifest = Manifest.FromPath(options.manifest_sym_path,
+ empty_if_missing=False)
+ local_manifest = Manifest.FromPath(options.local_manifest_path)
+ main_element = main_manifest.GetProject(name, path=path)
-if __name__ == '__main__':
- main(sys.argv)
+ if options.workon:
+ if main_element is None:
+ parser.error('No project named %r in the default manifest.' % name)
+ _AddProjectsToManifestGroups(options, main_element.attrib['name'])
+
+ elif main_element:
+ # Conflict will occur; complain.
+ parser.error("Requested project name=%r path=%r will conflict with "
+ "your current manifest %s" % (name, path, active_manifest))
+
+ elif local_manifest.GetProject(name, path=path) is not None:
+ parser.error("Requested project name=%r path=%r conflicts with "
+ "your local_manifest.xml" % (name, path))
+
+ else:
+ element = local_manifest.AddNonWorkonProject(name, path, options.remote)
+ _AddProjectsToManifestGroups(options, element.attrib['name'])
+
+ with open(options.local_manifest_path, 'w') as f:
+ f.write(local_manifest.ToString())
+ return 0