gclient: Don't allow None URLs (except in .gclient files)

This reverts commit crrev.com/4e9b50ab86b9b9f8ebf0b9ba6bd4954217ebeff9
and thus relands the following commits:

  ebdd0db493b20f0abeab8960e6ea0ceb7c6b379a: "gclient: Remove URLs from hierarchy."
  54a5c2ba8ac2f9b8f4a32fe79913f13545e4aab9: "gclient: Refactor PrintRevInfo"
  083eb25f9acbe034db94a1bd5c1659125b6ebf98: "gclient: Don't allow URL to be None."

When a None URL is specified in a .gclient file, and a DEPS file is
given, the DEPS file is treated as a .gclient file and its dependencies
are added.

Bug: 839925

Change-Id: I1068b66487874bfa0a788bf9da5273714b6ad39e
Reviewed-on: https://chromium-review.googlesource.com/1083340
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>
Reviewed-by: Michael Moss <mmoss@chromium.org>
diff --git a/gclient.py b/gclient.py
index 4d963c6..7ecc727 100755
--- a/gclient.py
+++ b/gclient.py
@@ -241,7 +241,7 @@
   """Immutable configuration settings."""
   def __init__(
       self, parent, url, managed, custom_deps, custom_vars,
-      custom_hooks, deps_file, should_process, relative, condition):
+      custom_hooks, deps_file, relative, condition):
     # These are not mutable:
     self._parent = parent
     self._deps_file = deps_file
@@ -249,13 +249,9 @@
     # The condition as string (or None). Useful to keep e.g. for flatten.
     self._condition = condition
     # 'managed' determines whether or not this dependency is synced/updated by
-    # gclient after gclient checks it out initially.  The difference between
-    # 'managed' and 'should_process' is that the user specifies 'managed' via
-    # the --unmanaged command-line flag or a .gclient config, where
-    # 'should_process' is dynamically set by gclient if it goes over its
-    # recursion limit and controls gclient's behavior so it does not misbehave.
+    # gclient after gclient checks it out initially. The user specifies
+    # 'managed' via the --unmanaged command-line flag or a .gclient config.
     self._managed = managed
-    self._should_process = should_process
     # If this is a recursed-upon sub-dependency, and the parent has
     # use_relative_paths set, then this dependency should check out its own
     # dependencies relative to that parent's path for this, rather than
@@ -270,15 +266,10 @@
     self._custom_deps = custom_deps or {}
     self._custom_hooks = custom_hooks or []
 
-    # Post process the url to remove trailing slashes.
-    if isinstance(self.url, basestring):
+    if self.url is not None:
       # urls are sometime incorrectly written as proto://host/path/@rev. Replace
       # it to proto://host/path@rev.
       self.set_url(self.url.replace('/@', '@'))
-    elif not isinstance(self.url, (None.__class__)):
-      raise gclient_utils.Error(
-          ('dependency url must be either string or None, '
-           'instead of %s') % self.url.__class__.__name__)
 
     # Make any deps_file path platform-appropriate.
     if self._deps_file:
@@ -306,11 +297,6 @@
     return self.parent.root
 
   @property
-  def should_process(self):
-    """True if this dependency should be processed, i.e. checked out."""
-    return self._should_process
-
-  @property
   def custom_vars(self):
     return self._custom_vars.copy()
 
@@ -357,12 +343,20 @@
   """Object that represents a dependency checkout."""
 
   def __init__(self, parent, name, url, managed, custom_deps,
-               custom_vars, custom_hooks, deps_file, should_process,
-               should_recurse, relative, condition, print_outbuf=False):
+               custom_vars, custom_hooks, deps_file, should_recurse, relative,
+               condition, print_outbuf=False):
     gclient_utils.WorkItem.__init__(self, name)
     DependencySettings.__init__(
-        self, parent, url, managed, custom_deps, custom_vars,
-        custom_hooks, deps_file, should_process, relative, condition)
+        self,
+        parent=parent,
+        url=url,
+        managed=managed,
+        custom_deps=custom_deps,
+        custom_vars=custom_vars,
+        custom_hooks=custom_hooks,
+        deps_file=deps_file,
+        relative=relative,
+        condition=condition)
 
     # This is in both .gclient and DEPS files:
     self._deps_hooks = []
@@ -411,9 +405,8 @@
     # Whether we should process this dependency's DEPS file.
     self._should_recurse = should_recurse
 
-    self._OverrideUrl()
-    # This is inherited from WorkItem.  We want the URL to be a resource.
-    if self.url and isinstance(self.url, basestring):
+    if self.url:
+      # This is inherited from WorkItem.  We want the URL to be a resource.
       # The url is usually given to gclient either as https://blah@123
       # or just https://blah.  The @123 portion is irrelevant.
       self.resources.append(self.url.split('@')[0])
@@ -422,43 +415,8 @@
     # dependency
     self.print_outbuf = print_outbuf
 
-    if not self.name and self.parent:
-      raise gclient_utils.Error('Dependency without name')
-
-  def _OverrideUrl(self):
-    """Resolves the parsed url from the parent hierarchy."""
-    parsed_url = self.get_custom_deps(self._name, self.url)
-    if parsed_url != self.url:
-      logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self._name,
-                   self.url, parsed_url)
-      self.set_url(parsed_url)
-
-    elif isinstance(self.url, basestring):
-      parsed_url = urlparse.urlparse(self.url)
-      if (not parsed_url[0] and
-          not re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2])):
-        path = parsed_url[2]
-        if not path.startswith('/'):
-          raise gclient_utils.Error(
-              'relative DEPS entry \'%s\' must begin with a slash' % self.url)
-        # A relative url. Get the parent url, strip from the last '/'
-        # (equivalent to unix basename), and append the relative url.
-        parent_url = self.parent.url
-        parsed_url = parent_url[:parent_url.rfind('/')] + self.url
-        logging.info('Dependency(%s)._OverrideUrl(%s) -> %s', self.name,
-                     self.url, parsed_url)
-        self.set_url(parsed_url)
-
-    elif self.url is None:
-      logging.info('Dependency(%s)._OverrideUrl(None) -> None', self._name)
-
-    else:
-      raise gclient_utils.Error('Unknown url type')
-
   def PinToActualRevision(self):
     """Updates self.url to the revision checked out on disk."""
-    if self.url is None:
-      return
     url = None
     scm = self.CreateSCM()
     if os.path.isdir(scm.checkout_path):
@@ -502,7 +460,7 @@
 
     if self.name:
       requirements |= set(
-          obj.name for obj in self.root.subtree(False)
+          obj.name for obj in self.root.subtree()
           if (obj is not self
               and obj.name and
               self.name.startswith(posixpath.join(obj.name, ''))))
@@ -525,15 +483,11 @@
       raise gclient_utils.Error(
           'The same name "%s" appears multiple times in the deps section' %
               self.name)
-    if not self.should_process:
-      # Return early, no need to set requirements.
-      return True
 
     # This require a full tree traversal with locks.
-    siblings = [d for d in self.root.subtree(False) if d.name == self.name]
+    siblings = [d for d in self.root.subtree() if d.name == self.name]
     for sibling in siblings:
-      # Allow to have only one to be None or ''.
-      if self.url != sibling.url and bool(self.url) == bool(sibling.url):
+      if self.url != sibling.url:
         raise gclient_utils.Error(
             ('Dependency %s specified more than once:\n'
             '  %s [%s]\n'
@@ -583,28 +537,47 @@
 
     return deps
 
+  def FormatUrl(self, name, url):
+    custom_deps_url = self.get_custom_deps(name, url)
+    if url != custom_deps_url:
+      return custom_deps_url
+    if url is None:
+      return None
+    if not isinstance(url, basestring):
+      raise gclient_utils.Error(
+          ('dependency url must be either string or None, '
+           'instead of %s') % self.url.__class__.__name__)
+    # For relative URLs, strip the parent url (self.url) from the last '/'
+    # and append the relative url.
+    if url[0] == '/':
+      if self.url is None:
+        raise gclient_utils.Error(
+            'Trying to set a relative url for %s, but parent has no url.' % name
+        )
+      url = self.url[:self.url.rfind('/')] + url
+    return url
+
   def _deps_to_objects(self, deps, use_relative_paths):
     """Convert a deps dict to a dict of Dependency objects."""
     deps_to_add = []
     for name, dep_value in deps.iteritems():
-      should_process = self.should_process
       if dep_value is None:
         continue
 
       condition = dep_value.get('condition')
-      dep_type = dep_value.get('dep_type')
+      dep_type = dep_value.get('dep_type', 'git')
 
-      if condition and not self._get_option('process_all_deps', False):
-        should_process = should_process and gclient_eval.EvaluateCondition(
+      should_process = True
+      if condition and not self.get_option('process_all_deps', False):
+        should_process = gclient_eval.EvaluateCondition(
             condition, self.get_vars())
 
+      if not should_process:
+        continue
+
       if dep_type == 'cipd':
         cipd_root = self.GetCipdRoot()
         for package in dep_value.get('packages', []):
-          if 'version' in package:
-            # Matches version to vars value.
-            version = package['version']
-            package['version'] = version
           deps_to_add.append(
               CipdDependency(
                   parent=self,
@@ -612,25 +585,24 @@
                   dep_value=package,
                   cipd_root=cipd_root,
                   custom_vars=self.custom_vars,
-                  should_process=should_process,
                   relative=use_relative_paths,
                   condition=condition))
       else:
-        url = dep_value.get('url')
-        deps_to_add.append(
-            GitDependency(
-                parent=self,
-                name=name,
-                url=url,
-                managed=None,
-                custom_deps=None,
-                custom_vars=self.custom_vars,
-                custom_hooks=None,
-                deps_file=self.recursedeps.get(name, self.deps_file),
-                should_process=should_process,
-                should_recurse=name in self.recursedeps,
-                relative=use_relative_paths,
-                condition=condition))
+        url = self.FormatUrl(name, dep_value.get('url'))
+        if url:
+          deps_to_add.append(
+              GitDependency(
+                  parent=self,
+                  name=name,
+                  url=url,
+                  managed=None,
+                  custom_deps=None,
+                  custom_vars=self.custom_vars,
+                  custom_hooks=None,
+                  deps_file=self.recursedeps.get(name, self.deps_file),
+                  should_recurse=name in self.recursedeps,
+                  relative=use_relative_paths,
+                  condition=condition))
 
     deps_to_add.sort(key=lambda x: x.name)
     return deps_to_add
@@ -666,7 +638,7 @@
     if deps_content:
       try:
         local_scope = gclient_eval.Parse(
-            deps_content, self._get_option('validate_syntax', False),
+            deps_content, self.get_option('validate_syntax', False),
             filepath, self.get_vars())
       except SyntaxError as e:
         gclient_utils.SyntaxErrorToError(filepath, e)
@@ -770,11 +742,8 @@
     self.add_dependencies_and_close(deps_to_add, hooks_to_run)
     logging.info('ParseDepsFile(%s) done' % self.name)
 
-  def _get_option(self, attr, default):
-    obj = self
-    while not hasattr(obj, '_options'):
-      obj = obj.parent
-    return getattr(obj._options, attr, default)
+  def get_option(self, attr, default=None):
+    return getattr(self.root._options, attr, default)
 
   def add_dependencies_and_close(self, deps_to_add, hooks):
     """Adds the dependencies, hooks and mark the parsing as done."""
@@ -800,10 +769,9 @@
       # Don't enforce this for custom_deps.
       if dep.name in self._custom_deps:
         continue
-      if isinstance(dep.url, basestring):
-        parsed_url = urlparse.urlparse(dep.url)
-        if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts:
-          bad_deps.append(dep)
+      parsed_url = urlparse.urlparse(dep.url)
+      if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts:
+        bad_deps.append(dep)
     return bad_deps
 
   def FuzzyMatchUrl(self, candidates):
@@ -845,8 +813,6 @@
     """Runs |command| then parse the DEPS file."""
     logging.info('Dependency(%s).run()' % self.name)
     assert self._file_list == []
-    if not self.should_process:
-      return
     # When running runhooks, there's no need to consult the SCM.
     # All known hooks are expected to run unconditionally regardless of working
     # copy state, so skip the SCM status check.
@@ -855,7 +821,7 @@
     file_list = [] if not options.nohooks else None
     revision_override = revision_overrides.pop(
         self.FuzzyMatchUrl(revision_overrides), None)
-    if run_scm and self.url:
+    if run_scm:
       # Create a shallow copy to mutate revision.
       options = copy.copy(options)
       options.revision = revision_override
@@ -896,8 +862,7 @@
         self.RunPreDepsHooks()
       # Parse the dependencies of this dependency.
       for s in self.dependencies:
-        if s.should_process:
-          work_queue.enqueue(s)
+        work_queue.enqueue(s)
 
     if command == 'recurse':
       # Skip file only checkout.
@@ -907,10 +872,8 @@
         # Pass in the SCM type as an env variable.  Make sure we don't put
         # unicode strings in the environment.
         env = os.environ.copy()
-        if scm:
-          env['GCLIENT_SCM'] = str(scm)
-        if self.url:
-          env['GCLIENT_URL'] = str(self.url)
+        env['GCLIENT_SCM'] = str(scm)
+        env['GCLIENT_URL'] = str(self.url)
         env['GCLIENT_DEP_PATH'] = str(self.name)
         if options.prepend_dir and scm == 'git':
           print_stdout = False
@@ -941,9 +904,7 @@
           print_stdout = True
           filter_fn = None
 
-        if self.url is None:
-          print('Skipped omitted dependency %s' % cwd, file=sys.stderr)
-        elif os.path.isdir(cwd):
+        if os.path.isdir(cwd):
           try:
             gclient_utils.CheckCallAndFilter(
                 args, cwd=cwd, env=env, print_stdout=print_stdout,
@@ -987,9 +948,6 @@
     RunOnDeps() must have been called before to load the DEPS.
     """
     result = []
-    if not self.should_process or not self.should_recurse:
-      # Don't run the hook when it is above recursion_limit.
-      return result
     # If "--force" was specified, run all hooks regardless of what files have
     # changed.
     if self.deps_hooks:
@@ -1032,14 +990,13 @@
       return None
     return self.root.GetCipdRoot()
 
-  def subtree(self, include_all):
+  def subtree(self):
     """Breadth first recursion excluding root node."""
     dependencies = self.dependencies
     for d in dependencies:
-      if d.should_process or include_all:
-        yield d
+      yield d
     for d in dependencies:
-      for i in d.subtree(include_all):
+      for i in d.subtree():
         yield i
 
   @gclient_utils.lockedmethod
@@ -1117,7 +1074,7 @@
   def __str__(self):
     out = []
     for i in ('name', 'url', 'custom_deps',
-              'custom_vars', 'deps_hooks', 'file_list', 'should_process',
+              'custom_vars', 'deps_hooks', 'file_list',
               'processed', 'hooks_ran', 'deps_parsed', 'requirements',
               'allowed_hosts'):
       # First try the native property if it exists.
@@ -1269,13 +1226,12 @@
     super(GClient, self).__init__(
         parent=None,
         name=None,
-        url=None,
+        url='None',
         managed=True,
         custom_deps=None,
         custom_vars=None,
         custom_hooks=None,
         deps_file='unused',
-        should_process=True,
         should_recurse=True,
         relative=None,
         condition=None,
@@ -1298,7 +1254,7 @@
     """Verify that the config matches the state of the existing checked-out
     solutions."""
     for dep in self.dependencies:
-      if dep.managed and dep.url:
+      if dep.managed:
         scm = dep.CreateSCM()
         actual_url = scm.GetActualRemoteURL(self._options)
         if actual_url and not scm.DoesRemoteURLMatch(self._options):
@@ -1368,8 +1324,38 @@
                                 'not specified')
 
     deps_to_add = []
-    for s in config_dict.get('solutions', []):
-      try:
+    solutions = config_dict.get('solutions', [])
+    if len(solutions) == 1 and solutions[0].get('url') is None:
+      s = solutions[0]
+      # If url is not given, or is None, but a DEPS file is specified, we
+      # treat the given DEPS file as a .gclient file and add its dependencies
+      # instead.
+      temp_dep = GitDependency(
+          parent=self,
+          name=s.get('name') or '.',
+          url=None,
+          managed=s.get('managed', True),
+          custom_deps=s.get('custom_deps', {}),
+          custom_vars=s.get('custom_vars', {}),
+          custom_hooks=s.get('custom_hooks', []),
+          deps_file=s.get('deps_file', 'DEPS'),
+          should_recurse=True,
+          relative=None,
+          condition=None,
+          print_outbuf=True)
+      temp_dep.ParseDepsFile()
+      for dep in temp_dep.dependencies:
+        # Note that hooks are not preserved, since they might depend on the
+        # existence of a checkout.
+        dep._custom_vars = temp_dep.get_vars()
+        dep._custom_deps = temp_dep.custom_deps
+        dep._parent = self
+      deps_to_add.extend(temp_dep.dependencies)
+    else:
+      for s in solutions:
+        if not s.get('name') or not s.get('url'):
+          raise gclient_utils.Error('Invalid .gclient file. Solution is '
+                                    'incomplete: %s' % s)
         deps_to_add.append(GitDependency(
             parent=self,
             name=s['name'],
@@ -1379,15 +1365,14 @@
             custom_vars=s.get('custom_vars', {}),
             custom_hooks=s.get('custom_hooks', []),
             deps_file=s.get('deps_file', 'DEPS'),
-            should_process=True,
             should_recurse=True,
             relative=None,
             condition=None,
             print_outbuf=True))
-      except KeyError:
-        raise gclient_utils.Error('Invalid .gclient file. Solution is '
-                                  'incomplete: %s' % s)
     self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', []))
+    if not self.dependencies:
+      raise gclient_utils.Error('No solution specified')
+
     logging.info('SetConfig() done')
 
   def SaveConfig(self):
@@ -1447,7 +1432,7 @@
     # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It
     # makes testing a bit too fun.
     result = 'entries = {\n'
-    for entry in self.root.subtree(False):
+    for entry in self.root.subtree():
       result += '  %s: %s,\n' % (pprint.pformat(entry.name),
           pprint.pformat(entry.url))
     result += '}\n'
@@ -1515,9 +1500,6 @@
       command: The command to use (e.g., 'status' or 'diff')
       args: list of str - extra arguments to add to the command line.
     """
-    if not self.dependencies:
-      raise gclient_utils.Error('No solution specified')
-
     revision_overrides = {}
     patch_refs = {}
     # It's unnecessary to check for revision overrides for 'recurse'.
@@ -1542,8 +1524,7 @@
         self._options.jobs, pm, ignore_requirements=ignore_requirements,
         verbose=self._options.verbose)
     for s in self.dependencies:
-      if s.should_process:
-        work_queue.enqueue(s)
+      work_queue.enqueue(s)
     work_queue.flush(revision_overrides, command, args, options=self._options,
                      patch_refs=patch_refs)
 
@@ -1580,7 +1561,7 @@
       # Notify the user if there is an orphaned entry in their working copy.
       # Only delete the directory if there are no changes in it, and
       # delete_unversioned_trees is set to true.
-      entries = [i.name for i in self.root.subtree(False) if i.url]
+      entries = [i.name for i in self.root.subtree()]
       full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep))
                       for e in entries]
 
@@ -1677,68 +1658,59 @@
     work_queue = gclient_utils.ExecutionQueue(
         self._options.jobs, None, False, verbose=self._options.verbose)
     for s in self.dependencies:
-      if s.should_process:
-        work_queue.enqueue(s)
+      work_queue.enqueue(s)
     work_queue.flush({}, None, [], options=self._options, patch_refs=None)
 
-    def ShouldPrintRevision(dep):
+    def ShouldPrint(dep):
       return (not self._options.filter
               or dep.FuzzyMatchUrl(self._options.filter))
 
+    for dep in self.subtree():
+      if self._options.snapshot or self._options.actual:
+        dep.PinToActualRevision()
+
     if self._options.snapshot:
-      json_output = []
-      # First level at .gclient
-      for d in self.dependencies:
-        entries = {}
-        def GrabDeps(dep):
-          """Recursively grab dependencies."""
-          for d in dep.dependencies:
-            d.PinToActualRevision()
-            if ShouldPrintRevision(d):
-              entries[d.name] = d.url
-            GrabDeps(d)
-        GrabDeps(d)
-        json_output.append({
-            'name': d.name,
-            'solution_url': d.url,
-            'deps_file': d.deps_file,
-            'managed': d.managed,
-            'custom_deps': entries,
-        })
-      if self._options.output_json == '-':
-        print(json.dumps(json_output, indent=2, separators=(',', ': ')))
-      elif self._options.output_json:
-        with open(self._options.output_json, 'w') as f:
-          json.dump(json_output, f)
-      else:
-        # Print the snapshot configuration file
-        print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {
-            'solution_list': pprint.pformat(json_output, indent=2),
-        })
+      json_output = [
+          {
+              'name': d.name,
+              'solution_url': d.url,
+              'deps_file': d.deps_file,
+              'managed': d.managed,
+              'custom_deps': {
+                  subdep.name: subdep.url
+                  for subdep in d.subtree()
+                  if ShouldPrint(subdep)
+              },
+          }
+          for d in self.dependencies
+          if ShouldPrint(d)
+      ]
+      output = json.dumps(json_output, indent=2, separators=(',', ': '))
+      if not self._options.output_json:
+        output = self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': output}
+    elif self._options.output_json:
+      json_output = {
+          d.name: {
+              'url': d.url.split('@')[0],
+              'rev': d.url.split('@')[1] if '@' in d.url else None,
+          }
+          for d in self.subtree()
+          if ShouldPrint(d)
+      }
+      output = json.dumps(json_output, indent=2, separators=(',', ': '))
     else:
-      entries = {}
-      for d in self.root.subtree(False):
-        if self._options.actual:
-          d.PinToActualRevision()
-        if ShouldPrintRevision(d):
-          entries[d.name] = d.url
-      if self._options.output_json:
-        json_output = {
-            name: {
-                'url': rev.split('@')[0] if rev else None,
-                'rev': rev.split('@')[1] if rev and '@' in rev else None,
-            }
-            for name, rev in entries.iteritems()
-        }
-        if self._options.output_json == '-':
-          print(json.dumps(json_output, indent=2, separators=(',', ': ')))
-        else:
-          with open(self._options.output_json, 'w') as f:
-            json.dump(json_output, f)
-      else:
-        keys = sorted(entries.keys())
-        for x in keys:
-          print('%s: %s' % (x, entries[x]))
+      output = '\n'.join(
+          '%s: %s' % (d.name, d.url)
+          for d in self.subtree()
+          if ShouldPrint(d)
+      )
+
+    if self._options.output_json and self._options.output_json != '-':
+      with open(self._options.output_json, 'w') as f:
+        f.write(output)
+    else:
+      print(output)
+
     logging.info(str(self))
 
   def ParseDepsFile(self):
@@ -1784,9 +1756,8 @@
 class CipdDependency(Dependency):
   """A Dependency object that represents a single CIPD package."""
 
-  def __init__(
-      self, parent, name, dep_value, cipd_root,
-      custom_vars, should_process, relative, condition):
+  def __init__(self, parent, name, dep_value, cipd_root, custom_vars, relative,
+               condition):
     package = dep_value['package']
     version = dep_value['version']
     url = urlparse.urljoin(
@@ -1800,7 +1771,6 @@
         custom_vars=custom_vars,
         custom_hooks=None,
         deps_file=None,
-        should_process=should_process,
         should_recurse=False,
         relative=relative,
         condition=condition)
@@ -1812,6 +1782,7 @@
     self._cipd_root = cipd_root
     self._cipd_subdir = os.path.relpath(
         os.path.join(self.root.root_dir, name), cipd_root.root_dir)
+    self._package_path = name
     self._package_name = package
     self._package_version = version
 
@@ -1820,8 +1791,6 @@
           patch_refs):
     """Runs |command| then parse the DEPS file."""
     logging.info('CipdDependency(%s).run()' % self.name)
-    if not self.should_process:
-      return
     self._CreatePackageIfNecessary()
     super(CipdDependency, self).run(revision_overrides, command, args,
                                     work_queue, options, patch_refs)
@@ -1857,6 +1826,10 @@
         self.url, self.root.root_dir, self.name, self.outbuf, out_cb,
         root=self._cipd_root, package=self._cipd_package)
 
+  def hierarchy(self, include_url=True):
+    """Returns a human-readable hierarchical reference to a Dependency."""
+    return self.parent.hierarchy(include_url) + ' -> ' + self._package_path
+
   def ToLines(self):
     """Return a list of lines representing this in a DEPS file."""
     s = []
@@ -1984,9 +1957,6 @@
     Arguments:
       dep (Dependency): dependency to process
     """
-    if dep.url is None:
-      return
-
     # Make sure the revision is always fully specified (a hash),
     # as opposed to refs or tags which might change. Similarly,
     # shortened shas might become ambiguous; make sure to always
@@ -2004,7 +1974,8 @@
     """
     for solution in self._client.dependencies:
       self._add_dep(solution)
-      self._flatten_dep(solution)
+      if solution.should_recurse:
+        self._flatten_dep(solution)
 
     if pin_all_deps:
       for dep in self._deps.itervalues():
@@ -2023,7 +1994,6 @@
         deps_path = os.path.join(self._client.root_dir, dep.name, deps_file)
         if not os.path.exists(deps_path):
           return
-      assert dep.url
       self._deps_files.add((dep.url, deps_file, dep.hierarchy_data()))
     for dep in self._deps.itervalues():
       add_deps_file(dep)
@@ -2049,8 +2019,7 @@
     """
     assert dep.name not in self._deps or self._deps.get(dep.name) == dep, (
         dep.name, self._deps.get(dep.name))
-    if dep.url:
-      self._deps[dep.name] = dep
+    self._deps[dep.name] = dep
 
   def _flatten_dep(self, dep):
     """Visits a dependency in order to flatten it (see CMDflatten).
@@ -2561,16 +2530,12 @@
   ret = client.RunOnDeps('update', args)
   if options.output_json:
     slns = {}
-    for d in client.subtree(True):
+    for d in client.subtree():
       normed = d.name.replace('\\', '/').rstrip('/') + '/'
-      if normed in slns and not d.should_process:
-        # If an unprocessed dependency would override an existing dependency,
-        # ignore it.
-        continue
       slns[normed] = {
           'revision': d.got_revision,
           'scm': d.used_scm.name if d.used_scm else None,
-          'url': str(d.url) if d.url else None,
+          'url': str(d.url),
       }
     with open(options.output_json, 'wb') as f:
       json.dump({'solutions': slns}, f)