blob: 150c3fcc7dc4b4a353dbbd71a1af20e139536cc3 [file] [log] [blame]
Kuang-che Wu6e4beca2018-06-27 17:45:02 +08001# -*- coding: utf-8 -*-
Kuang-che Wu3eb6b502018-06-06 16:15:18 +08002# Copyright 2018 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Gclient utility."""
6
7from __future__ import print_function
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +08008import collections
9import itertools
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080010import logging
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +080011import operator
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080012import os
Kuang-che Wu6948ecc2018-09-11 17:43:49 +080013import pprint
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +080014import queue
Kuang-che Wu067ff292019-02-14 18:16:23 +080015import shutil
Kuang-che Wub17b3b92018-09-04 18:12:11 +080016import sys
Kuang-che Wu1e49f512018-12-06 15:27:42 +080017import tempfile
Kuang-che Wu999893c2020-04-13 22:06:22 +080018import urllib.parse
Kuang-che Wua7ddf9b2019-11-25 18:59:57 +080019
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +080020# from third_party
21from depot_tools import gclient_eval
22
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080023from bisect_kit import codechange
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +080024from bisect_kit import git_util
Kuang-che Wu1e49f512018-12-06 15:27:42 +080025from bisect_kit import locking
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080026from bisect_kit import util
27
28logger = logging.getLogger(__name__)
Kuang-che Wuced2dbf2019-01-30 23:13:24 +080029emitted_warnings = set()
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080030
Kuang-che Wu88e96312020-10-20 16:21:11 +080031# If the dependency is not pinned in DEPS file, the default branch.
32# ref: gclient_scm.py GitWrapper.update default_rev
33# TODO(kcwu): follow gclient to change the default branch name
34DEFAULT_BRANCH_NAME = 'master'
35
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080036
Kuang-che Wu41e8b592018-09-25 17:01:30 +080037def config(gclient_dir,
38 url=None,
39 cache_dir=None,
40 deps_file=None,
41 custom_var=None,
Kuang-che Wucb8a80e2021-01-18 15:22:27 +080042 gclientfile=None,
43 spec=None,
44 target_os=None):
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080045 """Simply wrapper of `gclient config`.
46
47 Args:
48 gclient_dir: root directory of gclient project
49 url: URL of gclient configuration files
50 cache_dir: gclient's git cache folder
51 deps_file: override the default DEPS file name
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +080052 custom_var: custom variables
Kuang-che Wucb8a80e2021-01-18 15:22:27 +080053 gclientfile: alternate .gclient file name
Kuang-che Wu41e8b592018-09-25 17:01:30 +080054 spec: content of gclient file
Kuang-che Wucb8a80e2021-01-18 15:22:27 +080055 target_os: target OS
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080056 """
57 cmd = ['gclient', 'config']
58 if deps_file:
59 cmd += ['--deps-file', deps_file]
60 if cache_dir:
61 cmd += ['--cache-dir', cache_dir]
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +080062 if custom_var:
63 cmd += ['--custom-var', custom_var]
Kuang-che Wucb8a80e2021-01-18 15:22:27 +080064 if gclientfile:
65 cmd += ['--gclientfile', gclientfile]
Kuang-che Wu41e8b592018-09-25 17:01:30 +080066 if spec:
67 cmd += ['--spec', spec]
68 if url:
69 cmd.append(url)
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080070
71 util.check_call(*cmd, cwd=gclient_dir)
72
Kuang-che Wucb8a80e2021-01-18 15:22:27 +080073 # 'target_os' is mandatory for chromeos build, but 'gclient config' doesn't
74 # recognize it. Here add it to gclient file explicitly.
75 if target_os:
76 if not gclientfile:
77 gclientfile = '.gclient'
78 with open(os.path.join(gclient_dir, gclientfile), 'a') as f:
79 f.write('target_os = ["%s"]\n' % target_os)
80
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080081
Kuang-che Wu6ee24b52020-11-02 11:33:49 +080082def sync(gclient_dir, with_branch_heads=False, with_tags=False, jobs=8):
Kuang-che Wu3eb6b502018-06-06 16:15:18 +080083 """Simply wrapper of `gclient sync`.
84
85 Args:
86 gclient_dir: root directory of gclient project
87 with_branch_heads: whether to clone git `branch_heads` refspecs
88 with_tags: whether to clone git tags
89 jobs: how many workers running in parallel
90 """
Kuang-che Wua9639eb2019-03-19 17:15:08 +080091 # Work around gclient issue crbug/943430
92 # gclient rejected to sync if there are untracked symlink even with --force
93 for path in [
94 'src/chromeos/assistant/libassistant/src/deps',
95 'src/chromeos/assistant/libassistant/src/libassistant',
96 ]:
97 if os.path.islink(os.path.join(gclient_dir, path)):
98 os.unlink(os.path.join(gclient_dir, path))
99
100 cmd = [
101 'gclient',
102 'sync',
103 '--jobs=%d' % jobs,
104 '--delete_unversioned_trees',
105 # --force is necessary because runhook may generate some untracked files.
106 '--force',
107 ]
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800108 if with_branch_heads:
109 cmd.append('--with_branch_heads')
110 if with_tags:
111 cmd.append('--with_tags')
Kuang-che Wudc714412018-10-17 16:06:39 +0800112
Kuang-che Wu067ff292019-02-14 18:16:23 +0800113 try:
114 old_projects = load_gclient_entries(gclient_dir)
115 except IOError:
116 old_projects = {}
117
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800118 util.check_call(*cmd, cwd=gclient_dir)
119
Kuang-che Wu067ff292019-02-14 18:16:23 +0800120 # Remove dead .git folder after sync.
121 # Ideally, this should be handled by gclient but sometimes gclient didn't
122 # (crbug/930047).
123 new_projects = load_gclient_entries(gclient_dir)
124 for path in old_projects:
125 if path in new_projects:
126 continue
127 old_git_dir = os.path.join(gclient_dir, path)
Kuang-che Wucbe12432019-03-18 19:35:03 +0800128 if not os.path.exists(old_git_dir):
129 continue
130
131 if git_util.is_git_root(old_git_dir):
Kuang-che Wu067ff292019-02-14 18:16:23 +0800132 logger.warning(
133 '%s was removed from .gclient_entries but %s still exists; remove it',
134 path, old_git_dir)
135 shutil.rmtree(old_git_dir)
Kuang-che Wucbe12432019-03-18 19:35:03 +0800136 else:
137 logger.warning(
138 '%s was removed from .gclient_entries but %s still exists;'
139 ' keep it because it is not git root', path, old_git_dir)
140
141
Kuang-che Wucb8a80e2021-01-18 15:22:27 +0800142def runhook(gclient_dir, jobs=8, gclientfile=None):
Kuang-che Wucbe12432019-03-18 19:35:03 +0800143 """Simply wrapper of `gclient runhook`.
144
145 Args:
146 gclient_dir: root directory of gclient project
147 jobs: how many workers running in parallel
148 """
Kuang-che Wucb8a80e2021-01-18 15:22:27 +0800149 cmd = ['gclient', 'runhook', '--job', str(jobs)]
150 if gclientfile:
151 cmd += ['--gclientfile', gclientfile]
152 util.check_call(*cmd, cwd=gclient_dir)
Kuang-che Wu067ff292019-02-14 18:16:23 +0800153
154
155def load_gclient_entries(gclient_dir):
156 """Loads .gclient_entries."""
157 repo_project_list = os.path.join(gclient_dir, '.gclient_entries')
158 scope = {}
Kuang-che Wud2646f42019-11-27 18:31:08 +0800159 with open(repo_project_list) as f:
160 code = compile(f.read(), repo_project_list, 'exec')
Kuang-che Wub1b18ea2021-01-18 15:03:30 +0800161 # pylint: disable=exec-used
162 exec(code, scope)
Kuang-che Wu058ac042019-03-18 14:14:53 +0800163 entries = scope.get('entries', {})
164
165 # normalize path: remove trailing slash
166 entries = dict((os.path.normpath(path), url) for path, url in entries.items())
167
168 return entries
Kuang-che Wu067ff292019-02-14 18:16:23 +0800169
170
171def write_gclient_entries(gclient_dir, projects):
172 """Writes .gclient_entries."""
173 repo_project_list = os.path.join(gclient_dir, '.gclient_entries')
174 content = 'entries = {\n'
Kuang-che Wuc89f2a22019-11-26 15:30:50 +0800175 for path, repo_url in sorted(projects.items()):
176 content += ' %s: %s,\n' % (pprint.pformat(path), pprint.pformat(repo_url))
Kuang-che Wu067ff292019-02-14 18:16:23 +0800177 content += '}\n'
178 with open(repo_project_list, 'w') as f:
179 f.write(content)
180
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800181
Kuang-che Wu1e49f512018-12-06 15:27:42 +0800182def mirror(code_storage, repo_url):
183 """Mirror git repo.
184
185 This function mimics the caching behavior of 'gclient sync' with 'cache_dir'.
186
187 Args:
188 code_storage: CodeStorage object
189 repo_url: remote repo url
190 """
191 logger.info('mirror %s', repo_url)
192 tmp_dir = tempfile.mkdtemp(dir=code_storage.cache_dir)
193 git_root = code_storage.cached_git_root(repo_url)
194 assert not os.path.exists(git_root)
195
196 util.check_call('git', 'init', '--bare', cwd=tmp_dir)
197
198 # These config parameters are copied from gclient.
199 git_util.config(tmp_dir, 'gc.autodetach', '0')
200 git_util.config(tmp_dir, 'gc.autopacklimit', '0')
201 git_util.config(tmp_dir, 'core.deltaBaseCacheLimit', '2g')
202 git_util.config(tmp_dir, 'remote.origin.url', repo_url)
203 git_util.config(tmp_dir, '--replace-all', 'remote.origin.fetch',
204 '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*')
205 git_util.config(tmp_dir, '--replace-all', 'remote.origin.fetch',
206 '+refs/tags/*:refs/tags/*', r'\+refs/tags/\*:.*')
207 git_util.config(tmp_dir, '--replace-all', 'remote.origin.fetch',
208 '+refs/branch-heads/*:refs/branch-heads/*',
209 r'\+refs/branch-heads/\*:.*')
210
211 git_util.fetch(tmp_dir, 'origin', '+refs/heads/*:refs/heads/*')
212 git_util.fetch(tmp_dir, 'origin', '+refs/tags/*:refs/tags/*')
213 git_util.fetch(tmp_dir, 'origin', '+refs/branch-heads/*:refs/branch-heads/*')
214
215 # Rename to correct name atomically.
216 os.rename(tmp_dir, git_root)
217
218
Kuang-che Wub17b3b92018-09-04 18:12:11 +0800219# Copied from depot_tools' gclient.py
220_PLATFORM_MAPPING = {
221 'cygwin': 'win',
222 'darwin': 'mac',
223 'linux2': 'linux',
Kuang-che Wud2646f42019-11-27 18:31:08 +0800224 'linux': 'linux',
Kuang-che Wub17b3b92018-09-04 18:12:11 +0800225 'win32': 'win',
226 'aix6': 'aix',
227}
228
229
230def _detect_host_os():
231 return _PLATFORM_MAPPING[sys.platform]
232
233
Kuang-che Wu23192ad2020-03-11 18:12:46 +0800234class Dep:
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800235 """Represent one entry of DEPS's deps.
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800236
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800237 One Dep object means one subproject inside DEPS file. It recorded what to
238 checkout (like git or cipd) content of each subproject.
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800239
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800240 Attributes:
241 path: subproject path, relative to project root
242 variables: the variables of the containing DEPS file; these variables will
243 be applied to fields of this object (like 'url' and 'condition') and
244 children projects.
245 condition: whether to checkout this subproject
246 dep_type: 'git' or 'cipd'
247 url: if dep_type='git', the url of remote repo and associated branch/commit
248 packages: if dep_type='cipd', cipd package version and location
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800249 """
250
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800251 def __init__(self, path, variables, entry):
252 self.path = path
253 self.variables = variables
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800254
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800255 self.url = None # only valid for dep_type='git'
256 self.packages = None # only valid for dep_type='cipd'
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800257
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800258 if isinstance(entry, str):
259 self.dep_type = 'git'
260 self.url = entry
261 self.condition = None
262 else:
263 self.dep_type = entry.get('dep_type', 'git')
264 self.condition = entry.get('condition')
265 if self.dep_type == 'git':
266 self.url = entry['url']
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800267 else:
Kuang-che Wufe1e88a2019-09-10 21:52:25 +0800268 assert self.dep_type == 'cipd', 'unknown dep_type:' + self.dep_type
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800269 self.packages = entry['packages']
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800270
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800271 if self.dep_type == 'git':
272 self.url = self.url.format(**self.variables)
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800273
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800274 def __eq__(self, rhs):
275 return vars(self) == vars(rhs)
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800276
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800277 def __ne__(self, rhs):
278 return not self.__eq__(rhs)
279
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800280 def set_url(self, repo_url, at):
281 assert self.dep_type == 'git'
282 self.url = '%s@%s' % (repo_url, at)
283
284 def set_revision(self, at):
285 assert self.dep_type == 'git'
286 repo_url, _ = self.parse_url()
287 self.set_url(repo_url, at)
288
289 def parse_url(self):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800290 assert self.dep_type == 'git'
291
292 if '@' in self.url:
293 repo_url, at = self.url.split('@')
294 else:
Kuang-che Wu88e96312020-10-20 16:21:11 +0800295 # If the dependency is not pinned, the default branch.
296 repo_url, at = self.url, DEFAULT_BRANCH_NAME
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800297 return repo_url, at
298
299 def as_path_spec(self):
300 repo_url, at = self.parse_url()
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800301 return codechange.PathSpec(self.path, repo_url, at)
302
303 def eval_condition(self):
304 """Evaluate condition for DEPS parsing.
305
306 Returns:
307 eval result
308 """
309 if not self.condition:
310 return True
311
Kuang-che Wu3e182ec2019-02-21 15:04:30 +0800312 # Currently, we only support chromeos as target_os.
313 # TODO(kcwu): make it configurable if we need to bisect for other os.
Kuang-che Wu0d7409c2019-03-18 12:29:03 +0800314 # We don't specify `target_os_only`, so `unix` will be considered by
315 # gclient as well.
316 target_os = ['chromeos', 'unix']
Kuang-che Wu3e182ec2019-02-21 15:04:30 +0800317
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800318 vars_dict = {
Kuang-che Wu3e182ec2019-02-21 15:04:30 +0800319 'checkout_android': 'android' in target_os,
320 'checkout_chromeos': 'chromeos' in target_os,
321 'checkout_fuchsia': 'fuchsia' in target_os,
322 'checkout_ios': 'ios' in target_os,
323 'checkout_linux': 'unix' in target_os,
324 'checkout_mac': 'mac' in target_os,
325 'checkout_win': 'win' in target_os,
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800326 # default cpu: x64
327 'checkout_arm64': False,
328 'checkout_arm': False,
329 'checkout_mips': False,
330 'checkout_ppc': False,
331 'checkout_s390': False,
332 'checkout_x64': True,
333 'checkout_x86': False,
334 'host_os': _detect_host_os(),
335 'False': False,
336 'None': None,
337 'True': True,
338 }
339 vars_dict.update(self.variables)
340 # pylint: disable=eval-used
341 return eval(self.condition, vars_dict)
342
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800343 def to_lines(self):
344 s = []
345 condition_part = ([' "condition": %r,' %
346 self.condition] if self.condition else [])
347 if self.dep_type == 'cipd':
348 s.extend([
349 ' "%s": {' % (self.path.split(':')[0],),
350 ' "packages": [',
351 ])
352 for p in sorted(self.packages, key=lambda x: x['package']):
353 s.extend([
354 ' {',
355 ' "package": "%s",' % p['package'],
356 ' "version": "%s",' % p['version'],
357 ' },',
358 ])
359 s.extend([
360 ' ],',
361 ' "dep_type": "cipd",',
362 ] + condition_part + [
363 ' },',
364 '',
365 ])
366 else:
367 s.extend([
368 ' "%s": {' % (self.path,),
369 ' "url": "%s",' % (self.url,),
370 ] + condition_part + [
371 ' },',
372 '',
373 ])
374 return s
375
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800376
Kuang-che Wu23192ad2020-03-11 18:12:46 +0800377class Deps:
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800378 """DEPS parsed result.
379
380 Attributes:
381 variables: 'vars' dict in DEPS file; these variables will be applied
382 recursively to children.
383 entries: dict of Dep objects
384 recursedeps: list of recursive projects
385 """
386
387 def __init__(self):
388 self.variables = {}
389 self.entries = {}
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800390 self.ignored_entries = {}
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800391 self.recursedeps = []
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800392 self.allowed_hosts = set()
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800393 self.gn_args_from = None
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800394 self.gn_args_file = None
395 self.gn_args = []
396 self.hooks = []
397 self.pre_deps_hooks = []
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800398 self.modified = set()
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800399
400 def _gn_settings_to_lines(self):
401 s = []
402 if self.gn_args_file:
403 s.extend([
404 'gclient_gn_args_file = "%s"' % self.gn_args_file,
405 'gclient_gn_args = %r' % self.gn_args,
406 ])
407 return s
408
409 def _allowed_hosts_to_lines(self):
410 """Converts |allowed_hosts| set to list of lines for output."""
411 if not self.allowed_hosts:
412 return []
413 s = ['allowed_hosts = [']
414 for h in sorted(self.allowed_hosts):
415 s.append(' "%s",' % h)
416 s.extend([']', ''])
417 return s
418
419 def _entries_to_lines(self):
420 """Converts |entries| dict to list of lines for output."""
421 entries = self.ignored_entries
422 entries.update(self.entries)
423 if not entries:
424 return []
425 s = ['deps = {']
426 for _, dep in sorted(entries.items()):
427 s.extend(dep.to_lines())
428 s.extend(['}', ''])
429 return s
430
431 def _vars_to_lines(self):
432 """Converts |variables| dict to list of lines for output."""
433 if not self.variables:
434 return []
435 s = ['vars = {']
436 for key, value in sorted(self.variables.items()):
437 s.extend([
438 ' "%s": %r,' % (key, value),
439 '',
440 ])
441 s.extend(['}', ''])
442 return s
443
444 def _hooks_to_lines(self, name, hooks):
445 """Converts |hooks| list to list of lines for output."""
446 if not hooks:
447 return []
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800448 hooks.sort(key=lambda x: x.get('name', ''))
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800449 s = ['%s = [' % name]
450 for hook in hooks:
451 s.extend([
452 ' {',
453 ])
454 if hook.get('name') is not None:
455 s.append(' "name": "%s",' % hook.get('name'))
456 if hook.get('pattern') is not None:
457 s.append(' "pattern": "%s",' % hook.get('pattern'))
458 if hook.get('condition') is not None:
459 s.append(' "condition": %r,' % hook.get('condition'))
460 # Flattened hooks need to be written relative to the root gclient dir
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800461 cwd = os.path.relpath(os.path.normpath(hook.get('cwd', '.')))
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800462 s.extend([' "cwd": "%s",' % cwd] + [' "action": ['] +
463 [' "%s",' % arg for arg in hook.get('action', [])] +
464 [' ]', ' },', ''])
465 s.extend([']', ''])
466 return s
467
468 def to_string(self):
469 """Return flatten DEPS string."""
470 return '\n'.join(
471 self._gn_settings_to_lines() + self._allowed_hosts_to_lines() +
472 self._entries_to_lines() + self._hooks_to_lines('hooks', self.hooks) +
473 self._hooks_to_lines('pre_deps_hooks', self.pre_deps_hooks) +
474 self._vars_to_lines() + ['']) # Ensure newline at end of file.
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800475
Zheng-Jie Changc0d3cd72020-06-03 02:46:43 +0800476 def remove_src(self):
477 """Return src_revision for buildbucket use."""
478 assert 'src' in self.entries
479 _, src_rev = self.entries['src'].parse_url()
480 del self.entries['src']
481 return src_rev
482
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800483 def apply_commit(self, path, revision, overwrite=True):
484 """Set revision to a project by path.
485
486 Args:
487 path: A project's path.
488 revision: A git commit id.
489 overwrite: Overwrite flag, the project won't change if overwrite=False
490 and it was modified before.
491 """
492 if path in self.modified and not overwrite:
493 return
494 self.modified.add(path)
495
496 if path not in self.entries:
497 logger.warning('path: %s not found in DEPS', path)
498 return
499 self.entries[path].set_revision(revision)
500
501 def apply_action_groups(self, action_groups):
502 """Apply multiple action groups to DEPS.
503
504 If there are multiple actions in one repo, only last one is applied.
505
506 Args:
507 action_groups: A list of action groups.
508 """
509 # Apply in reversed order with overwrite=False,
510 # so each repo is on the state of last action.
511 for action_group in reversed(action_groups):
512 for action in reversed(action_group.actions):
513 if isinstance(action, codechange.GitCheckoutCommit):
514 self.apply_commit(action.path, action.rev, overwrite=False)
515 if isinstance(action, codechange.GitAddRepo):
516 self.apply_commit(action.path, action.rev, overwrite=False)
517 if isinstance(action, codechange.GitRemoveRepo):
518 assert action.path not in self.entries
519 self.modified.add(action.path)
520
521 def apply_deps(self, deps):
522 for path, dep in deps.entries.items():
523 if path in self.entries:
524 _, rev = dep.parse_url()
525 self.apply_commit(path, rev, overwrite=False)
526
527 # hooks, vars, ignored_entries are ignored and should be set by float_spec
528
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800529
Kuang-che Wu23192ad2020-03-11 18:12:46 +0800530class TimeSeriesTree:
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800531 """Data structure for generating snapshots of historical dependency tree.
532
533 This is a tree structure with time information. Each tree node represents not
534 only typical tree data and tree children information, but also historical
535 value of those tree data and tree children.
536
537 To be more specific in terms of DEPS parsing, one TimeSeriesTree object
538 represent a DEPS file. The caller will add_snapshot() to add parsed result of
539 historical DEPS instances. After that, the tree root of this class can
540 reconstruct the every historical moment of the project dependency state.
541
542 This class is slight abstraction of git_util.get_history_recursively() to
543 support more than single git repo and be version control system independent.
544 """
545
546 # TODO(kcwu): refactor git_util.get_history_recursively() to reuse this class.
547
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800548 def __init__(self, parent_deps, entry, start_time, end_time):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800549 """TimeSeriesTree constructor.
550
551 Args:
552 parent_deps: parent DEPS of the given period. None if this is tree root.
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800553 entry: project entry
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800554 start_time: start time
555 end_time: end time
556 """
557 self.parent_deps = parent_deps
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800558 self.entry = entry
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800559 self.snapshots = {}
560 self.start_time = start_time
561 self.end_time = end_time
562
563 # Intermediate dict to keep track alive children for the time being.
564 # Maintained by add_snapshot() and no_more_snapshot().
565 self.alive_children = {}
566
567 # All historical children (TimeSeriesTree object) between start_time and
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800568 # end_time. It's possible that children with the same entry appear more than
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800569 # once in this list because they are removed and added back to the DEPS
570 # file.
571 self.subtrees = []
572
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800573 def subtree_eq(self, deps_a, deps_b, child_entry):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800574 """Compares subtree of two Deps.
575
576 Args:
577 deps_a: Deps object
578 deps_b: Deps object
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800579 child_entry: the subtree to compare
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800580
581 Returns:
582 True if the said subtree of these two Deps equal
583 """
584 # Need to compare variables because they may influence subtree parsing
585 # behavior
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800586 path = child_entry[0]
587 return (deps_a.entries[path] == deps_b.entries[path] and
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800588 deps_a.variables == deps_b.variables)
589
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800590 def add_snapshot(self, timestamp, deps, children_entries):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800591 """Adds parsed DEPS result and children.
592
593 For example, if a given DEPS file has N revisions between start_time and
594 end_time, the caller should call this method N times to feed all parsed
595 results in order (timestamp increasing).
596
597 Args:
598 timestamp: timestamp of `deps`
599 deps: Deps object
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800600 children_entries: list of names of deps' children
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800601 """
602 assert timestamp not in self.snapshots
603 self.snapshots[timestamp] = deps
604
Kuang-che Wua7ddf9b2019-11-25 18:59:57 +0800605 for child_entry in set(list(self.alive_children.keys()) + children_entries):
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800606 # `child_entry` is added at `timestamp`
607 if child_entry not in self.alive_children:
608 self.alive_children[child_entry] = timestamp, deps
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800609
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800610 # `child_entry` is removed at `timestamp`
611 elif child_entry not in children_entries:
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800612 self.subtrees.append(
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800613 TimeSeriesTree(self.alive_children[child_entry][1], child_entry,
614 self.alive_children[child_entry][0], timestamp))
615 del self.alive_children[child_entry]
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800616
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800617 # `child_entry` is alive before and after `timestamp`
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800618 else:
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800619 last_deps = self.alive_children[child_entry][1]
620 if not self.subtree_eq(last_deps, deps, child_entry):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800621 self.subtrees.append(
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800622 TimeSeriesTree(last_deps, child_entry,
623 self.alive_children[child_entry][0], timestamp))
624 self.alive_children[child_entry] = timestamp, deps
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800625
Kuang-che Wua7ddf9b2019-11-25 18:59:57 +0800626 def no_more_snapshot(self):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800627 """Indicates all snapshots are added.
628
629 add_snapshot() should not be invoked after no_more_snapshot().
630 """
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800631 for child_entry, (timestamp, deps) in self.alive_children.items():
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800632 if timestamp == self.end_time:
633 continue
634 self.subtrees.append(
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800635 TimeSeriesTree(deps, child_entry, timestamp, self.end_time))
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800636 self.alive_children = None
637
638 def events(self):
639 """Gets children added/removed events of this subtree.
640
641 Returns:
642 list of (timestamp, deps_name, deps, end_flag):
643 timestamp: timestamp of event
644 deps_name: name of this subtree
645 deps: Deps object of given project
646 end_flag: True indicates this is the last event of this deps tree
647 """
648 assert self.snapshots
649 assert self.alive_children is None, ('events() is valid only after '
650 'no_more_snapshot() is invoked')
651
652 result = []
653
654 last_deps = None
655 for timestamp, deps in self.snapshots.items():
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800656 result.append((timestamp, self.entry, deps, False))
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800657 last_deps = deps
658
659 assert last_deps
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800660 result.append((self.end_time, self.entry, last_deps, True))
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800661
662 for subtree in self.subtrees:
663 for event in subtree.events():
664 result.append(event)
665
Kuang-che Wu1ad2c0e2019-02-26 00:41:10 +0800666 result.sort(key=lambda x: x[0])
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800667
668 return result
669
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800670 def iter_forest(self):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800671 """Iterates snapshots of project dependency state.
672
673 Yields:
674 (timestamp, path_specs):
675 timestamp: time of snapshot
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800676 forest: A dict indicates path => deps mapping
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800677 """
678 forest = {}
679 # Group by timestamp
680 for timestamp, events in itertools.groupby(self.events(),
681 operator.itemgetter(0)):
682 # It's possible that one deps is removed and added at the same timestamp,
683 # i.e. modification, so use counter to track.
684 end_counter = collections.Counter()
685
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800686 for timestamp, entry, deps, end in events:
687 forest[entry] = deps
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800688 if end:
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800689 end_counter[entry] += 1
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800690 else:
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800691 end_counter[entry] -= 1
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800692
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800693 yield timestamp, forest
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800694
695 # Remove deps which are removed at this timestamp.
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800696 for entry, count in end_counter.items():
697 assert -1 <= count <= 1, (timestamp, entry)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800698 if count == 1:
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800699 del forest[entry]
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800700
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800701 def iter_path_specs(self):
702 """Iterates snapshots of project dependency state.
703
704 Yields:
705 (timestamp, path_specs):
706 timestamp: time of snapshot
707 path_specs: dict of path_spec entries
708 """
709 for timestamp, forest in self.iter_forest():
710 path_specs = {}
711 # Merge Deps at time `timestamp` into single path_specs.
712 for deps in forest.values():
713 for path, dep in deps.entries.items():
714 path_specs[path] = dep.as_path_spec()
715 yield timestamp, path_specs
716
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800717
Kuang-che Wu23192ad2020-03-11 18:12:46 +0800718class DepsParser:
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800719 """Gclient DEPS file parser."""
720
721 def __init__(self, project_root, code_storage):
722 self.project_root = project_root
723 self.code_storage = code_storage
724
Kuang-che Wu067ff292019-02-14 18:16:23 +0800725 def load_single_deps(self, content):
726
727 def var_function(name):
728 return '{%s}' % name
729
Zheng-Jie Chang381e4162020-08-03 14:26:54 +0800730 def str_function(name):
731 return str(name)
732
733 global_scope = dict(Var=var_function, Str=str_function)
Kuang-che Wu067ff292019-02-14 18:16:23 +0800734 local_scope = {}
Kuang-che Wub1b18ea2021-01-18 15:03:30 +0800735 # pylint: disable=exec-used
736 exec(content, global_scope, local_scope)
Kuang-che Wu067ff292019-02-14 18:16:23 +0800737 return local_scope
738
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800739 def parse_single_deps(self,
740 content,
741 parent_vars=None,
742 parent_path='',
743 parent_dep=None):
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800744 """Parses DEPS file without recursion.
745
746 Args:
747 content: file content of DEPS file
748 parent_vars: variables inherent from parent DEPS
749 parent_path: project path of parent DEPS file
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800750 parent_dep: A corresponding Dep object in parent DEPS
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800751
752 Returns:
753 Deps object
754 """
755
Kuang-che Wu067ff292019-02-14 18:16:23 +0800756 local_scope = self.load_single_deps(content)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800757 deps = Deps()
Kuang-che Wu067ff292019-02-14 18:16:23 +0800758
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800759 local_scope.setdefault('vars', {})
760 if parent_vars:
761 local_scope['vars'].update(parent_vars)
762 deps.variables = local_scope['vars']
763
764 # Warnings for old usages which we don't support.
765 for name in deps.variables:
766 if name.startswith('RECURSEDEPS_') or name.endswith('_DEPS_file'):
767 logger.warning('%s is deprecated and not supported recursion syntax',
768 name)
769 if 'deps_os' in local_scope:
770 logger.warning('deps_os is no longer supported')
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800771
772 if 'allowed_hosts' in local_scope:
773 deps.allowed_hosts = set(local_scope.get('allowed_hosts'))
774 deps.hooks = local_scope.get('hooks', [])
775 deps.pre_deps_hooks = local_scope.get('pre_deps_hooks', [])
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800776 deps.gn_args_from = local_scope.get('gclient_gn_args_from')
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800777 deps.gn_args_file = local_scope.get('gclient_gn_args_file')
778 deps.gn_args = local_scope.get('gclient_gn_args', [])
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800779
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800780 # recalculate hook path
781 use_relative_hooks = local_scope.get('use_relative_hooks', False)
782 if use_relative_hooks:
783 assert local_scope.get('use_relative_paths', False)
784 for hook in deps.hooks:
785 hook['cwd'] = os.path.join(parent_path, hook.get('cwd', ''))
786 for pre_deps_hook in deps.pre_deps_hooks:
787 pre_deps_hook['cwd'] = os.path.join(parent_path,
788 pre_deps_hook.get('cwd', ''))
789
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800790 for path, dep_entry in local_scope['deps'].items():
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800791 # recalculate path
Kuang-che Wu058ac042019-03-18 14:14:53 +0800792 path = path.format(**deps.variables)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800793 if local_scope.get('use_relative_paths', False):
794 path = os.path.join(parent_path, path)
Kuang-che Wu058ac042019-03-18 14:14:53 +0800795 path = os.path.normpath(path)
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800796
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800797 dep = Dep(path, deps.variables, dep_entry)
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800798 eval_condition = dep.eval_condition()
799
800 # update condition
801 if parent_dep and parent_dep.condition:
802 tmp_dict = {'condition': dep.condition}
803 gclient_eval.UpdateCondition(tmp_dict, 'and', parent_dep.condition)
804 dep.condition = tmp_dict['condition']
805
806 if not eval_condition:
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800807 deps.ignored_entries[path] = dep
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800808 continue
809
810 # TODO(kcwu): support dep_type=cipd http://crbug.com/846564
811 if dep.dep_type != 'git':
Kuang-che Wuced2dbf2019-01-30 23:13:24 +0800812 warning_key = ('dep_type', dep.dep_type, path)
813 if warning_key not in emitted_warnings:
814 emitted_warnings.add(warning_key)
815 logger.warning('dep_type=%s is not supported yet: %s', dep.dep_type,
816 path)
Zheng-Jie Chang7ab92082020-05-29 19:39:59 +0800817 deps.ignored_entries[path] = dep
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800818 continue
819
820 deps.entries[path] = dep
821
822 recursedeps = []
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800823 for recurse_entry in local_scope.get('recursedeps', []):
824 # Normalize entries.
825 if isinstance(recurse_entry, tuple):
826 path, deps_file = recurse_entry
827 else:
828 assert isinstance(path, str)
829 path, deps_file = recurse_entry, 'DEPS'
830
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800831 if local_scope.get('use_relative_paths', False):
832 path = os.path.join(parent_path, path)
833 path = path.format(**deps.variables)
834 if path in deps.entries:
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800835 recursedeps.append((path, deps_file))
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800836 deps.recursedeps = recursedeps
837
838 return deps
839
840 def construct_deps_tree(self,
841 tstree,
842 repo_url,
843 at,
844 after,
845 before,
846 parent_vars=None,
847 parent_path='',
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800848 parent_dep=None,
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800849 deps_file='DEPS'):
850 """Processes DEPS recursively of given time period.
851
852 This method parses all commits of DEPS between time `after` and `before`,
853 segments recursive dependencies into subtrees if they are changed, and
854 processes subtrees recursively.
855
856 The parsed results (multiple revisions of DEPS file) are stored in `tstree`.
857
858 Args:
859 tstree: TimeSeriesTree object
860 repo_url: remote repo url
861 at: branch or git commit id
862 after: begin of period
863 before: end of period
864 parent_vars: DEPS variables inherit from parent DEPS (including
865 custom_vars)
866 parent_path: the path of parent project of current DEPS file
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800867 parent_dep: A corresponding Dep object in parent DEPS
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800868 deps_file: filename of DEPS file, relative to the git repo, repo_rul
869 """
870 if '://' in repo_url:
871 git_repo = self.code_storage.cached_git_root(repo_url)
Kuang-che Wu1e49f512018-12-06 15:27:42 +0800872 if not os.path.exists(git_repo):
873 with locking.lock_file(
874 os.path.join(self.code_storage.cache_dir,
875 locking.LOCK_FILE_FOR_MIRROR_SYNC)):
876 mirror(self.code_storage, repo_url)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800877 else:
878 git_repo = repo_url
879
880 if git_util.is_git_rev(at):
881 history = [
882 (after, at),
883 (before, at),
884 ]
885 else:
886 history = git_util.get_history(
887 git_repo,
888 deps_file,
889 branch=at,
890 after=after,
891 before=before,
Zheng-Jie Chang313eec32020-02-18 16:17:07 +0800892 padding_begin=True,
893 padding_end=True)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800894 assert history
895
896 # If not equal, it means the file was deleted but is still referenced by
897 # its parent.
898 assert history[-1][0] == before
899
900 # TODO(kcwu): optimization: history[-1] is unused
901 for timestamp, git_rev in history[:-1]:
902 content = git_util.get_file_from_revision(git_repo, git_rev, deps_file)
903
904 deps = self.parse_single_deps(
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800905 content,
906 parent_vars=parent_vars,
907 parent_path=parent_path,
908 parent_dep=parent_dep)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800909 tstree.add_snapshot(timestamp, deps, deps.recursedeps)
910
Kuang-che Wua7ddf9b2019-11-25 18:59:57 +0800911 tstree.no_more_snapshot()
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800912
913 for subtree in tstree.subtrees:
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800914 path, deps_file = subtree.entry
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800915 path_spec = subtree.parent_deps.entries[path].as_path_spec()
916 self.construct_deps_tree(
917 subtree,
918 path_spec.repo_url,
919 path_spec.at,
920 subtree.start_time,
921 subtree.end_time,
922 parent_vars=subtree.parent_deps.variables,
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800923 parent_path=path,
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800924 parent_dep=subtree.parent_deps.entries[path],
Kuang-che Wu3a7352d2018-10-20 17:22:03 +0800925 deps_file=deps_file)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800926
927 def enumerate_path_specs(self, start_time, end_time, path):
928 tstree = TimeSeriesTree(None, path, start_time, end_time)
Kuang-che Wu88e96312020-10-20 16:21:11 +0800929 self.construct_deps_tree(tstree, path, DEFAULT_BRANCH_NAME, start_time,
930 end_time)
Kuang-che Wu8a28a9d2018-09-11 17:43:36 +0800931 return tstree.iter_path_specs()
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800932
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800933 def enumerate_gclient_solutions(self, start_time, end_time, path):
934 tstree = TimeSeriesTree(None, path, start_time, end_time)
Kuang-che Wu88e96312020-10-20 16:21:11 +0800935 self.construct_deps_tree(tstree, path, DEFAULT_BRANCH_NAME, start_time,
936 end_time)
Zheng-Jie Chang8b8a4822020-06-24 13:24:02 +0800937 return tstree.iter_forest()
938
939 def flatten(self, solutions, entry_point):
940 """Flatten all given Deps
941
942 Args:
943 solutions: A name => Deps dict, name can be either a str or a tuple.
944 entry_point (str): An entry_point name of solutions.
945
946 Returns:
947 Deps: A flatten Deps.
948 """
949
950 def _add_unvisited_recursedeps(deps_queue, visited, deps):
951 for name in deps.recursedeps:
952 if name not in visited:
953 visited.add(name)
954 deps_queue.put(name)
955
956 result = solutions[entry_point]
957 deps_queue = queue.SimpleQueue()
958 visited = set()
959 visited.add(entry_point)
960 _add_unvisited_recursedeps(deps_queue, visited, solutions[entry_point])
961
962 # BFS to merge `deps` into `result`
963 while not deps_queue.empty():
964 deps_name = deps_queue.get()
965 deps = solutions[deps_name]
966
967 result.allowed_hosts.update(deps.allowed_hosts)
968 for key, value in deps.variables.items():
969 assert key not in result.variables or deps.variables[key] == value
970 result.variables[key] = value
971 result.pre_deps_hooks += deps.pre_deps_hooks
972 result.hooks += deps.hooks
973
974 for dep in deps.entries.values():
975 assert dep.path not in result.entries or result.entries.get(
976 dep.path) == dep
977 result.entries[dep.path] = dep
978
979 for dep in deps.ignored_entries.values():
980 assert (dep.path not in result.ignored_entries or
981 result.ignored_entries.get(dep.path) == dep)
982 result.ignored_entries[dep.path] = dep
983
984 _add_unvisited_recursedeps(deps_queue, visited, deps)
985
986 # If gn_args_from is set in root DEPS, overwrite gn arguments
987 if solutions[entry_point].gn_args_from:
988 gn_args_dep = solutions[(solutions[entry_point].gn_args_from, 'DEPS')]
989 result.gn_args = gn_args_dep.gn_args
990 result.gn_args_file = gn_args_dep.gn_args_file
991
992 return result
993
Kuang-che Wu3eb6b502018-06-06 16:15:18 +0800994
995class GclientCache(codechange.CodeStorage):
996 """Gclient git cache."""
997
998 def __init__(self, cache_dir):
999 self.cache_dir = cache_dir
1000
1001 def _url_to_cache_dir(self, url):
1002 # ref: depot_tools' git_cache.Mirror.UrlToCacheDir
Kuang-che Wua7ddf9b2019-11-25 18:59:57 +08001003 parsed = urllib.parse.urlparse(url)
Kuang-che Wu3eb6b502018-06-06 16:15:18 +08001004 norm_url = parsed.netloc + parsed.path
1005 if norm_url.endswith('.git'):
1006 norm_url = norm_url[:-len('.git')]
Kuang-che Wu5fef2912019-04-15 16:18:29 +08001007 norm_url = norm_url.replace('googlesource.com/a/', 'googlesource.com/')
Kuang-che Wu3eb6b502018-06-06 16:15:18 +08001008 return norm_url.replace('-', '--').replace('/', '-').lower()
1009
1010 def cached_git_root(self, repo_url):
1011 cache_path = self._url_to_cache_dir(repo_url)
1012 return os.path.join(self.cache_dir, cache_path)
Kuang-che Wu6948ecc2018-09-11 17:43:49 +08001013
Kuang-che Wu6948ecc2018-09-11 17:43:49 +08001014 def add_to_project_list(self, project_root, path, repo_url):
Kuang-che Wu067ff292019-02-14 18:16:23 +08001015 projects = load_gclient_entries(project_root)
Kuang-che Wu6948ecc2018-09-11 17:43:49 +08001016
1017 projects[path] = repo_url
1018
Kuang-che Wu067ff292019-02-14 18:16:23 +08001019 write_gclient_entries(project_root, projects)
Kuang-che Wu6948ecc2018-09-11 17:43:49 +08001020
1021 def remove_from_project_list(self, project_root, path):
Kuang-che Wu067ff292019-02-14 18:16:23 +08001022 projects = load_gclient_entries(project_root)
Kuang-che Wu6948ecc2018-09-11 17:43:49 +08001023
1024 if path in projects:
1025 del projects[path]
1026
Kuang-che Wu067ff292019-02-14 18:16:23 +08001027 write_gclient_entries(project_root, projects)