blob: a913484ad1048b5cbb524a9b63bd1dfd94054b5d [file] [log] [blame]
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -07001# Copyright 2014 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Install debug symbols for specified packages.
6
7Only reinstall the debug symbols if they are not already installed to save time.
8
9The debug symbols are packaged outside of the prebuilt package in a
10.debug.tbz2 archive when FEATURES=separatedebug is set (by default on
11builders). On local machines, separatedebug is not set and the debug symbols
12are part of the prebuilt package.
13"""
14
15from __future__ import print_function
16
17import argparse
18import logging
19import os
20import pickle
21import sys
22import tempfile
Bertrand SIMONNET8e537282014-12-04 15:04:42 -080023import urlparse
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -070024
25from chromite.lib import binpkg
26from chromite.lib import cache
27from chromite.lib import commandline
28from chromite.lib import cros_build_lib
29from chromite.lib import osutils
30from chromite.lib import parallel
31from chromite.lib import gs
32from portage import create_trees
33
34
35DEBUG_SYMS_EXT = '.debug.tbz2'
36
37
Bertrand SIMONNET1e146e52014-12-11 14:11:56 -080038# We cache the package indexes. When the format of what we store changes,
39# bump the cache version to avoid problems.
40CACHE_VERSION = '1'
41
42
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -070043class DebugSymbolsInstaller(object):
44 """Container for enviromnent objects, needed to make multiprocessing work.
45
46 This also redirects stdout to null when stdout_to_null=True to avoid
47 polluting the output with portage QA warnings.
48 """
49 _old_stdout = None
50 _null = None
51
52 def __init__(self, vartree, gs_context, sysroot, stdout_to_null):
53 self._vartree = vartree
54 self._gs_context = gs_context
55 self._sysroot = sysroot
56 self._stdout_to_null = stdout_to_null
57
58 def __enter__(self):
59 if self._stdout_to_null:
60 self._old_stdout = sys.stdout
61 self._null = open(os.devnull, 'w')
62 sys.stdout = self._null
63 return self
64
65 def __exit__(self, _exc_type, _exc_val, _exc_tb):
66 if self._stdout_to_null:
67 sys.stdout = self._old_stdout
68 self._null.close()
69
70 def Install(self, cpv, url):
71 """Install the debug symbols for |cpv|.
72
73 This will install the debug symbols tarball in PKGDIR so that it can be
74 used later.
75
76 Args:
77 cpv: the cpv of the package to build. This assumes that the cpv is
78 installed in the sysroot.
79 url: url of the debug symbols archive. This could be a Google Storage url
80 or a local path.
81 """
82 archive = os.path.join(self._vartree.settings['PKGDIR'],
83 cpv + DEBUG_SYMS_EXT)
Bertrand SIMONNET1e146e52014-12-11 14:11:56 -080084 # GsContext does not understand file:// scheme so we need to extract the
85 # path ourselves.
86 parsed_url = urlparse.urlsplit(url)
87 if not parsed_url.scheme or parsed_url.scheme == 'file':
88 url = parsed_url.path
89
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -070090 if not os.path.isfile(archive):
91 self._gs_context.Copy(url, archive, debug_level=logging.DEBUG)
92
93 with osutils.TempDir(sudo_rm=True) as tempdir:
94 cros_build_lib.SudoRunCommand(['tar', '-I', 'bzip2 -q', '-xf', archive,
95 '-C', tempdir], quiet=True)
96
97 with open(self._vartree.getpath(cpv, filename='CONTENTS'),
98 'a') as content_file:
99 # Merge the content of the temporary dir into the sysroot.
100 # pylint: disable=protected-access
101 link = self._vartree.dbapi._dblink(cpv)
102 link.mergeme(tempdir, self._sysroot, content_file, None, '', {}, None)
103
104
105def ParseArgs(argv):
106 """Parse arguments and initialize field.
107
108 Args:
109 argv: arguments passed to the script.
110 """
111 parser = commandline.ArgumentParser(description=__doc__)
112 parser.add_argument('--board', help='Board name (required).', required=True)
113 parser.add_argument('--all', dest='all', action='store_true',
114 help='Install the debug symbols for all installed '
115 'packages', default=False)
116 parser.add_argument('packages', nargs=argparse.REMAINDER,
117 help='list of packages that need the debug symbols.')
118
119 advanced = parser.add_argument_group('Advanced options')
120 advanced.add_argument('--nocachebinhost', dest='cachebinhost', default=True,
121 action='store_false', help="Don't cache the list of"
122 " files contained in binhosts. (Default: cache)")
123 advanced.add_argument('--clearcache', dest='clearcache', action='store_true',
124 default=False, help='Clear the binhost cache.')
125 advanced.add_argument('--jobs', default=None, type=int,
126 help='Number of processes to run in parallel.')
127
128 options = parser.parse_args(argv)
129 options.Freeze()
130
131 if options.all and options.packages:
132 cros_build_lib.Die('Cannot use --all with a list of packages')
133 return options
134
135
136def ShouldGetSymbols(cpv, vardb, remote_symbols):
137 """Return True if the symbols for cpv are available and are not installed.
138
139 We try to check if the symbols are installed before checking availability as
140 a GS request is more expensive than checking locally.
141
142 Args:
143 cpv: cpv of the package
144 vardb: a vartree dbapi
145 remote_symbols: a mapping from cpv to debug symbols url
146
147 Returns:
148 True if |cpv|'s debug symbols are not installed and are available
149 """
150 features, contents = vardb.aux_get(cpv, ['FEATURES', 'CONTENTS'])
151
152 return ('separatedebug' in features and not '/usr/lib/debug/' in contents
153 and cpv in remote_symbols)
154
155
156def RemoteSymbols(vartree, binhost_cache=None):
157 """Get the cpv to debug symbols mapping.
158
159 If several binhost contain debug symbols for the same cpv, keep only the
160 highest priority one.
161
162 Args:
163 vartree: a vartree
164 binhost_cache: a cache containing the cpv to debug symbols url for all
165 known binhosts. None if we are not caching binhosts.
166
167 Returns:
168 a dictionary mapping the cpv to a remote debug symbols gsurl.
169 """
170 symbols_mapping = {}
171 for binhost in vartree.settings['PORTAGE_BINHOST'].split():
172 if binhost:
173 symbols_mapping.update(ListBinhost(binhost, binhost_cache))
174 return symbols_mapping
175
176
Bertrand SIMONNET1e146e52014-12-11 14:11:56 -0800177def GetPackageIndex(binhost, binhost_cache=None):
178 """Get the packages index for |binhost|.
179
180 If a cache is provided, use it to a cache remote packages index.
181
182 Args:
183 binhost: a portage binhost, local, google storage or http.
184 binhost_cache: a cache for the remote packages index.
185
186 Returns:
187 A PackageIndex object.
188 """
189 key = binhost.split('://')[-1]
190 key = key.rstrip('/').split('/')
191
192 if binhost_cache and binhost_cache.Lookup(key).Exists():
193 with open(binhost_cache.Lookup(key).path) as f:
194 return pickle.load(f)
195
196 pkgindex = binpkg.GrabRemotePackageIndex(binhost)
197 if pkgindex and binhost_cache:
198 # Only cache remote binhosts as local binhosts can change.
199 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
200 pickle.dump(pkgindex, temp_file)
201 temp_file.file.close()
202 binhost_cache.Lookup(key).Assign(temp_file.name)
203 elif pkgindex is None:
204 binhost = urlparse.urlsplit(binhost).path
205 if not os.path.isdir(binhost):
206 raise ValueError('unrecognized binhost format for %s.')
207 pkgindex = binpkg.GrabLocalPackageIndex(binhost)
208
209 return pkgindex
210
211
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -0700212def ListBinhost(binhost, binhost_cache=None):
213 """Return the cpv to debug symbols mapping for a given binhost.
214
215 List the content of the binhost to extract the cpv to debug symbols
216 mapping. If --cachebinhost is set, we cache the result to avoid the
217 cost of gsutil every time.
218
219 Args:
220 binhost: a portage binhost, local or on google storage.
221 binhost_cache: a cache containing mappings cpv to debug symbols url for a
222 given binhost (None if we don't want to cache).
223
224 Returns:
225 A cpv to debug symbols url mapping.
226 """
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -0700227
228 symbols = {}
Bertrand SIMONNET1e146e52014-12-11 14:11:56 -0800229 pkgindex = GetPackageIndex(binhost, binhost_cache)
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -0700230 for p in pkgindex.packages:
231 if p.get('DEBUG_SYMBOLS') == 'yes':
Bertrand SIMONNET1e146e52014-12-11 14:11:56 -0800232 path = p.get('PATH', p['CPV'] + '.tbz2')
233 base_url = pkgindex.header.get('URI', binhost)
234 symbols[p['CPV']] = os.path.join(base_url,
235 path.replace('.tbz2', DEBUG_SYMS_EXT))
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -0700236
237 return symbols
238
239
240def GetMatchingCPV(package, vardb):
241 """Return the cpv of the installed package matching |package|.
242
243 Args:
244 package: package name
245 vardb: a vartree dbapi
246
247 Returns:
248 The cpv of the installed package whose name matchex |package|.
249 """
250 matches = vardb.match(package)
251 if not matches:
252 cros_build_lib.Die('Could not find package %s' % package)
253 if len(matches) != 1:
254 cros_build_lib.Die('Ambiguous package name: %s.\n'
255 'Matching: %s' % (package, ' '.join(matches)))
256 return matches[0]
257
258
259def main(argv):
260 options = ParseArgs(argv)
261
262 cros_build_lib.AssertInsideChroot()
263 if os.geteuid() != 0:
264 cros_build_lib.Die('This script must be ran as root.')
265
266 # sysroot must have a trailing / as the tree dictionary produced by
267 # create_trees in indexed with a trailing /.
268 sysroot = cros_build_lib.GetSysroot(options.board) + '/'
269 trees = create_trees(target_root=sysroot, config_root=sysroot)
270
271 vartree = trees[sysroot]['vartree']
272
273 cache_dir = os.path.join(commandline.BaseParser.FindCacheDir(None, None),
Bertrand SIMONNET1e146e52014-12-11 14:11:56 -0800274 'cros_install_debug_syms-v' + CACHE_VERSION)
Bertrand SIMONNET6b6a1312014-10-29 18:37:51 -0700275
276 if options.clearcache:
277 osutils.RmDir(cache_dir, ignore_missing=True)
278
279 binhost_cache = None
280 if options.cachebinhost:
281 binhost_cache = cache.DiskCache(cache_dir)
282
283 gs_context = gs.GSContext()
284 symbols_mapping = RemoteSymbols(vartree, binhost_cache)
285
286 if options.all:
287 to_install = vartree.dbapi.cpv_all()
288 else:
289 to_install = [GetMatchingCPV(p, vartree.dbapi) for p in options.packages]
290
291 to_install = [p for p in to_install
292 if ShouldGetSymbols(p, vartree.dbapi, symbols_mapping)]
293
294 if not to_install:
295 cros_build_lib.Info('nothing to do, exit')
296 return
297
298 with DebugSymbolsInstaller(vartree, gs_context, sysroot,
299 not options.debug) as installer:
300 args = [(p, symbols_mapping[p]) for p in to_install]
301 parallel.RunTasksInProcessPool(installer.Install, args,
302 processes=options.jobs)
303
304 cros_build_lib.Debug('installation done, updating packages index file')
305 packages_dir = os.path.join(sysroot, 'packages')
306 packages_file = os.path.join(packages_dir, 'Packages')
307 # binpkg will set DEBUG_SYMBOLS automatically if it detects the debug symbols
308 # in the packages dir.
309 pkgindex = binpkg.GrabLocalPackageIndex(packages_dir)
310 with open(packages_file, 'w') as p:
311 pkgindex.Write(p)