Mike Frysinger | f1ba7ad | 2022-09-12 05:42:57 -0400 | [diff] [blame] | 1 | # Copyright 2013 The ChromiumOS Authors |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Generate minidump symbols for use by the Crash server. |
| 6 | |
| 7 | Note: This should be run inside the chroot. |
| 8 | |
| 9 | This produces files in the breakpad format required by minidump_stackwalk and |
| 10 | the crash server to dump stack information. |
| 11 | |
| 12 | Basically it scans all the split .debug files in /build/$BOARD/usr/lib/debug/ |
| 13 | and converts them over using the `dump_syms` programs. Those plain text .sym |
| 14 | files are then stored in /build/$BOARD/usr/lib/debug/breakpad/. |
| 15 | |
Mike Frysinger | 02e1e07 | 2013-11-10 22:11:34 -0500 | [diff] [blame] | 16 | If you want to actually upload things, see upload_symbols.py. |
| 17 | """ |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 18 | |
| 19 | import collections |
| 20 | import ctypes |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 21 | import enum |
Chris McDonald | b55b703 | 2021-06-17 16:41:32 -0600 | [diff] [blame] | 22 | import logging |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 23 | import multiprocessing |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 24 | import multiprocessing.sharedctypes |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 25 | import os |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 26 | import re |
| 27 | from typing import Optional |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 28 | |
Chris McDonald | b55b703 | 2021-06-17 16:41:32 -0600 | [diff] [blame] | 29 | from chromite.cbuildbot import cbuildbot_alerts |
Mike Frysinger | 06a51c8 | 2021-04-06 11:39:17 -0400 | [diff] [blame] | 30 | from chromite.lib import build_target_lib |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 31 | from chromite.lib import commandline |
| 32 | from chromite.lib import cros_build_lib |
| 33 | from chromite.lib import osutils |
| 34 | from chromite.lib import parallel |
Mike Frysinger | 96ad3f2 | 2014-04-24 23:27:27 -0400 | [diff] [blame] | 35 | from chromite.lib import signals |
Alex Klein | 1809f57 | 2021-09-09 11:28:37 -0600 | [diff] [blame] | 36 | from chromite.utils import file_util |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 37 | |
Mike Frysinger | 807d828 | 2022-04-28 22:45:17 -0400 | [diff] [blame] | 38 | |
Stephen Boyd | fc1c803 | 2021-10-06 20:58:37 -0700 | [diff] [blame] | 39 | # Elf files that don't exist but have a split .debug file installed. |
| 40 | ALLOWED_DEBUG_ONLY_FILES = { |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 41 | "boot/vmlinux", |
Stephen Boyd | fc1c803 | 2021-10-06 20:58:37 -0700 | [diff] [blame] | 42 | } |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 43 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 44 | # Allowlist of elf files that we know we can't symbolize in the normal way, but |
| 45 | # which we don't have an automatic way to detect. |
| 46 | EXPECTED_POOR_SYMBOLIZATION_FILES = ALLOWED_DEBUG_ONLY_FILES | { |
| 47 | # Git binaries are downloaded as binary blobs already stripped. |
| 48 | "usr/bin/git", |
| 49 | "usr/bin/git-receive-pack", |
| 50 | "usr/bin/git-upload-archive", |
| 51 | "usr/bin/git-upload-pack", |
| 52 | # Prebuild Android binary |
| 53 | "build/rootfs/opt/google/vms/android/etc/bin/XkbToKcmConverter", |
| 54 | # Pulled from |
| 55 | # https://skia.googlesource.com/buildbot/+/refs/heads/main/gold-client/, no |
| 56 | # need to resymbolize. |
| 57 | "usr/bin/goldctl", |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 58 | # TODO(b/273620044): The following 44 binaries are from |
| 59 | # app-benchmarks/lmbench; they hardcode some compiler options which prevent |
| 60 | # us from generating STACK records. Fix the package. |
| 61 | "usr/bin/lat_unix", |
| 62 | "usr/bin/lat_select", |
| 63 | "usr/bin/line.lmbench", |
| 64 | "usr/bin/lat_rpc", |
| 65 | "usr/bin/hello", |
| 66 | "usr/bin/lmhttp", |
| 67 | "usr/bin/lat_syscall", |
| 68 | "usr/bin/par_mem", |
| 69 | "usr/bin/lat_http", |
| 70 | "usr/bin/lat_proc", |
| 71 | "usr/bin/stream.lmbench", |
| 72 | "usr/bin/enough", |
| 73 | "usr/bin/lat_mem_rd", |
| 74 | "usr/bin/mhz", |
| 75 | "usr/bin/lat_fs", |
| 76 | "usr/bin/lat_sem", |
| 77 | "usr/bin/lmdd", |
| 78 | "usr/bin/lat_fcntl", |
| 79 | "usr/bin/lat_pipe", |
| 80 | "usr/bin/lat_mmap", |
| 81 | "usr/bin/bw_file_rd", |
| 82 | "usr/bin/bw_unix", |
| 83 | "usr/bin/lat_tcp", |
| 84 | "usr/bin/flushdisk", |
| 85 | "usr/bin/bw_pipe", |
| 86 | "usr/bin/msleep", |
| 87 | "usr/bin/bw_mmap_rd", |
| 88 | "usr/bin/bw_tcp", |
| 89 | "usr/bin/lat_ops", |
| 90 | "usr/bin/loop_o", |
| 91 | "usr/bin/bw_mem", |
| 92 | "usr/bin/lat_pagefault", |
| 93 | "usr/bin/lat_connect", |
| 94 | "usr/bin/timing_o", |
| 95 | "usr/bin/lat_ctx", |
| 96 | "usr/bin/tlb", |
| 97 | "usr/bin/lat_fifo", |
| 98 | "usr/bin/disk", |
| 99 | "usr/bin/lat_unix_connect", |
| 100 | "usr/bin/par_ops", |
| 101 | "usr/bin/lat_sig", |
| 102 | "usr/bin/lat_udp", |
| 103 | "usr/bin/lat_ops", |
| 104 | "usr/bin/memsize", |
| 105 | # This is complete for --board=eve and --board=kevin |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 106 | # TODO(b/241470012): Complete for other boards. |
| 107 | } |
| 108 | |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 109 | # Allowlist of patterns for ELF files that symbolize (dump_syms exits with |
| 110 | # success) but don't pass symbol file validation. Note that ELFs listed in |
| 111 | # EXPECTED_POOR_SYMBOLIZATION_FILES do not have their symbol files validated and |
| 112 | # do not need to be repeated here. |
| 113 | ALLOWLIST_NO_SYMBOL_FILE_VALIDATION = { |
| 114 | # Built in a weird way, see comments at top of |
| 115 | # https://source.chromium.org/chromium/chromium/src/+/main:native_client/src/trusted/service_runtime/linux/nacl_bootstrap.x |
| 116 | "opt/google/chrome/nacl_helper_bootstrap", |
| 117 | # TODO(b/273543528): Investigate why this doesn't have stack records on |
| 118 | # kevin builds. |
| 119 | "build/rootfs/dlc-scaled/screen-ai/package/root/libchromescreenai.so", |
| 120 | } |
| 121 | # Same but patterns not exact paths. |
| 122 | ALLOWLIST_NO_SYMBOL_FILE_VALIDATION_RE = tuple( |
| 123 | re.compile(x) |
| 124 | for x in ( |
| 125 | # Only has code if use.camera_feature_effects. Otherwise will have no |
| 126 | # STACK records. |
| 127 | r"usr/lib[^/]*/libcros_ml_core\.so", |
| 128 | # Prebuilt closed-source library. |
| 129 | r"usr/lib[^/]*/python[0-9]\.[0-9]/site-packages/.*/x_ignore_nofocus.so", |
| 130 | # b/273577373: Rarely used, and only by few programs that do |
| 131 | # non-standard encoding conversions |
| 132 | r"lib[^/]*/libnss_files\.so\.[0-9.]+", |
| 133 | r"lib[^/]*/libnss_dns\.so\.[0-9.]+", |
| 134 | r"usr/lib[^/]*/gconv/libISOIR165\.so", |
| 135 | r"usr/lib[^/]*/gconv/libGB\.so", |
| 136 | r"usr/lib[^/]*/gconv/libKSC\.so", |
| 137 | r"usr/lib[^/]*/gconv/libCNS\.so", |
| 138 | r"usr/lib[^/]*/gconv/libJISX0213\.so", |
| 139 | r"usr/lib[^/]*/gconv/libJIS\.so", |
| 140 | # TODO(b/273579075): Figure out why libcares.so is not getting STACK |
| 141 | # records on kevin. |
| 142 | r"usr/lib[^/]*/libcares\.so[0-9.]*", |
| 143 | # libevent-2.1.so.7.0.1 does not have executable code and thus no |
| 144 | # STACK records. See libevent-2.1.12-libevent-shrink.patch. |
| 145 | r"usr/lib[^/]*/libevent-[0-9.]+\.so[0-9.]*", |
| 146 | # TODO(b/273599604): Figure out why libdcerpc-samr.so.0.0.1 is not |
| 147 | # getting STACK records on kevin. |
| 148 | r"usr/lib[^/]*/libdcerpc-samr\.so[0-9.]*", |
| 149 | # TODO(b/272613635): Figure out why these libabsl shared libraries are |
| 150 | # not getting STACK records. |
| 151 | r"usr/lib[^/]*/libabsl_bad_variant_access\.so\.[0-9.]+", |
| 152 | r"usr/lib[^/]*/libabsl_random_internal_platform\.so\.[0-9.]+", |
| 153 | r"usr/lib[^/]*/libabsl_flags\.so\.[0-9.]+", |
| 154 | r"usr/lib[^/]*/libabsl_bad_any_cast_impl\.so\.[0-9.]+", |
| 155 | r"usr/lib[^/]*/libabsl_bad_optional_access\.so\.[0-9.]+", |
| 156 | # TODO(b/273607289): Figure out why libgrpc++_error_details.so.1.43.0 is |
| 157 | # not getting STACK records on kevin. |
| 158 | r"usr/lib[^/]*/libgrpc\+\+_error_details\.so\.[0-9.]+", |
| 159 | ) |
| 160 | ) |
| 161 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 162 | SymbolHeader = collections.namedtuple( |
| 163 | "SymbolHeader", |
| 164 | ( |
| 165 | "cpu", |
| 166 | "id", |
| 167 | "name", |
| 168 | "os", |
| 169 | ), |
| 170 | ) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 171 | |
| 172 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 173 | class SymbolGenerationResult(enum.Enum): |
| 174 | """Result of running dump_syms |
| 175 | |
| 176 | Return value of _DumpAllowingBasicFallback() and _DumpExpectingSymbols(). |
| 177 | """ |
| 178 | |
| 179 | SUCCESS = 1 |
| 180 | UNEXPECTED_FAILURE = 2 |
| 181 | EXPECTED_FAILURE = 3 |
| 182 | |
| 183 | |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 184 | class ExpectedFiles(enum.Enum): |
| 185 | """The files always expect to see dump_syms run on. |
| 186 | |
| 187 | We do extra validation on a few, semi-randomly chosen files. If we do not |
| 188 | create symbol files for these ELFs, something is very wrong. |
| 189 | """ |
| 190 | |
| 191 | ASH_CHROME = enum.auto() |
| 192 | LIBC = enum.auto() |
| 193 | CRASH_REPORTER = enum.auto() |
| 194 | LIBMETRICS = enum.auto() |
| 195 | |
| 196 | |
| 197 | ALL_EXPECTED_FILES = frozenset( |
| 198 | ( |
| 199 | ExpectedFiles.ASH_CHROME, |
| 200 | ExpectedFiles.LIBC, |
| 201 | ExpectedFiles.CRASH_REPORTER, |
| 202 | ExpectedFiles.LIBMETRICS, |
| 203 | ) |
| 204 | ) |
| 205 | |
| 206 | |
| 207 | class SymbolFileLineCounts: |
| 208 | """Counts of the various types of lines in a .sym file""" |
| 209 | |
| 210 | LINE_NUMBER_REGEX = re.compile(r"^([0-9a-f]+)") |
| 211 | |
| 212 | def __init__(self, sym_file: str, elf_file: str): |
| 213 | # https://chromium.googlesource.com/breakpad/breakpad/+/HEAD/docs/symbol_files.md |
| 214 | # explains what these line types are. |
| 215 | self.module_lines = 0 |
| 216 | self.file_lines = 0 |
| 217 | self.inline_origin_lines = 0 |
| 218 | self.func_lines = 0 |
| 219 | self.inline_lines = 0 |
| 220 | self.line_number_lines = 0 |
| 221 | self.public_lines = 0 |
| 222 | self.stack_lines = 0 |
| 223 | # Not listed in the documentation but still present. |
| 224 | self.info_lines = 0 |
| 225 | |
| 226 | with open(sym_file, mode="r", encoding="utf-8") as f: |
| 227 | for line in f: |
| 228 | words = line.split() |
| 229 | expected_words_max = None |
| 230 | if not words: |
| 231 | raise ValueError( |
| 232 | f"{elf_file}: symbol file has unexpected blank line" |
| 233 | ) |
| 234 | |
| 235 | line_type = words[0] |
| 236 | if line_type == "MODULE": |
| 237 | self.module_lines += 1 |
| 238 | expected_words_min = 5 |
| 239 | expected_words_max = 5 |
| 240 | elif line_type == "FILE": |
| 241 | self.file_lines += 1 |
| 242 | expected_words_min = 3 |
| 243 | # No max, filenames can have spaces. |
| 244 | elif line_type == "INLINE_ORIGIN": |
| 245 | self.inline_origin_lines += 1 |
| 246 | expected_words_min = 3 |
| 247 | # No max, function parameter lists can have spaces. |
| 248 | elif line_type == "FUNC": |
| 249 | self.func_lines += 1 |
| 250 | expected_words_min = 5 |
| 251 | # No max, function parameter lists can have spaces. |
| 252 | elif line_type == "INLINE": |
| 253 | self.inline_lines += 1 |
| 254 | expected_words_min = 5 |
| 255 | # No max, INLINE can have multiple address pairs. |
| 256 | elif SymbolFileLineCounts.LINE_NUMBER_REGEX.match(line_type): |
| 257 | self.line_number_lines += 1 |
| 258 | expected_words_min = 4 |
| 259 | expected_words_max = 4 |
| 260 | line_type = "line number" |
| 261 | elif line_type == "PUBLIC": |
| 262 | self.public_lines += 1 |
| 263 | expected_words_min = 4 |
| 264 | if os.path.basename(elf_file).startswith("libc.so"): |
| 265 | # TODO(b/251003272): libc.so sometimes produces PUBLIC |
| 266 | # records with no symbol name. This is an error but is |
| 267 | # not affecting our ability to decode stacks. |
| 268 | expected_words_min = 3 |
| 269 | # No max, function parameter lists can have spaces. |
| 270 | elif line_type == "STACK": |
| 271 | self.stack_lines += 1 |
| 272 | expected_words_min = 5 |
| 273 | # No max, expressions can be complex. |
| 274 | elif line_type == "INFO": |
| 275 | self.info_lines += 1 |
| 276 | # Not documented, so unclear what the min & max are |
| 277 | expected_words_min = None |
| 278 | else: |
| 279 | raise ValueError( |
| 280 | f"{elf_file}: symbol file has unknown line type " |
| 281 | f"{line_type}" |
| 282 | ) |
| 283 | |
| 284 | if expected_words_max is not None: |
| 285 | if not ( |
| 286 | expected_words_min <= len(words) <= expected_words_max |
| 287 | ): |
| 288 | raise ValueError( |
| 289 | f"{elf_file}: symbol file has {line_type} line " |
| 290 | f"with {len(words)} words (expected " |
| 291 | f"{expected_words_min} - {expected_words_max})" |
| 292 | ) |
| 293 | elif expected_words_min is not None: |
| 294 | if len(words) < expected_words_min: |
| 295 | raise ValueError( |
| 296 | f"{elf_file}: symbol file has {line_type} line " |
| 297 | f"with {len(words)} words (expected " |
| 298 | f"{expected_words_min} or more)" |
| 299 | ) |
| 300 | |
| 301 | |
| 302 | def ValidateSymbolFile( |
| 303 | sym_file: str, |
| 304 | elf_file: str, |
| 305 | sysroot: Optional[str], |
| 306 | found_files: Optional[multiprocessing.managers.ListProxy], |
| 307 | ) -> bool: |
| 308 | """Checks that the given sym_file has enough info for us to get good stacks. |
| 309 | |
| 310 | Validates that the given sym_file has enough information for us to get |
| 311 | good error reports -- enough STACK records to unwind the stack and enough |
| 312 | FUNC or PUBLIC records to turn the function addresses into human-readable |
| 313 | names. |
| 314 | |
| 315 | Args: |
| 316 | sym_file: The complete path to the breakpad symbol file to validate |
| 317 | elf_file: The complete path to the elf file which was the source of the |
| 318 | symbol file. |
| 319 | sysroot: If not None, the root of the build directory ('/build/eve', for |
| 320 | instance). |
| 321 | found_files: A multiprocessing.managers.ListProxy list containing |
| 322 | ExpectedFiles, representing which of the "should always be present" |
| 323 | files have been processed. |
| 324 | |
| 325 | Returns: |
| 326 | True if the symbol file passes validation. |
| 327 | """ |
| 328 | if sysroot is not None: |
| 329 | relative_path = os.path.relpath(elf_file, sysroot) |
| 330 | else: |
| 331 | relative_path = os.path.relpath(elf_file, "/") |
| 332 | |
| 333 | if relative_path in ALLOWLIST_NO_SYMBOL_FILE_VALIDATION: |
| 334 | return True |
| 335 | for regex in ALLOWLIST_NO_SYMBOL_FILE_VALIDATION_RE: |
| 336 | if regex.match(relative_path): |
| 337 | return True |
| 338 | |
| 339 | counts = SymbolFileLineCounts(sym_file, elf_file) |
| 340 | |
| 341 | errors = False |
| 342 | if counts.stack_lines == 0: |
| 343 | # Use the elf_file in error messages; sym_file is still a temporary |
| 344 | # file with a meaningless-to-humans name right now. |
| 345 | logging.warning("%s: Symbol file has no STACK records", elf_file) |
| 346 | errors = True |
| 347 | if counts.module_lines != 1: |
| 348 | logging.warning( |
| 349 | "%s: Symbol file has %d MODULE lines", elf_file, counts.module_lines |
| 350 | ) |
| 351 | errors = True |
| 352 | # Many shared object files have only PUBLIC functions. In theory, |
| 353 | # executables should always have at least one FUNC (main) and some line |
| 354 | # numbers, but for reasons I'm unclear on, C-based executables often just |
| 355 | # have PUBLIC records. dump_syms does not support line numbers after |
| 356 | # PUBLIC records, only FUNC records, so such executables will also have |
| 357 | # no line numbers. |
| 358 | if counts.public_lines == 0 and counts.func_lines == 0: |
| 359 | logging.warning( |
| 360 | "%s: Symbol file has no FUNC or PUBLIC records", elf_file |
| 361 | ) |
| 362 | errors = True |
| 363 | # However, if we get a FUNC record, we do want line numbers for it. |
| 364 | if counts.func_lines > 0 and counts.line_number_lines == 0: |
| 365 | logging.warning( |
| 366 | "%s: Symbol file has FUNC records but no line numbers", elf_file |
| 367 | ) |
| 368 | errors = True |
| 369 | |
| 370 | if counts.line_number_lines > 0 and counts.file_lines == 0: |
| 371 | logging.warning( |
| 372 | "%s: Symbol file has line number records but no FILE records", |
| 373 | elf_file, |
| 374 | ) |
| 375 | errors = True |
| 376 | if counts.inline_lines > 0 and counts.file_lines == 0: |
| 377 | logging.warning( |
| 378 | "%s: Symbol file has INLINE records but no FILE records", elf_file |
| 379 | ) |
| 380 | errors = True |
| 381 | |
| 382 | if counts.inline_lines > 0 and counts.inline_origin_lines == 0: |
| 383 | logging.warning( |
| 384 | "%s: Symbol file has INLINE records but no INLINE_ORIGIN records", |
| 385 | elf_file, |
| 386 | ) |
| 387 | errors = True |
| 388 | |
| 389 | def _AddFoundFile(files, found): |
| 390 | """Add another file to the list of expected files we've found.""" |
| 391 | if files is not None: |
| 392 | files.append(found) |
| 393 | |
| 394 | # Extra validation for a few ELF files which are special. Either these are |
| 395 | # unusually important to the system (chrome binary, which is where a large |
| 396 | # fraction of our crashes occur, and libc.so, which is in every stack), or |
| 397 | # they are some hand-chosen ELF files which stand in for "normal" platform2 |
| 398 | # binaries. Not all ELF files would pass the extra validation, so we can't |
| 399 | # run these checks on every ELF, but we want to make sure we don't end up |
| 400 | # with, say, a chrome build or a platform2 build with just one or two FUNC |
| 401 | # records on every binary. |
| 402 | if relative_path == "opt/google/chrome/chrome": |
| 403 | _AddFoundFile(found_files, ExpectedFiles.ASH_CHROME) |
| 404 | if counts.func_lines < 100000: |
| 405 | logging.warning( |
| 406 | "chrome should have at least 100,000 FUNC records, found %d", |
| 407 | counts.func_lines, |
| 408 | ) |
| 409 | errors = True |
| 410 | if counts.stack_lines < 1000000: |
| 411 | logging.warning( |
| 412 | "chrome should have at least 1,000,000 STACK records, found %d", |
| 413 | counts.stack_lines, |
| 414 | ) |
| 415 | errors = True |
| 416 | if counts.line_number_lines < 1000000: |
| 417 | logging.warning( |
| 418 | "chrome should have at least 1,000,000 line number records, " |
| 419 | "found %d", |
| 420 | counts.line_number_lines, |
| 421 | ) |
| 422 | errors = True |
| 423 | # Lacros symbol files are not generated as part of the ChromeOS build and |
| 424 | # can't be validated here. |
| 425 | # TODO(b/273836486): Add similar logic to the code that generates Lacros |
| 426 | # symbols. |
| 427 | elif os.path.basename(relative_path).startswith("libc.so"): |
| 428 | _AddFoundFile(found_files, ExpectedFiles.LIBC) |
| 429 | if counts.public_lines < 100: |
| 430 | logging.warning( |
| 431 | "%s should have at least 100 PUBLIC records, found %d", |
| 432 | elf_file, |
| 433 | counts.public_lines, |
| 434 | ) |
| 435 | errors = True |
| 436 | if counts.stack_lines < 10000: |
| 437 | logging.warning( |
| 438 | "%s should have at least 10000 STACK records, found %d", |
| 439 | elf_file, |
| 440 | counts.stack_lines, |
| 441 | ) |
| 442 | errors = True |
| 443 | elif relative_path == "sbin/crash_reporter": |
| 444 | # Representative platform2 executable. |
| 445 | _AddFoundFile(found_files, ExpectedFiles.CRASH_REPORTER) |
| 446 | if counts.stack_lines < 1000: |
| 447 | logging.warning( |
| 448 | "crash_reporter should have at least 1000 STACK records, " |
| 449 | "found %d", |
| 450 | counts.stack_lines, |
| 451 | ) |
| 452 | errors = True |
| 453 | if counts.func_lines < 1000: |
| 454 | logging.warning( |
| 455 | "crash_reporter should have at least 1000 FUNC records, " |
| 456 | "found %d", |
| 457 | counts.func_lines, |
| 458 | ) |
| 459 | errors = True |
| 460 | if counts.line_number_lines < 10000: |
| 461 | logging.warning( |
| 462 | "crash_reporter should have at least 10,000 line number " |
| 463 | "records, found %d", |
| 464 | counts.line_number_lines, |
| 465 | ) |
| 466 | errors = True |
| 467 | elif os.path.basename(relative_path) == "libmetrics.so": |
| 468 | # Representative platform2 shared library. |
| 469 | _AddFoundFile(found_files, ExpectedFiles.LIBMETRICS) |
| 470 | if counts.func_lines < 100: |
| 471 | logging.warning( |
| 472 | "libmetrics should have at least 100 FUNC records, found %d", |
| 473 | counts.func_lines, |
| 474 | ) |
| 475 | errors = True |
| 476 | if counts.public_lines == 0: |
| 477 | logging.warning( |
| 478 | "libmetrics should have at least 1 PUBLIC record, found %d", |
| 479 | counts.public_lines, |
| 480 | ) |
| 481 | errors = True |
| 482 | if counts.stack_lines < 1000: |
| 483 | logging.warning( |
| 484 | "libmetrics should have at least 1000 STACK records, found %d", |
| 485 | counts.stack_lines, |
| 486 | ) |
| 487 | errors = True |
| 488 | if counts.line_number_lines < 5000: |
| 489 | logging.warning( |
| 490 | "libmetrics should have at least 5000 line number records, " |
| 491 | "found %d", |
| 492 | counts.line_number_lines, |
| 493 | ) |
| 494 | errors = True |
| 495 | |
| 496 | return not errors |
| 497 | |
| 498 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 499 | def _ExpectGoodSymbols(elf_file, sysroot): |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 500 | """Determines if we expect dump_syms to create good symbols. |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 501 | |
| 502 | We know that certain types of files never generate good symbols. Distinguish |
| 503 | those from the majority of elf files which should generate good symbols. |
| 504 | |
| 505 | Args: |
| 506 | elf_file: The complete path to the file which we will pass to dump_syms |
| 507 | sysroot: If not None, the root of the build directory ('/build/eve', for |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 508 | instance) |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 509 | |
| 510 | Returns: |
| 511 | True if the elf file should generate good symbols, False if not. |
| 512 | """ |
| 513 | # .ko files (kernel object files) never produce good symbols. |
| 514 | if elf_file.endswith(".ko"): |
| 515 | return False |
| 516 | |
| 517 | # dump_syms doesn't understand Golang executables. |
| 518 | result = cros_build_lib.run( |
| 519 | ["/usr/bin/file", elf_file], print_cmd=False, stdout=True |
| 520 | ) |
| 521 | if b"Go BuildID" in result.stdout: |
| 522 | return False |
| 523 | |
| 524 | if sysroot is not None: |
| 525 | relative_path = os.path.relpath(elf_file, sysroot) |
| 526 | else: |
Ian Barkley-Yeung | e178576 | 2023-03-08 18:32:18 -0800 | [diff] [blame] | 527 | relative_path = os.path.relpath(elf_file, "/") |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 528 | |
| 529 | if relative_path in EXPECTED_POOR_SYMBOLIZATION_FILES: |
| 530 | return False |
| 531 | |
Ian Barkley-Yeung | e178576 | 2023-03-08 18:32:18 -0800 | [diff] [blame] | 532 | # Binaries in /usr/local are not actually shipped to end-users, so we |
| 533 | # don't care if they get good symbols -- we should never get crash reports |
| 534 | # for them anyways. |
| 535 | if relative_path.startswith("usr/local"): |
| 536 | return False |
| 537 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 538 | return True |
| 539 | |
| 540 | |
| 541 | def ReadSymsHeader(sym_file, name_for_errors): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 542 | """Parse the header of the symbol file |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 543 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 544 | The first line of the syms file will read like: |
| 545 | MODULE Linux arm F4F6FA6CCBDEF455039C8DE869C8A2F40 blkid |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 546 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 547 | https://code.google.com/p/google-breakpad/wiki/SymbolFiles |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 548 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 549 | Args: |
| 550 | sym_file: The symbol file to parse |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 551 | name_for_errors: A name for error strings. Can be the name of the elf file |
| 552 | that generated the symbol file, or the name of the symbol file if the |
| 553 | symbol file has already been moved to a meaningful location. |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 554 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 555 | Returns: |
| 556 | A SymbolHeader object |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 557 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 558 | Raises: |
| 559 | ValueError if the first line of |sym_file| is invalid |
| 560 | """ |
| 561 | with file_util.Open(sym_file, "rb") as f: |
| 562 | header = f.readline().decode("utf-8").split() |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 563 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 564 | if len(header) != 5 or header[0] != "MODULE": |
| 565 | raise ValueError( |
| 566 | f"header of sym file from {name_for_errors} is invalid" |
| 567 | ) |
Mike Frysinger | 50cedd3 | 2014-02-09 23:03:18 -0500 | [diff] [blame] | 568 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 569 | return SymbolHeader( |
| 570 | os=header[1], cpu=header[2], id=header[3], name=header[4] |
| 571 | ) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 572 | |
| 573 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 574 | def GenerateBreakpadSymbol( |
| 575 | elf_file, |
| 576 | debug_file=None, |
| 577 | breakpad_dir=None, |
| 578 | strip_cfi=False, |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 579 | sysroot=None, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 580 | num_errors=None, |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 581 | found_files=None, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 582 | dump_syms_cmd="dump_syms", |
| 583 | ): |
| 584 | """Generate the symbols for |elf_file| using |debug_file| |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 585 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 586 | Args: |
| 587 | elf_file: The file to dump symbols for |
| 588 | debug_file: Split debug file to use for symbol information |
| 589 | breakpad_dir: The dir to store the output symbol file in |
| 590 | strip_cfi: Do not generate CFI data |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 591 | sysroot: Path to the sysroot with the elf_file under it |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 592 | num_errors: An object to update with the error count (needs a .value member) |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 593 | found_files: A multiprocessing.managers.ListProxy list containing |
| 594 | ExpectedFiles, representing which of the "should always be present" |
| 595 | files have been processed. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 596 | dump_syms_cmd: Command to use for dumping symbols. |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 597 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 598 | Returns: |
| 599 | The name of symbol file written out on success, or the failure count. |
| 600 | """ |
| 601 | assert breakpad_dir |
| 602 | if num_errors is None: |
| 603 | num_errors = ctypes.c_int() |
| 604 | debug_file_only = not os.path.exists(elf_file) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 605 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 606 | cmd_base = [dump_syms_cmd, "-v"] |
| 607 | if strip_cfi: |
| 608 | cmd_base += ["-c"] |
| 609 | # Some files will not be readable by non-root (e.g. set*id /bin/su). |
| 610 | needs_sudo = not os.access(elf_file, os.R_OK) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 611 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 612 | def _DumpIt(cmd_args): |
| 613 | if needs_sudo: |
| 614 | run_command = cros_build_lib.sudo_run |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 615 | else: |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 616 | run_command = cros_build_lib.run |
| 617 | return run_command( |
| 618 | cmd_base + cmd_args, |
| 619 | stderr=True, |
| 620 | stdout=temp.name, |
| 621 | check=False, |
| 622 | debug_level=logging.DEBUG, |
| 623 | ) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 624 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 625 | def _CrashCheck(result, file_or_files, msg): |
| 626 | if result.returncode: |
| 627 | cbuildbot_alerts.PrintBuildbotStepWarnings() |
| 628 | if result.returncode < 0: |
| 629 | logging.warning( |
| 630 | "dump_syms %s crashed with %s; %s", |
| 631 | file_or_files, |
| 632 | signals.StrSignal(-result.returncode), |
| 633 | msg, |
| 634 | ) |
| 635 | else: |
| 636 | logging.warning( |
| 637 | "dump_syms %s returned %d; %s", |
| 638 | file_or_files, |
| 639 | result.returncode, |
| 640 | msg, |
| 641 | ) |
| 642 | logging.warning("output:\n%s", result.stderr.decode("utf-8")) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 643 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 644 | def _DumpAllowingBasicFallback(): |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 645 | """Dump symbols for an ELF when we do NOT expect to get good symbols. |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 646 | |
| 647 | Returns: |
| 648 | A SymbolGenerationResult |
| 649 | """ |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 650 | if debug_file: |
| 651 | # Try to dump the symbols using the debug file like normal. |
| 652 | if debug_file_only: |
| 653 | cmd_args = [debug_file] |
| 654 | file_or_files = debug_file |
| 655 | else: |
| 656 | cmd_args = [elf_file, os.path.dirname(debug_file)] |
| 657 | file_or_files = [elf_file, debug_file] |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 658 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 659 | result = _DumpIt(cmd_args) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 660 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 661 | if result.returncode: |
| 662 | # Sometimes dump_syms can crash because there's too much info. |
| 663 | # Try dumping and stripping the extended stuff out. At least |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 664 | # this way we'll get the extended symbols. |
| 665 | # https://crbug.com/266064 |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 666 | _CrashCheck(result, file_or_files, "retrying w/out CFI") |
| 667 | cmd_args = ["-c", "-r"] + cmd_args |
| 668 | result = _DumpIt(cmd_args) |
| 669 | _CrashCheck(result, file_or_files, "retrying w/out debug") |
Prathmesh Prabhu | 9995e9b | 2013-10-31 16:43:55 -0700 | [diff] [blame] | 670 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 671 | if not result.returncode: |
| 672 | return SymbolGenerationResult.SUCCESS |
Prathmesh Prabhu | 9995e9b | 2013-10-31 16:43:55 -0700 | [diff] [blame] | 673 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 674 | # If that didn't work (no debug, or dump_syms still failed), try |
| 675 | # dumping just the file itself directly. |
| 676 | result = _DumpIt([elf_file]) |
| 677 | if result.returncode: |
| 678 | # A lot of files (like kernel files) contain no debug information, |
| 679 | # do not consider such occurrences as errors. |
| 680 | cbuildbot_alerts.PrintBuildbotStepWarnings() |
| 681 | if b"file contains no debugging information" in result.stderr: |
| 682 | logging.warning("dump_syms failed; giving up entirely.") |
| 683 | logging.warning("No symbols found for %s", elf_file) |
| 684 | return SymbolGenerationResult.EXPECTED_FAILURE |
| 685 | else: |
| 686 | _CrashCheck(result, elf_file, "counting as failure") |
| 687 | return SymbolGenerationResult.UNEXPECTED_FAILURE |
| 688 | |
| 689 | return SymbolGenerationResult.SUCCESS |
| 690 | |
| 691 | def _DumpExpectingSymbols(): |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 692 | """Dump symbols for an ELF when we expect to get good symbols. |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 693 | |
| 694 | Returns: |
| 695 | A SymbolGenerationResult. We never expect failure, so the result |
| 696 | will always be SUCCESS or UNEXPECTED_FAILURE. |
| 697 | """ |
| 698 | if not debug_file: |
| 699 | logging.warning("%s must have debug file", elf_file) |
| 700 | return SymbolGenerationResult.UNEXPECTED_FAILURE |
| 701 | |
| 702 | cmd_args = [elf_file, os.path.dirname(debug_file)] |
| 703 | result = _DumpIt(cmd_args) |
| 704 | if result.returncode: |
| 705 | _CrashCheck(result, [elf_file, debug_file], "unexpected failure") |
| 706 | return SymbolGenerationResult.UNEXPECTED_FAILURE |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 707 | |
| 708 | # TODO(b/270240549): Remove try/except, allow exceptions to just |
| 709 | # fail the script. The try/except is just here until we are sure this |
| 710 | # will not break the build. |
| 711 | try: |
| 712 | if not ValidateSymbolFile( |
| 713 | temp.name, elf_file, sysroot, found_files |
| 714 | ): |
| 715 | logging.warning("%s: symbol file failed validation", elf_file) |
| 716 | return SymbolGenerationResult.UNEXPECTED_FAILURE |
| 717 | except ValueError as e: |
| 718 | logging.warning( |
| 719 | "%s: symbol file failed validation due to exception %s", |
| 720 | elf_file, |
| 721 | e, |
| 722 | ) |
| 723 | return SymbolGenerationResult.UNEXPECTED_FAILURE |
| 724 | |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 725 | return SymbolGenerationResult.SUCCESS |
| 726 | |
| 727 | osutils.SafeMakedirs(breakpad_dir) |
| 728 | with cros_build_lib.UnbufferedNamedTemporaryFile( |
| 729 | dir=breakpad_dir, delete=False |
| 730 | ) as temp: |
| 731 | if _ExpectGoodSymbols(elf_file, sysroot): |
| 732 | result = _DumpExpectingSymbols() |
| 733 | # Until the EXPECTED_POOR_SYMBOLIZATION_FILES allowlist is |
| 734 | # completely set up for all boards, don't fail the build if |
| 735 | # _ExpectGoodSymbols is wrong. |
| 736 | # TODO(b/241470012): Remove the call to _DumpAllowingBasicFallback() |
| 737 | # and just error out if _DumpExpectingSymbols fails. |
| 738 | if result == SymbolGenerationResult.UNEXPECTED_FAILURE: |
| 739 | result = _DumpAllowingBasicFallback() |
| 740 | else: |
| 741 | result = _DumpAllowingBasicFallback() |
| 742 | |
| 743 | if result == SymbolGenerationResult.UNEXPECTED_FAILURE: |
| 744 | num_errors.value += 1 |
| 745 | os.unlink(temp.name) |
| 746 | return num_errors.value |
| 747 | |
| 748 | if result == SymbolGenerationResult.EXPECTED_FAILURE: |
| 749 | os.unlink(temp.name) |
| 750 | return 0 |
Prathmesh Prabhu | 9995e9b | 2013-10-31 16:43:55 -0700 | [diff] [blame] | 751 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 752 | # Move the dumped symbol file to the right place: |
| 753 | # /build/$BOARD/usr/lib/debug/breakpad/<module-name>/<id>/<module-name>.sym |
Ian Barkley-Yeung | fdaaf2e | 2023-03-02 17:30:39 -0800 | [diff] [blame] | 754 | header = ReadSymsHeader(temp, elf_file) |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 755 | logging.info("Dumped %s as %s : %s", elf_file, header.name, header.id) |
| 756 | sym_file = os.path.join( |
| 757 | breakpad_dir, header.name, header.id, header.name + ".sym" |
| 758 | ) |
| 759 | osutils.SafeMakedirs(os.path.dirname(sym_file)) |
| 760 | os.rename(temp.name, sym_file) |
| 761 | os.chmod(sym_file, 0o644) |
Prathmesh Prabhu | 9995e9b | 2013-10-31 16:43:55 -0700 | [diff] [blame] | 762 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 763 | return sym_file |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 764 | |
Prathmesh Prabhu | 9995e9b | 2013-10-31 16:43:55 -0700 | [diff] [blame] | 765 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 766 | def GenerateBreakpadSymbols( |
| 767 | board, |
| 768 | breakpad_dir=None, |
| 769 | strip_cfi=False, |
| 770 | generate_count=None, |
| 771 | sysroot=None, |
| 772 | num_processes=None, |
| 773 | clean_breakpad=False, |
| 774 | exclude_dirs=(), |
| 775 | file_list=None, |
| 776 | ): |
| 777 | """Generate symbols for this board. |
Prathmesh Prabhu | 9995e9b | 2013-10-31 16:43:55 -0700 | [diff] [blame] | 778 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 779 | If |file_list| is None, symbols are generated for all executables, otherwise |
| 780 | only for the files included in |file_list|. |
Prathmesh Prabhu | 9995e9b | 2013-10-31 16:43:55 -0700 | [diff] [blame] | 781 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 782 | TODO(build): |
| 783 | This should be merged with buildbot_commands.GenerateBreakpadSymbols() |
| 784 | once we rewrite cros_generate_breakpad_symbols in python. |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 785 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 786 | Args: |
| 787 | board: The board whose symbols we wish to generate |
| 788 | breakpad_dir: The full path to the breakpad directory where symbols live |
| 789 | strip_cfi: Do not generate CFI data |
| 790 | generate_count: If set, only generate this many symbols (meant for testing) |
| 791 | sysroot: The root where to find the corresponding ELFs |
| 792 | num_processes: Number of jobs to run in parallel |
| 793 | clean_breakpad: Should we `rm -rf` the breakpad output dir first; note: we |
| 794 | do not do any locking, so do not run more than one in parallel when True |
| 795 | exclude_dirs: List of dirs (relative to |sysroot|) to not search |
| 796 | file_list: Only generate symbols for files in this list. Each file must be a |
| 797 | full path (including |sysroot| prefix). |
| 798 | TODO(build): Support paths w/o |sysroot|. |
| 799 | |
| 800 | Returns: |
| 801 | The number of errors that were encountered. |
| 802 | """ |
| 803 | if sysroot is None: |
| 804 | sysroot = build_target_lib.get_default_sysroot_path(board) |
| 805 | if breakpad_dir is None: |
| 806 | breakpad_dir = FindBreakpadDir(board, sysroot=sysroot) |
| 807 | if clean_breakpad: |
| 808 | logging.info("cleaning out %s first", breakpad_dir) |
| 809 | osutils.RmDir(breakpad_dir, ignore_missing=True, sudo=True) |
| 810 | # Make sure non-root can write out symbols as needed. |
| 811 | osutils.SafeMakedirs(breakpad_dir, sudo=True) |
| 812 | if not os.access(breakpad_dir, os.W_OK): |
| 813 | cros_build_lib.sudo_run(["chown", "-R", str(os.getuid()), breakpad_dir]) |
| 814 | debug_dir = FindDebugDir(board, sysroot=sysroot) |
| 815 | exclude_paths = [os.path.join(debug_dir, x) for x in exclude_dirs] |
| 816 | if file_list is None: |
| 817 | file_list = [] |
| 818 | file_filter = dict.fromkeys([os.path.normpath(x) for x in file_list], False) |
| 819 | |
| 820 | logging.info("generating breakpad symbols using %s", debug_dir) |
| 821 | |
| 822 | # Let's locate all the debug_files and elfs first along with the debug file |
| 823 | # sizes. This way we can start processing the largest files first in parallel |
| 824 | # with the small ones. |
| 825 | # If |file_list| was given, ignore all other files. |
| 826 | targets = [] |
| 827 | for root, dirs, files in os.walk(debug_dir): |
| 828 | if root in exclude_paths: |
| 829 | logging.info("Skipping excluded dir %s", root) |
| 830 | del dirs[:] |
| 831 | continue |
| 832 | |
| 833 | for debug_file in files: |
| 834 | debug_file = os.path.join(root, debug_file) |
| 835 | # Turn /build/$BOARD/usr/lib/debug/sbin/foo.debug into |
| 836 | # /build/$BOARD/sbin/foo. |
| 837 | elf_file = os.path.join( |
| 838 | sysroot, debug_file[len(debug_dir) + 1 : -6] |
| 839 | ) |
| 840 | |
| 841 | if file_filter: |
| 842 | if elf_file in file_filter: |
| 843 | file_filter[elf_file] = True |
| 844 | elif debug_file in file_filter: |
| 845 | file_filter[debug_file] = True |
| 846 | else: |
| 847 | continue |
| 848 | |
| 849 | # Filter out files based on common issues with the debug file. |
| 850 | if not debug_file.endswith(".debug"): |
| 851 | continue |
| 852 | |
| 853 | elif os.path.islink(debug_file): |
| 854 | # The build-id stuff is common enough to filter out by default. |
| 855 | if "/.build-id/" in debug_file: |
| 856 | msg = logging.debug |
| 857 | else: |
| 858 | msg = logging.warning |
| 859 | msg("Skipping symbolic link %s", debug_file) |
| 860 | continue |
| 861 | |
| 862 | # Filter out files based on common issues with the elf file. |
| 863 | elf_path = os.path.relpath(elf_file, sysroot) |
| 864 | debug_only = elf_path in ALLOWED_DEBUG_ONLY_FILES |
| 865 | if not os.path.exists(elf_file) and not debug_only: |
| 866 | # Sometimes we filter out programs from /usr/bin but leave behind |
| 867 | # the .debug file. |
| 868 | logging.warning("Skipping missing %s", elf_file) |
| 869 | continue |
| 870 | |
| 871 | targets.append((os.path.getsize(debug_file), elf_file, debug_file)) |
| 872 | |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 873 | with multiprocessing.Manager() as mp_manager: |
| 874 | bg_errors = parallel.WrapMultiprocessing(multiprocessing.Value, "i") |
| 875 | found_files = parallel.WrapMultiprocessing(mp_manager.list) |
| 876 | if file_filter: |
| 877 | files_not_found = [ |
| 878 | x for x, found in file_filter.items() if not found |
| 879 | ] |
| 880 | bg_errors.value += len(files_not_found) |
| 881 | if files_not_found: |
| 882 | logging.error( |
| 883 | "Failed to find requested files: %s", files_not_found |
| 884 | ) |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 885 | |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 886 | # Now start generating symbols for the discovered elfs. |
| 887 | with parallel.BackgroundTaskRunner( |
| 888 | GenerateBreakpadSymbol, |
| 889 | breakpad_dir=breakpad_dir, |
| 890 | strip_cfi=strip_cfi, |
| 891 | num_errors=bg_errors, |
| 892 | processes=num_processes, |
| 893 | sysroot=sysroot, |
| 894 | found_files=found_files, |
| 895 | ) as queue: |
| 896 | for _, elf_file, debug_file in sorted(targets, reverse=True): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 897 | if generate_count == 0: |
| 898 | break |
| 899 | |
Ian Barkley-Yeung | c5b6f58 | 2023-03-08 18:23:07 -0800 | [diff] [blame] | 900 | queue.put([elf_file, debug_file]) |
| 901 | if generate_count is not None: |
| 902 | generate_count -= 1 |
| 903 | if generate_count == 0: |
| 904 | break |
| 905 | |
| 906 | missing = ALL_EXPECTED_FILES - frozenset(found_files) |
| 907 | if missing and not file_filter and generate_count is None: |
| 908 | logging.warning( |
| 909 | "Not all expected files were processed successfully, " |
| 910 | "missing %s", |
| 911 | missing, |
| 912 | ) |
| 913 | # TODO(b/270240549): Increment bg_errors.value here once we check |
| 914 | # that this isn't going to fail any current builds. |
| 915 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 916 | return bg_errors.value |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 917 | |
| 918 | |
Mike Frysinger | 3f571af | 2016-08-31 23:56:53 -0400 | [diff] [blame] | 919 | def FindDebugDir(board, sysroot=None): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 920 | """Given a |board|, return the path to the split debug dir for it""" |
| 921 | if sysroot is None: |
| 922 | sysroot = build_target_lib.get_default_sysroot_path(board) |
| 923 | return os.path.join(sysroot, "usr", "lib", "debug") |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 924 | |
| 925 | |
Mike Frysinger | 3f571af | 2016-08-31 23:56:53 -0400 | [diff] [blame] | 926 | def FindBreakpadDir(board, sysroot=None): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 927 | """Given a |board|, return the path to the breakpad dir for it""" |
| 928 | return os.path.join(FindDebugDir(board, sysroot=sysroot), "breakpad") |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 929 | |
| 930 | |
| 931 | def main(argv): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 932 | parser = commandline.ArgumentParser(description=__doc__) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 933 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 934 | parser.add_argument( |
| 935 | "--board", default=None, help="board to generate symbols for" |
| 936 | ) |
| 937 | parser.add_argument( |
| 938 | "--breakpad_root", |
| 939 | type="path", |
| 940 | default=None, |
| 941 | help="root output directory for breakpad symbols", |
| 942 | ) |
| 943 | parser.add_argument( |
| 944 | "--sysroot", |
| 945 | type="path", |
| 946 | default=None, |
| 947 | help="root input directory for files", |
| 948 | ) |
| 949 | parser.add_argument( |
| 950 | "--exclude-dir", |
| 951 | type=str, |
| 952 | action="append", |
| 953 | default=[], |
| 954 | help="directory (relative to |board| root) to not search", |
| 955 | ) |
| 956 | parser.add_argument( |
| 957 | "--generate-count", |
| 958 | type=int, |
| 959 | default=None, |
| 960 | help="only generate # number of symbols", |
| 961 | ) |
| 962 | parser.add_argument( |
| 963 | "--noclean", |
| 964 | dest="clean", |
| 965 | action="store_false", |
| 966 | default=True, |
| 967 | help="do not clean out breakpad dir before running", |
| 968 | ) |
| 969 | parser.add_argument( |
| 970 | "--jobs", type=int, default=None, help="limit number of parallel jobs" |
| 971 | ) |
| 972 | parser.add_argument( |
| 973 | "--strip_cfi", |
| 974 | action="store_true", |
| 975 | default=False, |
| 976 | help="do not generate CFI data (pass -c to dump_syms)", |
| 977 | ) |
| 978 | parser.add_argument( |
| 979 | "file_list", |
| 980 | nargs="*", |
| 981 | default=None, |
| 982 | help="generate symbols for only these files " |
| 983 | "(e.g. /build/$BOARD/usr/bin/foo)", |
| 984 | ) |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 985 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 986 | opts = parser.parse_args(argv) |
| 987 | opts.Freeze() |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 988 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 989 | if opts.board is None and opts.sysroot is None: |
| 990 | cros_build_lib.Die("--board or --sysroot is required") |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 991 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 992 | ret = GenerateBreakpadSymbols( |
| 993 | opts.board, |
| 994 | breakpad_dir=opts.breakpad_root, |
| 995 | strip_cfi=opts.strip_cfi, |
| 996 | generate_count=opts.generate_count, |
| 997 | sysroot=opts.sysroot, |
| 998 | num_processes=opts.jobs, |
| 999 | clean_breakpad=opts.clean, |
| 1000 | exclude_dirs=opts.exclude_dir, |
| 1001 | file_list=opts.file_list, |
| 1002 | ) |
| 1003 | if ret: |
| 1004 | logging.error("encountered %i problem(s)", ret) |
| 1005 | # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently |
| 1006 | # return 0 in case we are a multiple of the mask. |
| 1007 | ret = 1 |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 1008 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 1009 | return ret |