Re-land "[dev-util] Add symbolicate_dump endpoint to dev server"
This reverts commit 50fe427cb8cebf7dca441e92463c407cb90fddf3.
Add an endpoint to the dev server that will symbolicate a minidump.
BUG=chromium-os:29850
TEST=unit
TEST=run dev server, use curl to make it download some artifacts; check to
TEST=see that debug symbols are staged in static/archive
TEST=once symbols are staged, run the dev server in your
TEST=chroot and use curl with a minidump file like this:
TEST= curl -F minidump=@/home/cmasone/chromeos/phooey/powerd.20120424.141235.1005.dmp http://localhost:8080/symbolicate_dump
Change-Id: Ib9c11afd780aea5a665358b43f03a210ecc83482
Reviewed-on: https://gerrit.chromium.org/gerrit/21541
Tested-by: Chris Masone <cmasone@chromium.org>
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Tested-by: Scott Zawalski <scottz@chromium.org>
Commit-Ready: Scott Zawalski <scottz@chromium.org>
diff --git a/devserver.py b/devserver.py
index bb22e22..07c85ee 100755
--- a/devserver.py
+++ b/devserver.py
@@ -7,11 +7,14 @@
"""A CherryPy-based webserver to host images and build packages."""
import cherrypy
+import cStringIO
import logging
import optparse
import os
import re
import sys
+import subprocess
+import tempfile
import autoupdate
import devserver_util
@@ -240,6 +243,41 @@
return return_obj
@cherrypy.expose
+ def symbolicate_dump(self, minidump):
+ """Symbolicates a minidump using pre-downloaded symbols, returns it.
+
+ Callers will need to POST to this URL with a body of MIME-type
+ "multipart/form-data".
+ The body should include a single argument, 'minidump', containing the
+ binary-formatted minidump to symbolicate.
+
+ It is up to the caller to ensure that the symbols they want are currently
+ staged.
+
+ Args:
+ minidump: The binary minidump file to symbolicate.
+ """
+ to_return = ''
+ with tempfile.NamedTemporaryFile() as local:
+ while True:
+ data = minidump.file.read(8192)
+ if not data:
+ break
+ local.write(data)
+ local.flush()
+ stackwalk = subprocess.Popen(['minidump_stackwalk',
+ local.name,
+ updater.static_dir + '/debug/breakpad'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ to_return, error_text = stackwalk.communicate()
+ if stackwalk.returncode != 0:
+ raise DevServerError("Can't generate stack trace: %s (rc=%d)" % (
+ error_text, stackwalk.returncode))
+
+ return to_return
+
+ @cherrypy.expose
def wait_for_status(self, **kwargs):
"""Waits for background artifacts to be downloaded from Google Storage.
diff --git a/devserver_util.py b/devserver_util.py
index eb498f4..e40ae7e 100644
--- a/devserver_util.py
+++ b/devserver_util.py
@@ -57,12 +57,15 @@
return full_payload_url, nton_payload_url, mton_payload_url
-def GatherArtifactDownloads(main_staging_dir, archive_url, build, build_dir):
+def GatherArtifactDownloads(main_staging_dir, archive_url, build, build_dir,
+ static_dir):
"""Generates artifacts that we mean to download and install for autotest.
This method generates the list of artifacts we will need for autotest. These
- artifacts are instances of downloadable_artifact.DownloadableArtifact.Note,
- these artifacts can be downloaded asynchronously iff !artifact.Synchronous().
+ artifacts are instances of downloadable_artifact.DownloadableArtifact.
+
+ Note, these artifacts can be downloaded asynchronously iff
+ !artifact.Synchronous().
"""
cmd = 'gsutil ls %s/*.bin' % archive_url
msg = 'Failed to get a list of payloads.'
@@ -87,6 +90,7 @@
mton_url, main_staging_dir, mton_payload))
# Next we gather the miscellaneous payloads.
+ debug_url = archive_url + '/' + downloadable_artifact.DEBUG_SYMBOLS
stateful_url = archive_url + '/' + downloadable_artifact.STATEFUL_UPDATE
autotest_url = archive_url + '/' + downloadable_artifact.AUTOTEST_PACKAGE
test_suites_url = (archive_url + '/' +
@@ -95,6 +99,8 @@
stateful_payload = os.path.join(build_dir,
downloadable_artifact.STATEFUL_UPDATE)
+ artifacts.append(downloadable_artifact.DebugTarball(
+ debug_url, main_staging_dir, static_dir))
artifacts.append(downloadable_artifact.DownloadableArtifact(
stateful_url, main_staging_dir, stateful_payload, synchronous=True))
artifacts.append(downloadable_artifact.AutotestTarball(
diff --git a/devserver_util_unittest.py b/devserver_util_unittest.py
index 15b41ff..f017208 100755
--- a/devserver_util_unittest.py
+++ b/devserver_util_unittest.py
@@ -286,7 +286,8 @@
['p1', 'p2', 'p3'])
expected_payloads = payloads + map(
lambda x: '/'.join([archive_url_prefix, x]),
- [downloadable_artifact.STATEFUL_UPDATE,
+ [downloadable_artifact.DEBUG_SYMBOLS,
+ downloadable_artifact.STATEFUL_UPDATE,
downloadable_artifact.AUTOTEST_PACKAGE,
downloadable_artifact.TEST_SUITES_PACKAGE])
self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
@@ -299,7 +300,8 @@
self.mox.ReplayAll()
artifacts = devserver_util.GatherArtifactDownloads(
- self._static_dir, archive_url_prefix, build, self._install_dir)
+ self._static_dir, archive_url_prefix, build, self._install_dir,
+ self._static_dir)
for index, artifact in enumerate(artifacts):
self.assertEqual(artifact._gs_path, expected_payloads[index])
self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
@@ -317,7 +319,8 @@
['p1', 'p2'])
expected_payloads = payloads + map(
lambda x: '/'.join([archive_url_prefix, x]),
- [downloadable_artifact.STATEFUL_UPDATE,
+ [downloadable_artifact.DEBUG_SYMBOLS,
+ downloadable_artifact.STATEFUL_UPDATE,
downloadable_artifact.AUTOTEST_PACKAGE,
downloadable_artifact.TEST_SUITES_PACKAGE])
self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
@@ -331,7 +334,8 @@
self.mox.ReplayAll()
artifacts = devserver_util.GatherArtifactDownloads(
- self._static_dir, archive_url_prefix, build, self._install_dir)
+ self._static_dir, archive_url_prefix, build, self._install_dir,
+ self._static_dir)
for index, artifact in enumerate(artifacts):
self.assertEqual(artifact._gs_path, expected_payloads[index])
self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
diff --git a/downloadable_artifact.py b/downloadable_artifact.py
index f4a3f82..bf16885 100644
--- a/downloadable_artifact.py
+++ b/downloadable_artifact.py
@@ -13,6 +13,7 @@
# Names of artifacts we care about.
+DEBUG_SYMBOLS = 'debug.tgz'
STATEFUL_UPDATE = 'stateful.tgz'
TEST_IMAGE = 'chromiumos_test_image.bin'
ROOT_UPDATE = 'update.gz'
@@ -137,3 +138,17 @@
# code.
cmd = 'cp %s/* %s' % (autotest_pkgs_dir, autotest_dir)
subprocess.check_call(cmd, shell=True)
+
+
+class DebugTarball(Tarball):
+ """Wrapper around the debug symbols tarball to download from gsutil."""
+
+ def _ExtractTarball(self):
+ """Extracts debug/breakpad from the tarball into the install_path."""
+ cmd = 'tar xzf %s --directory=%s debug/breakpad' % (
+ self._tmp_stage_path, self._install_path)
+ msg = 'An error occurred when attempting to untar %s' % self._tmp_stage_path
+ try:
+ subprocess.check_call(cmd, shell=True)
+ except subprocess.CalledProcessError, e:
+ raise ArtifactDownloadError('%s %s' % (msg, e))
diff --git a/downloader.py b/downloader.py
index 99988e1..d0ef6e8 100755
--- a/downloader.py
+++ b/downloader.py
@@ -75,7 +75,8 @@
cherrypy.log('Gathering download requirements %s' % self._archive_url,
'DOWNLOAD')
artifacts = devserver_util.GatherArtifactDownloads(
- self._staging_dir, self._archive_url, short_build, self._build_dir)
+ self._staging_dir, self._archive_url, short_build, self._build_dir,
+ self._static_dir)
devserver_util.PrepareBuildDirectory(self._build_dir)
cherrypy.log('Downloading foreground artifacts from %s' % archive_url,
diff --git a/downloader_unittest.py b/downloader_unittest.py
index 693689a..51182ac 100755
--- a/downloader_unittest.py
+++ b/downloader_unittest.py
@@ -12,7 +12,7 @@
import tempfile
import unittest
-import artifact_download
+import downloadable_artifact
import devserver
import devserver_util
import downloader
@@ -56,7 +56,7 @@
artifacts = []
for index in range(5):
- artifact = self.mox.CreateMock(artifact_download.DownloadableArtifact)
+ artifact = self.mox.CreateMock(downloadable_artifact.DownloadableArtifact)
# Make every other artifact synchronous.
if index % 2 == 0:
artifact.Synchronous = lambda: True
@@ -72,7 +72,7 @@
tempfile.mkdtemp(suffix=mox.IgnoreArg()).AndReturn(self._work_dir)
devserver_util.GatherArtifactDownloads(
self._work_dir, self.archive_url_prefix, self.build,
- self._work_dir).AndReturn(artifacts)
+ self._work_dir, self._work_dir).AndReturn(artifacts)
for index, artifact in enumerate(artifacts):
if index % 2 == 0: