Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 3 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | import cherrypy |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 8 | import multiprocessing |
Chris Sosa | 9164ca3 | 2012-03-28 11:04:50 -0700 | [diff] [blame] | 9 | import os |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 10 | import shutil |
| 11 | import tempfile |
| 12 | |
| 13 | import devserver_util |
| 14 | |
| 15 | |
| 16 | class Downloader(object): |
| 17 | """Download images to the devsever. |
| 18 | |
| 19 | Given a URL to a build on the archive server: |
| 20 | |
| 21 | - Determine if the build already exists. |
| 22 | - Download and extract the build to a staging directory. |
| 23 | - Package autotest tests. |
| 24 | - Install components to static dir. |
| 25 | """ |
| 26 | |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 27 | _LOG_TAG = 'DOWNLOAD' |
| 28 | |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 29 | def __init__(self, static_dir): |
| 30 | self._static_dir = static_dir |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 31 | self._build_dir = None |
| 32 | self._staging_dir = None |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 33 | self._status_queue = multiprocessing.Queue(maxsize=1) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 34 | self._lock_tag = None |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 35 | |
| 36 | @staticmethod |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 37 | def ParseUrl(archive_url): |
| 38 | """Parse archive_url into its component parts. |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 39 | |
| 40 | @param archive_url: a URL at which build artifacts are archived. |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 41 | @return a tuple of (build target, short build name) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 42 | """ |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 43 | target, short_build = archive_url.rsplit('/', 2)[-2:] |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 44 | return target, short_build |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 45 | |
| 46 | @staticmethod |
| 47 | def GenerateLockTag(target, short_build): |
| 48 | """Generate a name for a lock scoped to this target/build pair. |
| 49 | |
| 50 | @param target: the target the build was for. |
| 51 | @param short_build: short build name |
| 52 | @return a name to use with AcquireLock that will scope the lock. |
| 53 | """ |
| 54 | return '/'.join([target, short_build]) |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 55 | |
Chris Sosa | 9164ca3 | 2012-03-28 11:04:50 -0700 | [diff] [blame] | 56 | @staticmethod |
| 57 | def BuildStaged(archive_url, static_dir): |
| 58 | """Returns True if the build is already staged.""" |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 59 | target, short_build = Downloader.ParseUrl(archive_url) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 60 | sub_directory = Downloader.GenerateLockTag(target, short_build) |
Chris Sosa | 9164ca3 | 2012-03-28 11:04:50 -0700 | [diff] [blame] | 61 | return os.path.isdir(os.path.join(static_dir, sub_directory)) |
| 62 | |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 63 | def Download(self, archive_url, background=False): |
| 64 | """Downloads the given build artifacts defined by the |archive_url|. |
| 65 | |
| 66 | If background is set to True, will return back early before all artifacts |
| 67 | have been downloaded. The artifacts that can be backgrounded are all those |
| 68 | that are not set as synchronous. |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 69 | |
| 70 | TODO: refactor this into a common Download method, once unit tests are |
| 71 | fixed up to make iterating on the code easier. |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 72 | """ |
Chris Sosa | 9164ca3 | 2012-03-28 11:04:50 -0700 | [diff] [blame] | 73 | # Parse archive_url into target and short_build. |
| 74 | # e.g. gs://chromeos-image-archive/{target}/{short_build} |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 75 | target, short_build = self.ParseUrl(archive_url) |
| 76 | # This should never happen. The Devserver should only try to call this |
| 77 | # method if no previous downloads have been staged for this archive_url. |
| 78 | assert not Downloader.BuildStaged(archive_url, self._static_dir) |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 79 | # Bind build_dir and staging_dir here so we can tell if we need to do any |
| 80 | # cleanup after an exception occurs before build_dir is set. |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 81 | self._lock_tag = self.GenerateLockTag(target, short_build) |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 82 | try: |
| 83 | # Create Dev Server directory for this build and tell other Downloader |
| 84 | # instances we have processed this build. |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 85 | self._build_dir = devserver_util.AcquireLock( |
| 86 | static_dir=self._static_dir, tag=self._lock_tag) |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 87 | |
Chris Sosa | 9164ca3 | 2012-03-28 11:04:50 -0700 | [diff] [blame] | 88 | self._staging_dir = tempfile.mkdtemp(suffix='_'.join([target, |
| 89 | short_build])) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 90 | cherrypy.log('Gathering download requirements %s' % archive_url, |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 91 | self._LOG_TAG) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 92 | artifacts = self.GatherArtifactDownloads( |
| 93 | self._staging_dir, archive_url, short_build, self._build_dir) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 94 | devserver_util.PrepareBuildDirectory(self._build_dir) |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 95 | |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 96 | cherrypy.log('Downloading foreground artifacts from %s' % archive_url, |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 97 | self._LOG_TAG) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 98 | background_artifacts = [] |
| 99 | for artifact in artifacts: |
| 100 | if artifact.Synchronous(): |
| 101 | artifact.Download() |
| 102 | artifact.Stage() |
| 103 | else: |
| 104 | background_artifacts.append(artifact) |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 105 | |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 106 | if background: |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 107 | self._DownloadArtifactsInBackground(background_artifacts) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 108 | else: |
| 109 | self._DownloadArtifactsSerially(background_artifacts) |
| 110 | |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 111 | except Exception, e: |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 112 | # Release processing lock, which will remove build components directory |
| 113 | # so future runs can retry. |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 114 | if self._build_dir: |
| 115 | devserver_util.ReleaseLock(static_dir=self._static_dir, |
| 116 | tag=self._lock_tag) |
| 117 | |
| 118 | self._status_queue.put(e) |
| 119 | self._Cleanup() |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 120 | raise |
Frank Farzan | 37761d1 | 2011-12-01 14:29:08 -0800 | [diff] [blame] | 121 | |
| 122 | return 'Success' |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 123 | |
| 124 | def _Cleanup(self): |
| 125 | """Cleans up the staging dir for this downloader instanfce.""" |
| 126 | if self._staging_dir: |
| 127 | cherrypy.log('Cleaning up staging directory %s' % self._staging_dir, |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 128 | self._LOG_TAG) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 129 | shutil.rmtree(self._staging_dir) |
| 130 | |
| 131 | self._staging_dir = None |
| 132 | |
| 133 | def _DownloadArtifactsSerially(self, artifacts): |
| 134 | """Simple function to download all the given artifacts serially.""" |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 135 | cherrypy.log('Downloading background artifacts serially.', self._LOG_TAG) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 136 | try: |
| 137 | for artifact in artifacts: |
| 138 | artifact.Download() |
| 139 | artifact.Stage() |
| 140 | except Exception, e: |
| 141 | self._status_queue.put(e) |
| 142 | |
| 143 | # Release processing lock, which will remove build components directory |
| 144 | # so future runs can retry. |
| 145 | if self._build_dir: |
| 146 | devserver_util.ReleaseLock(static_dir=self._static_dir, |
| 147 | tag=self._lock_tag) |
| 148 | else: |
| 149 | self._status_queue.put('Success') |
| 150 | finally: |
| 151 | self._Cleanup() |
| 152 | |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 153 | def _DownloadArtifactsInBackground(self, artifacts): |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 154 | """Downloads |artifacts| in the background and signals when complete.""" |
| 155 | proc = multiprocessing.Process(target=self._DownloadArtifactsSerially, |
| 156 | args=(artifacts,)) |
Chris Sosa | b65973e | 2012-03-29 18:31:02 -0700 | [diff] [blame] | 157 | proc.start() |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 158 | |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 159 | def GatherArtifactDownloads(self, main_staging_dir, archive_url, short_build, |
| 160 | build_dir): |
| 161 | """Wrapper around devserver_util.GatherArtifactDownloads(). |
| 162 | |
| 163 | The wrapper allows mocking and overriding in derived classes. |
| 164 | """ |
| 165 | return devserver_util.GatherArtifactDownloads(main_staging_dir, archive_url, |
| 166 | short_build, build_dir) |
| 167 | |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 168 | def GetStatusOfBackgroundDownloads(self): |
| 169 | """Returns the status of the background downloads. |
| 170 | |
| 171 | This commands returns the status of the background downloads and blocks |
| 172 | until a status is returned. |
| 173 | """ |
| 174 | status = self._status_queue.get() |
| 175 | # In case anyone else is calling. |
| 176 | self._status_queue.put(status) |
| 177 | # It's possible we received an exception, if so, re-raise it here. |
| 178 | if isinstance(status, Exception): |
| 179 | raise status |
| 180 | |
| 181 | return status |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 182 | |
| 183 | |
| 184 | class SymbolDownloader(Downloader): |
| 185 | """Download and stage debug symbols for a build on the devsever. |
| 186 | |
| 187 | Given a URL to a build on the archive server: |
| 188 | |
| 189 | - Determine if the build already exists. |
| 190 | - Download and extract the debug symbols to a staging directory. |
| 191 | - Install symbols to static dir. |
| 192 | """ |
| 193 | |
| 194 | _DONE_FLAG = 'done' |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 195 | _LOG_TAG = 'SYMBOL_DOWNLOAD' |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 196 | |
| 197 | @staticmethod |
| 198 | def GenerateLockTag(target, short_build): |
| 199 | return '/'.join([target, short_build, 'symbols']) |
| 200 | |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 201 | def Download(self, archive_url, _background=False): |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 202 | """Downloads debug symbols for the build defined by the |archive_url|. |
| 203 | |
| 204 | The symbols will be downloaded synchronously |
| 205 | """ |
| 206 | # Parse archive_url into target and short_build. |
| 207 | # e.g. gs://chromeos-image-archive/{target}/{short_build} |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 208 | target, short_build = self.ParseUrl(archive_url) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 209 | |
| 210 | # Bind build_dir and staging_dir here so we can tell if we need to do any |
| 211 | # cleanup after an exception occurs before build_dir is set. |
| 212 | self._lock_tag = self.GenerateLockTag(target, short_build) |
| 213 | if self.SymbolsStaged(archive_url, self._static_dir): |
| 214 | cherrypy.log( |
| 215 | 'Symbols for build %s have already been staged.' % self._lock_tag, |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 216 | self._LOG_TAG) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 217 | return 'Success' |
| 218 | |
| 219 | try: |
| 220 | # Create Dev Server directory for this build and tell other Downloader |
| 221 | # instances we have processed this build. |
| 222 | self._build_dir = devserver_util.AcquireLock( |
| 223 | static_dir=self._static_dir, tag=self._lock_tag) |
| 224 | |
| 225 | self._staging_dir = tempfile.mkdtemp(suffix='_'.join([target, |
| 226 | short_build])) |
| 227 | cherrypy.log('Downloading debug symbols from %s' % archive_url, |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 228 | self._LOG_TAG) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 229 | |
| 230 | [symbol_artifact] = self.GatherArtifactDownloads( |
| 231 | self._staging_dir, archive_url, '', self._static_dir) |
| 232 | symbol_artifact.Download() |
| 233 | symbol_artifact.Stage() |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 234 | self.MarkSymbolsStaged() |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 235 | |
| 236 | except Exception: |
| 237 | # Release processing "lock", which will indicate to future runs that we |
| 238 | # did not succeed, and so they should try again. |
| 239 | if self._build_dir: |
| 240 | devserver_util.ReleaseLock(static_dir=self._static_dir, |
| 241 | tag=self._lock_tag) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 242 | raise |
Chris Masone | a22d938 | 2012-05-18 12:38:51 -0700 | [diff] [blame] | 243 | finally: |
| 244 | self._Cleanup() |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 245 | return 'Success' |
| 246 | |
| 247 | def GatherArtifactDownloads(self, temp_download_dir, archive_url, short_build, |
| 248 | static_dir): |
| 249 | """Call SymbolDownloader-appropriate artifact gathering method. |
| 250 | |
| 251 | @param temp_download_dir: the tempdir into which we're downloading artifacts |
| 252 | prior to staging them. |
| 253 | @param archive_url: the google storage url of the bucket where the debug |
| 254 | symbols for the desired build are stored. |
| 255 | @param short_build: IGNORED |
| 256 | @param staging_dir: the dir into which to stage the symbols |
| 257 | |
| 258 | @return an iterable of one DebugTarball pointing to the right debug symbols. |
| 259 | This is an iterable so that it's similar to GatherArtifactDownloads. |
| 260 | Also, it's possible that someday we might have more than one. |
| 261 | """ |
| 262 | return devserver_util.GatherSymbolArtifactDownloads(temp_download_dir, |
| 263 | archive_url, |
| 264 | static_dir) |
| 265 | |
| 266 | def MarkSymbolsStaged(self): |
| 267 | """Puts a flag file on disk to signal that symbols are staged.""" |
| 268 | with open(os.path.join(self._build_dir, self._DONE_FLAG), 'w') as flag: |
| 269 | flag.write(self._DONE_FLAG) |
| 270 | |
| 271 | def SymbolsStaged(self, archive_url, static_dir): |
| 272 | """Returns True if the build is already staged.""" |
Chris Sosa | cde6bf4 | 2012-05-31 18:36:39 -0700 | [diff] [blame] | 273 | target, short_build = self.ParseUrl(archive_url) |
Chris Masone | 816e38c | 2012-05-02 12:22:36 -0700 | [diff] [blame] | 274 | sub_directory = self.GenerateLockTag(target, short_build) |
| 275 | return os.path.isfile(os.path.join(static_dir, |
| 276 | sub_directory, |
| 277 | self._DONE_FLAG)) |