blob: 3d4ea23b8651cc3409c15da9ef4691a725735fb8 [file] [log] [blame]
Frank Farzan37761d12011-12-01 14:29:08 -08001#!/usr/bin/python
2#
Chris Sosa47a7d4e2012-03-28 11:26:55 -07003# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Frank Farzan37761d12011-12-01 14:29:08 -08004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import cherrypy
Chris Sosa47a7d4e2012-03-28 11:26:55 -07008import multiprocessing
Chris Sosa9164ca32012-03-28 11:04:50 -07009import os
Frank Farzan37761d12011-12-01 14:29:08 -080010import shutil
11import tempfile
12
13import devserver_util
14
15
16class Downloader(object):
17 """Download images to the devsever.
18
19 Given a URL to a build on the archive server:
20
21 - Determine if the build already exists.
22 - Download and extract the build to a staging directory.
23 - Package autotest tests.
24 - Install components to static dir.
25 """
26
Chris Masonea22d9382012-05-18 12:38:51 -070027 _LOG_TAG = 'DOWNLOAD'
28
Frank Farzan37761d12011-12-01 14:29:08 -080029 def __init__(self, static_dir):
30 self._static_dir = static_dir
Chris Sosa47a7d4e2012-03-28 11:26:55 -070031 self._build_dir = None
32 self._staging_dir = None
Chris Sosacde6bf42012-05-31 18:36:39 -070033 self._status_queue = multiprocessing.Queue(maxsize=1)
Chris Sosa47a7d4e2012-03-28 11:26:55 -070034 self._lock_tag = None
Chris Masone816e38c2012-05-02 12:22:36 -070035
36 @staticmethod
Chris Sosacde6bf42012-05-31 18:36:39 -070037 def ParseUrl(archive_url):
38 """Parse archive_url into its component parts.
Chris Masone816e38c2012-05-02 12:22:36 -070039
40 @param archive_url: a URL at which build artifacts are archived.
Chris Sosacde6bf42012-05-31 18:36:39 -070041 @return a tuple of (build target, short build name)
Chris Masone816e38c2012-05-02 12:22:36 -070042 """
Chris Masone816e38c2012-05-02 12:22:36 -070043 target, short_build = archive_url.rsplit('/', 2)[-2:]
Chris Sosacde6bf42012-05-31 18:36:39 -070044 return target, short_build
Chris Masone816e38c2012-05-02 12:22:36 -070045
46 @staticmethod
47 def GenerateLockTag(target, short_build):
48 """Generate a name for a lock scoped to this target/build pair.
49
50 @param target: the target the build was for.
51 @param short_build: short build name
52 @return a name to use with AcquireLock that will scope the lock.
53 """
54 return '/'.join([target, short_build])
Frank Farzan37761d12011-12-01 14:29:08 -080055
Chris Sosa9164ca32012-03-28 11:04:50 -070056 @staticmethod
57 def BuildStaged(archive_url, static_dir):
58 """Returns True if the build is already staged."""
Chris Sosacde6bf42012-05-31 18:36:39 -070059 target, short_build = Downloader.ParseUrl(archive_url)
Chris Masone816e38c2012-05-02 12:22:36 -070060 sub_directory = Downloader.GenerateLockTag(target, short_build)
Chris Sosa9164ca32012-03-28 11:04:50 -070061 return os.path.isdir(os.path.join(static_dir, sub_directory))
62
Chris Sosa47a7d4e2012-03-28 11:26:55 -070063 def Download(self, archive_url, background=False):
64 """Downloads the given build artifacts defined by the |archive_url|.
65
66 If background is set to True, will return back early before all artifacts
67 have been downloaded. The artifacts that can be backgrounded are all those
68 that are not set as synchronous.
Chris Masone816e38c2012-05-02 12:22:36 -070069
70 TODO: refactor this into a common Download method, once unit tests are
71 fixed up to make iterating on the code easier.
Chris Sosa47a7d4e2012-03-28 11:26:55 -070072 """
Chris Sosa9164ca32012-03-28 11:04:50 -070073 # Parse archive_url into target and short_build.
74 # e.g. gs://chromeos-image-archive/{target}/{short_build}
Chris Sosacde6bf42012-05-31 18:36:39 -070075 target, short_build = self.ParseUrl(archive_url)
76 # This should never happen. The Devserver should only try to call this
77 # method if no previous downloads have been staged for this archive_url.
78 assert not Downloader.BuildStaged(archive_url, self._static_dir)
Frank Farzan37761d12011-12-01 14:29:08 -080079 # Bind build_dir and staging_dir here so we can tell if we need to do any
80 # cleanup after an exception occurs before build_dir is set.
Chris Masone816e38c2012-05-02 12:22:36 -070081 self._lock_tag = self.GenerateLockTag(target, short_build)
Frank Farzan37761d12011-12-01 14:29:08 -080082 try:
83 # Create Dev Server directory for this build and tell other Downloader
84 # instances we have processed this build.
Chris Masone816e38c2012-05-02 12:22:36 -070085 self._build_dir = devserver_util.AcquireLock(
86 static_dir=self._static_dir, tag=self._lock_tag)
Frank Farzan37761d12011-12-01 14:29:08 -080087
Chris Sosa9164ca32012-03-28 11:04:50 -070088 self._staging_dir = tempfile.mkdtemp(suffix='_'.join([target,
89 short_build]))
Chris Masone816e38c2012-05-02 12:22:36 -070090 cherrypy.log('Gathering download requirements %s' % archive_url,
Chris Masonea22d9382012-05-18 12:38:51 -070091 self._LOG_TAG)
Chris Masone816e38c2012-05-02 12:22:36 -070092 artifacts = self.GatherArtifactDownloads(
93 self._staging_dir, archive_url, short_build, self._build_dir)
Chris Sosa47a7d4e2012-03-28 11:26:55 -070094 devserver_util.PrepareBuildDirectory(self._build_dir)
Frank Farzan37761d12011-12-01 14:29:08 -080095
Chris Sosa47a7d4e2012-03-28 11:26:55 -070096 cherrypy.log('Downloading foreground artifacts from %s' % archive_url,
Chris Masonea22d9382012-05-18 12:38:51 -070097 self._LOG_TAG)
Chris Sosa47a7d4e2012-03-28 11:26:55 -070098 background_artifacts = []
99 for artifact in artifacts:
100 if artifact.Synchronous():
101 artifact.Download()
102 artifact.Stage()
103 else:
104 background_artifacts.append(artifact)
Frank Farzan37761d12011-12-01 14:29:08 -0800105
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700106 if background:
Chris Sosacde6bf42012-05-31 18:36:39 -0700107 self._DownloadArtifactsInBackground(background_artifacts)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700108 else:
109 self._DownloadArtifactsSerially(background_artifacts)
110
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700111 except Exception, e:
Frank Farzan37761d12011-12-01 14:29:08 -0800112 # Release processing lock, which will remove build components directory
113 # so future runs can retry.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700114 if self._build_dir:
115 devserver_util.ReleaseLock(static_dir=self._static_dir,
116 tag=self._lock_tag)
117
118 self._status_queue.put(e)
119 self._Cleanup()
Frank Farzan37761d12011-12-01 14:29:08 -0800120 raise
Frank Farzan37761d12011-12-01 14:29:08 -0800121
122 return 'Success'
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700123
124 def _Cleanup(self):
125 """Cleans up the staging dir for this downloader instanfce."""
126 if self._staging_dir:
127 cherrypy.log('Cleaning up staging directory %s' % self._staging_dir,
Chris Masonea22d9382012-05-18 12:38:51 -0700128 self._LOG_TAG)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700129 shutil.rmtree(self._staging_dir)
130
131 self._staging_dir = None
132
133 def _DownloadArtifactsSerially(self, artifacts):
134 """Simple function to download all the given artifacts serially."""
Chris Masonea22d9382012-05-18 12:38:51 -0700135 cherrypy.log('Downloading background artifacts serially.', self._LOG_TAG)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700136 try:
137 for artifact in artifacts:
138 artifact.Download()
139 artifact.Stage()
140 except Exception, e:
141 self._status_queue.put(e)
142
143 # Release processing lock, which will remove build components directory
144 # so future runs can retry.
145 if self._build_dir:
146 devserver_util.ReleaseLock(static_dir=self._static_dir,
147 tag=self._lock_tag)
148 else:
149 self._status_queue.put('Success')
150 finally:
151 self._Cleanup()
152
Chris Sosacde6bf42012-05-31 18:36:39 -0700153 def _DownloadArtifactsInBackground(self, artifacts):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700154 """Downloads |artifacts| in the background and signals when complete."""
155 proc = multiprocessing.Process(target=self._DownloadArtifactsSerially,
156 args=(artifacts,))
Chris Sosab65973e2012-03-29 18:31:02 -0700157 proc.start()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700158
Chris Masone816e38c2012-05-02 12:22:36 -0700159 def GatherArtifactDownloads(self, main_staging_dir, archive_url, short_build,
160 build_dir):
161 """Wrapper around devserver_util.GatherArtifactDownloads().
162
163 The wrapper allows mocking and overriding in derived classes.
164 """
165 return devserver_util.GatherArtifactDownloads(main_staging_dir, archive_url,
166 short_build, build_dir)
167
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700168 def GetStatusOfBackgroundDownloads(self):
169 """Returns the status of the background downloads.
170
171 This commands returns the status of the background downloads and blocks
172 until a status is returned.
173 """
174 status = self._status_queue.get()
175 # In case anyone else is calling.
176 self._status_queue.put(status)
177 # It's possible we received an exception, if so, re-raise it here.
178 if isinstance(status, Exception):
179 raise status
180
181 return status
Chris Masone816e38c2012-05-02 12:22:36 -0700182
183
184class SymbolDownloader(Downloader):
185 """Download and stage debug symbols for a build on the devsever.
186
187 Given a URL to a build on the archive server:
188
189 - Determine if the build already exists.
190 - Download and extract the debug symbols to a staging directory.
191 - Install symbols to static dir.
192 """
193
194 _DONE_FLAG = 'done'
Chris Masonea22d9382012-05-18 12:38:51 -0700195 _LOG_TAG = 'SYMBOL_DOWNLOAD'
Chris Masone816e38c2012-05-02 12:22:36 -0700196
197 @staticmethod
198 def GenerateLockTag(target, short_build):
199 return '/'.join([target, short_build, 'symbols'])
200
Chris Sosacde6bf42012-05-31 18:36:39 -0700201 def Download(self, archive_url, _background=False):
Chris Masone816e38c2012-05-02 12:22:36 -0700202 """Downloads debug symbols for the build defined by the |archive_url|.
203
204 The symbols will be downloaded synchronously
205 """
206 # Parse archive_url into target and short_build.
207 # e.g. gs://chromeos-image-archive/{target}/{short_build}
Chris Sosacde6bf42012-05-31 18:36:39 -0700208 target, short_build = self.ParseUrl(archive_url)
Chris Masone816e38c2012-05-02 12:22:36 -0700209
210 # Bind build_dir and staging_dir here so we can tell if we need to do any
211 # cleanup after an exception occurs before build_dir is set.
212 self._lock_tag = self.GenerateLockTag(target, short_build)
213 if self.SymbolsStaged(archive_url, self._static_dir):
214 cherrypy.log(
215 'Symbols for build %s have already been staged.' % self._lock_tag,
Chris Masonea22d9382012-05-18 12:38:51 -0700216 self._LOG_TAG)
Chris Masone816e38c2012-05-02 12:22:36 -0700217 return 'Success'
218
219 try:
220 # Create Dev Server directory for this build and tell other Downloader
221 # instances we have processed this build.
222 self._build_dir = devserver_util.AcquireLock(
223 static_dir=self._static_dir, tag=self._lock_tag)
224
225 self._staging_dir = tempfile.mkdtemp(suffix='_'.join([target,
226 short_build]))
227 cherrypy.log('Downloading debug symbols from %s' % archive_url,
Chris Masonea22d9382012-05-18 12:38:51 -0700228 self._LOG_TAG)
Chris Masone816e38c2012-05-02 12:22:36 -0700229
230 [symbol_artifact] = self.GatherArtifactDownloads(
231 self._staging_dir, archive_url, '', self._static_dir)
232 symbol_artifact.Download()
233 symbol_artifact.Stage()
Chris Masonea22d9382012-05-18 12:38:51 -0700234 self.MarkSymbolsStaged()
Chris Masone816e38c2012-05-02 12:22:36 -0700235
236 except Exception:
237 # Release processing "lock", which will indicate to future runs that we
238 # did not succeed, and so they should try again.
239 if self._build_dir:
240 devserver_util.ReleaseLock(static_dir=self._static_dir,
241 tag=self._lock_tag)
Chris Masone816e38c2012-05-02 12:22:36 -0700242 raise
Chris Masonea22d9382012-05-18 12:38:51 -0700243 finally:
244 self._Cleanup()
Chris Masone816e38c2012-05-02 12:22:36 -0700245 return 'Success'
246
247 def GatherArtifactDownloads(self, temp_download_dir, archive_url, short_build,
248 static_dir):
249 """Call SymbolDownloader-appropriate artifact gathering method.
250
251 @param temp_download_dir: the tempdir into which we're downloading artifacts
252 prior to staging them.
253 @param archive_url: the google storage url of the bucket where the debug
254 symbols for the desired build are stored.
255 @param short_build: IGNORED
256 @param staging_dir: the dir into which to stage the symbols
257
258 @return an iterable of one DebugTarball pointing to the right debug symbols.
259 This is an iterable so that it's similar to GatherArtifactDownloads.
260 Also, it's possible that someday we might have more than one.
261 """
262 return devserver_util.GatherSymbolArtifactDownloads(temp_download_dir,
263 archive_url,
264 static_dir)
265
266 def MarkSymbolsStaged(self):
267 """Puts a flag file on disk to signal that symbols are staged."""
268 with open(os.path.join(self._build_dir, self._DONE_FLAG), 'w') as flag:
269 flag.write(self._DONE_FLAG)
270
271 def SymbolsStaged(self, archive_url, static_dir):
272 """Returns True if the build is already staged."""
Chris Sosacde6bf42012-05-31 18:36:39 -0700273 target, short_build = self.ParseUrl(archive_url)
Chris Masone816e38c2012-05-02 12:22:36 -0700274 sub_directory = self.GenerateLockTag(target, short_build)
275 return os.path.isfile(os.path.join(static_dir,
276 sub_directory,
277 self._DONE_FLAG))