blob: 85426f187e40f288e00b624f6a855767c75a45c3 [file] [log] [blame]
Gabe Black3b567202015-09-23 14:07:59 -07001#!/usr/bin/python2
Chris Sosa968a1062013-08-02 17:42:50 -07002
Chris Sosa47a7d4e2012-03-28 11:26:55 -07003# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module containing classes that wrap artifact downloads."""
8
Gabe Black3b567202015-09-23 14:07:59 -07009from __future__ import print_function
10
11import itertools
Chris Sosa47a7d4e2012-03-28 11:26:55 -070012import os
Dan Shi6e50c722013-08-19 15:05:06 -070013import pickle
Gilad Arnold950569b2013-08-27 14:38:01 -070014import re
Chris Sosa47a7d4e2012-03-28 11:26:55 -070015import shutil
16import subprocess
xixuan56252ff2017-03-09 15:40:31 -080017import traceback
Chris Sosa47a7d4e2012-03-28 11:26:55 -070018
Chris Sosa76e44b92013-01-31 12:11:38 -080019import artifact_info
20import common_util
joychen3cb228e2013-06-12 12:13:13 -070021import devserver_constants
Gilad Arnoldc65330c2012-09-20 15:17:48 -070022import log_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070023
Don Garrettfb15e322016-06-21 19:12:08 -070024# We do a number of things with args/kwargs arguments that confuse pylint.
25# pylint: disable=docstring-misnamed-args
Chris Sosa47a7d4e2012-03-28 11:26:55 -070026
Chris Sosa76e44b92013-01-31 12:11:38 -080027_AU_BASE = 'au'
28_NTON_DIR_SUFFIX = '_nton'
29_MTON_DIR_SUFFIX = '_mton'
30
31############ Actual filenames of artifacts in Google Storage ############
32
33AU_SUITE_FILE = 'au_control.tar.bz2'
Chris Sosa968a1062013-08-02 17:42:50 -070034PAYGEN_AU_SUITE_FILE_TEMPLATE = 'paygen_au_%(channel)s_control.tar.bz2'
Chris Sosa76e44b92013-01-31 12:11:38 -080035AUTOTEST_FILE = 'autotest.tar'
Simran Basiea0590d2014-10-29 11:31:26 -070036CONTROL_FILES_FILE = 'control_files.tar'
37AUTOTEST_PACKAGES_FILE = 'autotest_packages.tar'
Chris Sosa76e44b92013-01-31 12:11:38 -080038AUTOTEST_COMPRESSED_FILE = 'autotest.tar.bz2'
Simran Basi6459bba2015-02-04 14:47:23 -080039AUTOTEST_SERVER_PACKAGE_FILE = 'autotest_server_package.tar.bz2'
Chris Sosa76e44b92013-01-31 12:11:38 -080040DEBUG_SYMBOLS_FILE = 'debug.tgz'
Dan Shi55d0f972016-10-04 11:45:00 -070041DEBUG_SYMBOLS_ONLY_FILE = 'debug_breakpad.tar.xz'
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -080042FACTORY_FILE = 'ChromeOS-factory*.zip'
Mike Frysingera0e6a282016-09-01 17:29:08 -040043FACTORY_SHIM_FILE = 'factory_image.zip'
Chris Sosa76e44b92013-01-31 12:11:38 -080044FIRMWARE_FILE = 'firmware_from_source.tar.bz2'
45IMAGE_FILE = 'image.zip'
Chris Sosa76e44b92013-01-31 12:11:38 -080046TEST_SUITES_FILE = 'test_suites.tar.bz2'
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -080047BASE_IMAGE_FILE = 'chromiumos_base_image.tar.xz'
48TEST_IMAGE_FILE = 'chromiumos_test_image.tar.xz'
49RECOVERY_IMAGE_FILE = 'recovery_image.tar.xz'
Justin Giorgib7590522017-02-07 13:36:24 -080050LIBIOTA_TEST_BINARIES_FILE = 'test_binaries.tar.gz'
51LIBIOTA_BOARD_UTILS_FILE = 'board_utils.tar.gz'
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -080052
Gabe Black3b567202015-09-23 14:07:59 -070053############ Actual filenames of Android build artifacts ############
54
Dan Shiba4e00f2015-10-27 12:03:53 -070055ANDROID_IMAGE_ZIP = r'.*-img-[^-]*\.zip'
Gabe Black3b567202015-09-23 14:07:59 -070056ANDROID_RADIO_IMAGE = 'radio.img'
57ANDROID_BOOTLOADER_IMAGE = 'bootloader.img'
58ANDROID_FASTBOOT = 'fastboot'
59ANDROID_TEST_ZIP = r'[^-]*-tests-.*\.zip'
Dan Shi74136ae2015-12-01 14:40:06 -080060ANDROID_VENDOR_PARTITION_ZIP = r'[^-]*-vendor_partitions-.*\.zip'
Dan Shi6c2b2a22016-03-04 15:52:19 -080061ANDROID_AUTOTEST_SERVER_PACKAGE = r'[^-]*-autotest_server_package-.*\.tar.bz2'
62ANDROID_TEST_SUITES = r'[^-]*-test_suites-.*\.tar.bz2'
63ANDROID_CONTROL_FILES = r'[^-]*-autotest_control_files-.*\.tar'
Simran Basi05be7212016-03-16 13:08:23 -070064ANDROID_NATIVETESTS_FILE = r'[^-]*-brillo-tests-.*\.zip'
Ralph Nathan6c7fe5e2016-06-22 15:50:10 -070065ANDROID_CONTINUOUS_NATIVE_TESTS_FILE = r'[^-]*-continuous_native_tests-.*\.zip'
Justin Giorgibb32ac42016-08-04 10:34:35 -070066ANDROID_CONTINUOUS_INSTRUMENTATION_TESTS_FILE = (
67 r'[^-]*-continuous_instrumentation_tests-.*\.zip')
Justin Giorgi4faa8902016-08-19 19:54:30 -070068ANDROID_CTS_FILE = 'android-cts.zip'
Justin Giorgi576c1542016-06-24 08:24:20 -070069ANDROID_TARGET_FILES_ZIP = r'[^-]*-target_files-.*\.zip'
70ANDROID_DTB_ZIP = r'[^-]*-dtb-.*\.zip'
Chris Sosa76e44b92013-01-31 12:11:38 -080071
72_build_artifact_locks = common_util.LockDict()
Chris Sosa47a7d4e2012-03-28 11:26:55 -070073
74
75class ArtifactDownloadError(Exception):
76 """Error used to signify an issue processing an artifact."""
77 pass
78
79
Gabe Black3b567202015-09-23 14:07:59 -070080class ArtifactMeta(type):
81 """metaclass for an artifact type.
82
83 This metaclass is for class Artifact and its subclasses to have a meaningful
84 string composed of class name and the corresponding artifact name, e.g.,
85 `Artifact_full_payload`. This helps to better logging, refer to logging in
86 method Downloader.Download.
87 """
88
89 ARTIFACT_NAME = None
90
91 def __str__(cls):
92 return '%s_%s' % (cls.__name__, cls.ARTIFACT_NAME)
93
94 def __repr__(cls):
95 return str(cls)
96
97
98class Artifact(log_util.Loggable):
99 """Wrapper around an artifact to download using a fetcher.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700100
101 The purpose of this class is to download objects from Google Storage
102 and install them to a local directory. There are two main functions, one to
103 download/prepare the artifacts in to a temporary staging area and the second
104 to stage it into its final destination.
Chris Sosa76e44b92013-01-31 12:11:38 -0800105
Gilad Arnold950569b2013-08-27 14:38:01 -0700106 IMPORTANT! (i) `name' is a glob expression by default (and not a regex), be
107 attentive when adding new artifacts; (ii) name matching semantics differ
108 between a glob (full name string match) and a regex (partial match).
109
Chris Sosa76e44b92013-01-31 12:11:38 -0800110 Class members:
Gabe Black3b567202015-09-23 14:07:59 -0700111 fetcher: An object which knows how to fetch the artifact.
Gilad Arnold950569b2013-08-27 14:38:01 -0700112 name: Name given for artifact; in fact, it is a pattern that captures the
113 names of files contained in the artifact. This can either be an
114 ordinary shell-style glob (the default), or a regular expression (if
115 is_regex_name is True).
116 is_regex_name: Whether the name value is a regex (default: glob).
Chris Sosa76e44b92013-01-31 12:11:38 -0800117 build: The version of the build i.e. R26-2342.0.0.
118 marker_name: Name used to define the lock marker for the artifacts to
119 prevent it from being re-downloaded. By default based on name
120 but can be overriden by children.
Dan Shi6e50c722013-08-19 15:05:06 -0700121 exception_file_path: Path to a file containing the serialized exception,
122 which was raised in Process method. The file is located
123 in the parent folder of install_dir, since the
124 install_dir will be deleted if the build does not
125 existed.
joychen0a8e34e2013-06-24 17:58:36 -0700126 install_path: Path to artifact.
Gabe Black3b567202015-09-23 14:07:59 -0700127 install_subdir: Directory within install_path where the artifact is actually
128 stored.
Chris Sosa76e44b92013-01-31 12:11:38 -0800129 install_dir: The final location where the artifact should be staged to.
130 single_name: If True the name given should only match one item. Note, if not
131 True, self.name will become a list of items returned.
Gilad Arnold1638d822013-11-07 23:38:16 -0800132 installed_files: A list of files that were the final result of downloading
133 and setting up the artifact.
134 store_installed_files: Whether the list of installed files is stored in the
135 marker file.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700136 """
Gilad Arnold950569b2013-08-27 14:38:01 -0700137
Gabe Black3b567202015-09-23 14:07:59 -0700138 __metaclass__ = ArtifactMeta
139
140 def __init__(self, name, install_dir, build, install_subdir='',
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800141 is_regex_name=False, optional_name=None):
Gilad Arnold950569b2013-08-27 14:38:01 -0700142 """Constructor.
143
144 Args:
Chris Sosa76e44b92013-01-31 12:11:38 -0800145 install_dir: Where to install the artifact.
Chris Sosa76e44b92013-01-31 12:11:38 -0800146 name: Identifying name to be used to find/store the artifact.
147 build: The name of the build e.g. board/release.
Gabe Black3b567202015-09-23 14:07:59 -0700148 install_subdir: Directory within install_path where the artifact is
149 actually stored.
Gilad Arnold950569b2013-08-27 14:38:01 -0700150 is_regex_name: Whether the name pattern is a regex (default: glob).
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800151 optional_name: An alternative name to find the artifact, which can lead
152 to faster download. Unlike |name|, there is no guarantee that an
153 artifact named |optional_name| is/will be on Google Storage. If it
154 exists, we download it. Otherwise, we fall back to wait for |name|.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700155 """
Gabe Black3b567202015-09-23 14:07:59 -0700156 super(Artifact, self).__init__()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700157
Chris Sosa76e44b92013-01-31 12:11:38 -0800158 # In-memory lock to keep the devserver from colliding with itself while
159 # attempting to stage the same artifact.
160 self._process_lock = None
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700161
Chris Sosa76e44b92013-01-31 12:11:38 -0800162 self.name = name
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800163 self.optional_name = optional_name
Gilad Arnold950569b2013-08-27 14:38:01 -0700164 self.is_regex_name = is_regex_name
Chris Sosa76e44b92013-01-31 12:11:38 -0800165 self.build = build
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700166
Chris Sosa76e44b92013-01-31 12:11:38 -0800167 self.marker_name = '.' + self._SanitizeName(name)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700168
Dan Shi6e50c722013-08-19 15:05:06 -0700169 exception_file_name = ('.' + self._SanitizeName(build) + self.marker_name +
170 '.exception')
171 # The exception file needs to be located in parent folder, since the
172 # install_dir will be deleted is the build does not exist.
173 self.exception_file_path = os.path.join(os.path.dirname(install_dir),
Gilad Arnold950569b2013-08-27 14:38:01 -0700174 exception_file_name)
Dan Shi6e50c722013-08-19 15:05:06 -0700175
joychen0a8e34e2013-06-24 17:58:36 -0700176 self.install_path = None
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700177
Chris Sosa76e44b92013-01-31 12:11:38 -0800178 self.install_dir = install_dir
Gabe Black3b567202015-09-23 14:07:59 -0700179 self.install_subdir = install_subdir
Chris Sosa76e44b92013-01-31 12:11:38 -0800180
181 self.single_name = True
182
Gilad Arnold1638d822013-11-07 23:38:16 -0800183 self.installed_files = []
184 self.store_installed_files = True
185
Chris Sosa76e44b92013-01-31 12:11:38 -0800186 @staticmethod
187 def _SanitizeName(name):
188 """Sanitizes name to be used for creating a file on the filesystem.
189
190 '.','/' and '*' have special meaning in FS lingo. Replace them with words.
Gilad Arnold950569b2013-08-27 14:38:01 -0700191
192 Args:
193 name: A file name/path.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800194
Gilad Arnold950569b2013-08-27 14:38:01 -0700195 Returns:
196 The sanitized name/path.
Chris Sosa76e44b92013-01-31 12:11:38 -0800197 """
198 return name.replace('*', 'STAR').replace('.', 'DOT').replace('/', 'SLASH')
199
Dan Shif8eb0d12013-08-01 17:52:06 -0700200 def ArtifactStaged(self):
Gilad Arnold1638d822013-11-07 23:38:16 -0800201 """Returns True if artifact is already staged.
202
203 This checks for (1) presence of the artifact marker file, and (2) the
204 presence of each installed file listed in this marker. Both must hold for
205 the artifact to be considered staged. Note that this method is safe for use
206 even if the artifacts were not stageed by this instance, as it is assumed
Gabe Black3b567202015-09-23 14:07:59 -0700207 that any Artifact instance that did the staging wrote the list of
Gilad Arnold1638d822013-11-07 23:38:16 -0800208 files actually installed into the marker.
209 """
210 marker_file = os.path.join(self.install_dir, self.marker_name)
211
212 # If the marker is missing, it's definitely not staged.
213 if not os.path.exists(marker_file):
Aviv Keshet57d18172016-06-18 20:39:09 -0700214 self._Log('No marker file, %s is not staged.', self)
Gilad Arnold1638d822013-11-07 23:38:16 -0800215 return False
216
217 # We want to ensure that every file listed in the marker is actually there.
218 if self.store_installed_files:
219 with open(marker_file) as f:
220 files = [line.strip() for line in f]
221
222 # Check to see if any of the purportedly installed files are missing, in
223 # which case the marker is outdated and should be removed.
224 missing_files = [fname for fname in files if not os.path.exists(fname)]
225 if missing_files:
226 self._Log('***ATTENTION*** %s files listed in %s are missing:\n%s',
227 'All' if len(files) == len(missing_files) else 'Some',
228 marker_file, '\n'.join(missing_files))
229 os.remove(marker_file)
Aviv Keshet57d18172016-06-18 20:39:09 -0700230 self._Log('Missing files, %s is not staged.', self)
Gilad Arnold1638d822013-11-07 23:38:16 -0800231 return False
232
Aviv Keshet57d18172016-06-18 20:39:09 -0700233 self._Log('ArtifactStaged() -> yes, %s is staged.', self)
Gilad Arnold1638d822013-11-07 23:38:16 -0800234 return True
Chris Sosa76e44b92013-01-31 12:11:38 -0800235
236 def _MarkArtifactStaged(self):
237 """Marks the artifact as staged."""
238 with open(os.path.join(self.install_dir, self.marker_name), 'w') as f:
Gilad Arnold1638d822013-11-07 23:38:16 -0800239 f.write('\n'.join(self.installed_files))
Chris Sosa76e44b92013-01-31 12:11:38 -0800240
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800241 def _UpdateName(self, names):
242 if self.single_name and len(names) > 1:
243 raise ArtifactDownloadError('Too many artifacts match %s' % self.name)
Chris Sosa76e44b92013-01-31 12:11:38 -0800244
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800245 self.name = names[0]
Chris Sosa76e44b92013-01-31 12:11:38 -0800246
joychen0a8e34e2013-06-24 17:58:36 -0700247 def _Setup(self):
Gilad Arnold1638d822013-11-07 23:38:16 -0800248 """Process the downloaded content, update the list of installed files."""
249 # In this primitive case, what was downloaded (has to be a single file) is
250 # what's installed.
251 self.installed_files = [self.install_path]
joychen0a8e34e2013-06-24 17:58:36 -0700252
Dan Shi6e50c722013-08-19 15:05:06 -0700253 def _ClearException(self):
254 """Delete any existing exception saved for this artifact."""
255 if os.path.exists(self.exception_file_path):
256 os.remove(self.exception_file_path)
257
258 def _SaveException(self, e):
xixuan56252ff2017-03-09 15:40:31 -0800259 """Save the exception and traceback to a file for downloader.IsStaged.
Dan Shi6e50c722013-08-19 15:05:06 -0700260
Gilad Arnold950569b2013-08-27 14:38:01 -0700261 Args:
262 e: Exception object to be saved.
Dan Shi6e50c722013-08-19 15:05:06 -0700263 """
264 with open(self.exception_file_path, 'w') as f:
xixuan56252ff2017-03-09 15:40:31 -0800265 pickle.dump('%s\n%s' % (e, str(traceback.format_exc())), f)
Dan Shi6e50c722013-08-19 15:05:06 -0700266
267 def GetException(self):
268 """Retrieve any exception that was raised in Process method.
269
Gilad Arnold950569b2013-08-27 14:38:01 -0700270 Returns:
271 An Exception object that was raised when trying to process the artifact.
272 Return None if no exception was found.
Dan Shi6e50c722013-08-19 15:05:06 -0700273 """
274 if not os.path.exists(self.exception_file_path):
275 return None
276 with open(self.exception_file_path, 'r') as f:
277 return pickle.load(f)
Chris Sosa76e44b92013-01-31 12:11:38 -0800278
Gabe Black3b567202015-09-23 14:07:59 -0700279 def Process(self, downloader, no_wait):
Chris Sosa76e44b92013-01-31 12:11:38 -0800280 """Main call point to all artifacts. Downloads and Stages artifact.
281
282 Downloads and Stages artifact from Google Storage to the install directory
283 specified in the constructor. It multi-thread safe and does not overwrite
284 the artifact if it's already been downloaded or being downloaded. After
285 processing, leaves behind a marker to indicate to future invocations that
286 the artifact has already been staged based on the name of the artifact.
287
288 Do not override as it modifies important private variables, ensures thread
289 safety, and maintains cache semantics.
290
291 Note: this may be a blocking call when the artifact is already in the
292 process of being staged.
293
294 Args:
Gabe Black3b567202015-09-23 14:07:59 -0700295 downloader: A downloader instance containing the logic to download
296 artifacts.
Chris Sosa76e44b92013-01-31 12:11:38 -0800297 no_wait: If True, don't block waiting for artifact to exist if we fail to
298 immediately find it.
299
300 Raises:
301 ArtifactDownloadError: If the artifact fails to download from Google
302 Storage for any reason or that the regexp
303 defined by name is not specific enough.
304 """
305 if not self._process_lock:
306 self._process_lock = _build_artifact_locks.lock(
307 os.path.join(self.install_dir, self.name))
308
Gabe Black3b567202015-09-23 14:07:59 -0700309 real_install_dir = os.path.join(self.install_dir, self.install_subdir)
Chris Sosa76e44b92013-01-31 12:11:38 -0800310 with self._process_lock:
Gabe Black3b567202015-09-23 14:07:59 -0700311 common_util.MkDirP(real_install_dir)
Dan Shif8eb0d12013-08-01 17:52:06 -0700312 if not self.ArtifactStaged():
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800313 # Delete any existing exception saved for this artifact.
314 self._ClearException()
315 found_artifact = False
316 if self.optional_name:
317 try:
Gabe Black3b567202015-09-23 14:07:59 -0700318 # Check if the artifact named |optional_name| exists.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800319 # Because this artifact may not always exist, don't bother
320 # to wait for it (set timeout=1).
Gabe Black3b567202015-09-23 14:07:59 -0700321 new_names = downloader.Wait(
322 self.optional_name, self.is_regex_name, timeout=1)
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800323 self._UpdateName(new_names)
324
325 except ArtifactDownloadError:
326 self._Log('Unable to download %s; fall back to download %s',
327 self.optional_name, self.name)
328 else:
329 found_artifact = True
330
Dan Shi6e50c722013-08-19 15:05:06 -0700331 try:
Dan Shi6e50c722013-08-19 15:05:06 -0700332 # If the artifact should already have been uploaded, don't waste
333 # cycles waiting around for it to exist.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800334 if not found_artifact:
335 timeout = 1 if no_wait else 10
Gabe Black3b567202015-09-23 14:07:59 -0700336 new_names = downloader.Wait(
337 self.name, self.is_regex_name, timeout)
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800338 self._UpdateName(new_names)
339
340 self._Log('Downloading file %s', self.name)
Gabe Black3b567202015-09-23 14:07:59 -0700341 self.install_path = downloader.Fetch(self.name, real_install_dir)
Dan Shi6e50c722013-08-19 15:05:06 -0700342 self._Setup()
343 self._MarkArtifactStaged()
344 except Exception as e:
345 # Save the exception to a file for downloader.IsStaged to retrieve.
346 self._SaveException(e)
Gilad Arnold02dc6552013-11-14 11:27:54 -0800347
348 # Convert an unknown exception into an ArtifactDownloadError.
349 if type(e) is ArtifactDownloadError:
350 raise
351 else:
352 raise ArtifactDownloadError('An error occurred: %s' % e)
Chris Sosa76e44b92013-01-31 12:11:38 -0800353 else:
354 self._Log('%s is already staged.', self)
355
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700356 def __str__(self):
357 """String representation for the download."""
Gabe Black3b567202015-09-23 14:07:59 -0700358 return '%s->%s' % (self.name, self.install_dir)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700359
Chris Sosab26b1202013-08-16 16:40:55 -0700360 def __repr__(self):
361 return str(self)
362
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700363
Gabe Black3b567202015-09-23 14:07:59 -0700364class AUTestPayload(Artifact):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700365 """Wrapper for AUTest delta payloads which need additional setup."""
Gilad Arnold950569b2013-08-27 14:38:01 -0700366
joychen0a8e34e2013-06-24 17:58:36 -0700367 def _Setup(self):
Gabe Black3b567202015-09-23 14:07:59 -0700368 super(AUTestPayload, self)._Setup()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700369
Chris Sosa76e44b92013-01-31 12:11:38 -0800370 # Rename to update.gz.
Gabe Black3b567202015-09-23 14:07:59 -0700371 install_path = os.path.join(self.install_dir, self.install_subdir,
372 self.name)
373 new_install_path = os.path.join(self.install_dir, self.install_subdir,
joychen7c2054a2013-07-25 11:14:07 -0700374 devserver_constants.UPDATE_FILE)
Chris Sosa76e44b92013-01-31 12:11:38 -0800375 shutil.move(install_path, new_install_path)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700376
Gilad Arnold1638d822013-11-07 23:38:16 -0800377 # Reflect the rename in the list of installed files.
378 self.installed_files.remove(install_path)
379 self.installed_files = [new_install_path]
380
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700381
Gabe Black3b567202015-09-23 14:07:59 -0700382class DeltaPayloadBase(AUTestPayload):
Chris Sosa76e44b92013-01-31 12:11:38 -0800383 """Delta payloads from the archive_url.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700384
Gabe Black3b567202015-09-23 14:07:59 -0700385 These artifacts are super strange. They custom handle directories and
386 pull in all delta payloads. We can't specify exactly what we want
Chris Sosa76e44b92013-01-31 12:11:38 -0800387 because unlike other artifacts, this one does not conform to something a
388 client might know. The client doesn't know the version of n-1 or whether it
389 was even generated.
Gilad Arnold950569b2013-08-27 14:38:01 -0700390
391 IMPORTANT! Note that this artifact simply ignores the `name' argument because
Gabe Black3b567202015-09-23 14:07:59 -0700392 that name is derived internally.
Chris Sosa76e44b92013-01-31 12:11:38 -0800393 """
Gilad Arnold950569b2013-08-27 14:38:01 -0700394
joychen0a8e34e2013-06-24 17:58:36 -0700395 def _Setup(self):
Gabe Black3b567202015-09-23 14:07:59 -0700396 super(DeltaPayloadBase, self)._Setup()
397 # Setup symlink so that AU will work for this payload.
398 stateful_update_symlink = os.path.join(
399 self.install_dir, self.install_subdir,
400 devserver_constants.STATEFUL_FILE)
401 os.symlink(os.path.join(os.pardir, os.pardir,
402 devserver_constants.STATEFUL_FILE),
403 stateful_update_symlink)
404 self.installed_files.append(stateful_update_symlink)
Yu-Ju Honge61cbe92012-07-10 14:10:26 -0700405
Chris Sosa76e44b92013-01-31 12:11:38 -0800406
Gabe Black3b567202015-09-23 14:07:59 -0700407class BundledArtifact(Artifact):
Chris Sosa76e44b92013-01-31 12:11:38 -0800408 """A single build artifact bundle e.g. zip file or tar file."""
Chris Sosa76e44b92013-01-31 12:11:38 -0800409
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800410 def __init__(self, *args, **kwargs):
Gabe Black3b567202015-09-23 14:07:59 -0700411 """Takes Artifact args with some additional ones.
Gilad Arnold950569b2013-08-27 14:38:01 -0700412
413 Args:
Gabe Black3b567202015-09-23 14:07:59 -0700414 *args: See Artifact documentation.
415 **kwargs: See Artifact documentation.
Gilad Arnold950569b2013-08-27 14:38:01 -0700416 files_to_extract: A list of files to extract. If set to None, extract
417 all files.
418 exclude: A list of files to exclude. If None, no files are excluded.
Chris Sosa76e44b92013-01-31 12:11:38 -0800419 """
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800420 self._files_to_extract = kwargs.pop('files_to_extract', None)
421 self._exclude = kwargs.pop('exclude', None)
Gabe Black3b567202015-09-23 14:07:59 -0700422 super(BundledArtifact, self).__init__(*args, **kwargs)
Chris Sosa76e44b92013-01-31 12:11:38 -0800423
424 # We modify the marker so that it is unique to what was staged.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800425 if self._files_to_extract:
Chris Sosa76e44b92013-01-31 12:11:38 -0800426 self.marker_name = self._SanitizeName(
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800427 '_'.join(['.' + self.name] + self._files_to_extract))
Chris Sosa76e44b92013-01-31 12:11:38 -0800428
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800429 def _RunUnzip(self, list_only):
430 # Unzip is weird. It expects its args before any excludes and expects its
431 # excludes in a list following the -x.
432 cmd = ['unzip', '-qql' if list_only else '-o', self.install_path]
433 if not list_only:
434 cmd += ['-d', self.install_dir]
Chris Sosa76e44b92013-01-31 12:11:38 -0800435
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800436 if self._files_to_extract:
437 cmd.extend(self._files_to_extract)
438
439 if self._exclude:
440 cmd.append('-x')
441 cmd.extend(self._exclude)
442
443 try:
444 return subprocess.check_output(cmd).strip('\n').splitlines()
445 except subprocess.CalledProcessError, e:
446 raise ArtifactDownloadError(
447 'An error occurred when attempting to unzip %s:\n%s' %
448 (self.install_path, e))
Chris Sosa76e44b92013-01-31 12:11:38 -0800449
joychen0a8e34e2013-06-24 17:58:36 -0700450 def _Setup(self):
Gilad Arnold1638d822013-11-07 23:38:16 -0800451 extract_result = self._Extract()
452 if self.store_installed_files:
453 # List both the archive and the extracted files.
454 self.installed_files.append(self.install_path)
455 self.installed_files.extend(extract_result)
Chris Sosa76e44b92013-01-31 12:11:38 -0800456
Chris Sosa76e44b92013-01-31 12:11:38 -0800457 def _Extract(self):
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800458 """Extracts files into the install path."""
459 if self.name.endswith('.zip'):
460 return self._ExtractZipfile()
461 else:
462 return self._ExtractTarball()
463
464 def _ExtractZipfile(self):
465 """Extracts a zip file using unzip."""
466 file_list = [os.path.join(self.install_dir, line[30:].strip())
467 for line in self._RunUnzip(True)
468 if not line.endswith('/')]
469 if file_list:
470 self._RunUnzip(False)
471
472 return file_list
473
474 def _ExtractTarball(self):
Chris Sosa76e44b92013-01-31 12:11:38 -0800475 """Extracts a tarball using tar.
476
477 Detects whether the tarball is compressed or not based on the file
478 extension and extracts the tarball into the install_path.
479 """
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700480 try:
Gilad Arnold1638d822013-11-07 23:38:16 -0800481 return common_util.ExtractTarball(self.install_path, self.install_dir,
482 files_to_extract=self._files_to_extract,
483 excluded_files=self._exclude,
484 return_extracted_files=True)
Simran Basi4baad082013-02-14 13:39:18 -0800485 except common_util.CommonUtilError as e:
486 raise ArtifactDownloadError(str(e))
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700487
488
Gabe Black3b567202015-09-23 14:07:59 -0700489class AutotestTarball(BundledArtifact):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700490 """Wrapper around the autotest tarball to download from gsutil."""
491
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800492 def __init__(self, *args, **kwargs):
Gabe Black3b567202015-09-23 14:07:59 -0700493 super(AutotestTarball, self).__init__(*args, **kwargs)
Gilad Arnold1638d822013-11-07 23:38:16 -0800494 # We don't store/check explicit file lists in Autotest tarball markers;
495 # this can get huge and unwieldy, and generally make little sense.
496 self.store_installed_files = False
497
joychen0a8e34e2013-06-24 17:58:36 -0700498 def _Setup(self):
Chris Sosa76e44b92013-01-31 12:11:38 -0800499 """Extracts the tarball into the install path excluding test suites."""
Gabe Black3b567202015-09-23 14:07:59 -0700500 super(AutotestTarball, self)._Setup()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700501
Chris Sosa76e44b92013-01-31 12:11:38 -0800502 # Deal with older autotest packages that may not be bundled.
joychen3cb228e2013-06-12 12:13:13 -0700503 autotest_dir = os.path.join(self.install_dir,
504 devserver_constants.AUTOTEST_DIR)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700505 autotest_pkgs_dir = os.path.join(autotest_dir, 'packages')
506 if not os.path.exists(autotest_pkgs_dir):
507 os.makedirs(autotest_pkgs_dir)
508
509 if not os.path.exists(os.path.join(autotest_pkgs_dir, 'packages.checksum')):
Prashant Malani20e83712017-02-28 01:30:41 -0800510 cmd = ['autotest/utils/packager.py', '--action=upload', '--repository',
Chris Sosa76e44b92013-01-31 12:11:38 -0800511 autotest_pkgs_dir, '--all']
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700512 try:
joychen0a8e34e2013-06-24 17:58:36 -0700513 subprocess.check_call(cmd, cwd=self.install_dir)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700514 except subprocess.CalledProcessError, e:
Chris Sosa76e44b92013-01-31 12:11:38 -0800515 raise ArtifactDownloadError(
516 'Failed to create autotest packages!:\n%s' % e)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700517 else:
Gilad Arnoldf5843132012-09-25 00:31:20 -0700518 self._Log('Using pre-generated packages from autotest')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700519
Chris Masone816e38c2012-05-02 12:22:36 -0700520
Gabe Black3b567202015-09-23 14:07:59 -0700521def _CreateNewArtifact(tag, base, name, *fixed_args, **fixed_kwargs):
522 """Get a data wrapper that describes an artifact's implementation.
Gilad Arnold950569b2013-08-27 14:38:01 -0700523
Gabe Black3b567202015-09-23 14:07:59 -0700524 Args:
525 tag: Tag of the artifact, defined in artifact_info.
526 base: Class of the artifact, e.g., BundledArtifact.
527 name: Name of the artifact, e.g., image.zip.
528 *fixed_args: Fixed arguments that are additional to the one used in base
529 class.
530 **fixed_kwargs: Fixed keyword arguments that are additional to the one used
531 in base class.
Chris Sosa76e44b92013-01-31 12:11:38 -0800532
Gabe Black3b567202015-09-23 14:07:59 -0700533 Returns:
534 A data wrapper that describes an artifact's implementation.
Gabe Black3b567202015-09-23 14:07:59 -0700535 """
Don Garrettfb15e322016-06-21 19:12:08 -0700536 # pylint: disable=super-on-old-class
Gabe Black3b567202015-09-23 14:07:59 -0700537 class NewArtifact(base):
538 """A data wrapper that describes an artifact's implementation."""
539 ARTIFACT_TAG = tag
540 ARTIFACT_NAME = name
541
542 def __init__(self, *args, **kwargs):
543 all_args = fixed_args + args
544 all_kwargs = {}
545 all_kwargs.update(fixed_kwargs)
546 all_kwargs.update(kwargs)
547 super(NewArtifact, self).__init__(self.ARTIFACT_NAME,
548 *all_args, **all_kwargs)
549
550 NewArtifact.__name__ = base.__name__
551 return NewArtifact
Chris Sosa968a1062013-08-02 17:42:50 -0700552
Chris Sosa76e44b92013-01-31 12:11:38 -0800553
Gabe Black3b567202015-09-23 14:07:59 -0700554# TODO(dshi): Refactor the code here to split out the logic of creating the
555# artifacts mapping to a different module.
556chromeos_artifact_map = {}
Chris Sosa76e44b92013-01-31 12:11:38 -0800557
Chris Sosa76e44b92013-01-31 12:11:38 -0800558
Gabe Black3b567202015-09-23 14:07:59 -0700559def _AddCrOSArtifact(tag, base, name, *fixed_args, **fixed_kwargs):
Don Garrettfb15e322016-06-21 19:12:08 -0700560 """Add a data wrapper for ChromeOS artifacts.
561
562 Add a data wrapper that describes a ChromeOS artifact's implementation to
Gabe Black3b567202015-09-23 14:07:59 -0700563 chromeos_artifact_map.
564 """
565 artifact = _CreateNewArtifact(tag, base, name, *fixed_args, **fixed_kwargs)
566 chromeos_artifact_map.setdefault(tag, []).append(artifact)
Chris Sosa76e44b92013-01-31 12:11:38 -0800567
beepsc3d0f872013-07-31 21:50:40 -0700568
Gabe Black3b567202015-09-23 14:07:59 -0700569_AddCrOSArtifact(artifact_info.FULL_PAYLOAD, AUTestPayload, '*_full_*')
570
571
572class DeltaPayloadNtoN(DeltaPayloadBase):
573 """ChromeOS Delta payload artifact for updating from version N to N."""
574 ARTIFACT_TAG = artifact_info.DELTA_PAYLOADS
575 ARTIFACT_NAME = 'NOT_APPLICABLE'
576
577 def __init__(self, install_dir, build, *args, **kwargs):
578 name = 'chromeos_%s*_delta_*' % build
579 install_subdir = os.path.join(_AU_BASE, build + _NTON_DIR_SUFFIX)
580 super(DeltaPayloadNtoN, self).__init__(name, install_dir, build, *args,
581 install_subdir=install_subdir,
582 **kwargs)
583
584
585class DeltaPayloadMtoN(DeltaPayloadBase):
586 """ChromeOS Delta payload artifact for updating from version M to N."""
587 ARTIFACT_TAG = artifact_info.DELTA_PAYLOADS
588 ARTIFACT_NAME = 'NOT_APPLICABLE'
589
590 def __init__(self, install_dir, build, *args, **kwargs):
591 name = ('chromeos_(?!%s).*_delta_.*' % re.escape(build))
592 install_subdir = os.path.join(_AU_BASE, build + _MTON_DIR_SUFFIX)
593 super(DeltaPayloadMtoN, self).__init__(name, install_dir, build, *args,
594 install_subdir=install_subdir,
595 is_regex_name=True, **kwargs)
596
597
598chromeos_artifact_map[artifact_info.DELTA_PAYLOADS] = [DeltaPayloadNtoN,
599 DeltaPayloadMtoN]
600
601
602_AddCrOSArtifact(artifact_info.STATEFUL_PAYLOAD, Artifact,
603 devserver_constants.STATEFUL_FILE)
604_AddCrOSArtifact(artifact_info.BASE_IMAGE, BundledArtifact, IMAGE_FILE,
605 optional_name=BASE_IMAGE_FILE,
606 files_to_extract=[devserver_constants.BASE_IMAGE_FILE])
607_AddCrOSArtifact(artifact_info.RECOVERY_IMAGE, BundledArtifact, IMAGE_FILE,
608 optional_name=RECOVERY_IMAGE_FILE,
609 files_to_extract=[devserver_constants.RECOVERY_IMAGE_FILE])
610_AddCrOSArtifact(artifact_info.DEV_IMAGE, BundledArtifact, IMAGE_FILE,
611 files_to_extract=[devserver_constants.IMAGE_FILE])
612_AddCrOSArtifact(artifact_info.TEST_IMAGE, BundledArtifact, IMAGE_FILE,
613 optional_name=TEST_IMAGE_FILE,
614 files_to_extract=[devserver_constants.TEST_IMAGE_FILE])
615_AddCrOSArtifact(artifact_info.AUTOTEST, AutotestTarball, AUTOTEST_FILE,
616 files_to_extract=None, exclude=['autotest/test_suites'])
617_AddCrOSArtifact(artifact_info.CONTROL_FILES, BundledArtifact,
618 CONTROL_FILES_FILE)
619_AddCrOSArtifact(artifact_info.AUTOTEST_PACKAGES, AutotestTarball,
620 AUTOTEST_PACKAGES_FILE)
621_AddCrOSArtifact(artifact_info.TEST_SUITES, BundledArtifact, TEST_SUITES_FILE)
622_AddCrOSArtifact(artifact_info.AU_SUITE, BundledArtifact, AU_SUITE_FILE)
623_AddCrOSArtifact(artifact_info.AUTOTEST_SERVER_PACKAGE, Artifact,
624 AUTOTEST_SERVER_PACKAGE_FILE)
625_AddCrOSArtifact(artifact_info.FIRMWARE, Artifact, FIRMWARE_FILE)
626_AddCrOSArtifact(artifact_info.SYMBOLS, BundledArtifact, DEBUG_SYMBOLS_FILE,
627 files_to_extract=['debug/breakpad'])
Dan Shi55d0f972016-10-04 11:45:00 -0700628_AddCrOSArtifact(artifact_info.SYMBOLS_ONLY, BundledArtifact,
629 DEBUG_SYMBOLS_ONLY_FILE,
630 files_to_extract=['debug/breakpad'])
Gabe Black3b567202015-09-23 14:07:59 -0700631_AddCrOSArtifact(artifact_info.FACTORY_IMAGE, BundledArtifact, FACTORY_FILE,
632 files_to_extract=[devserver_constants.FACTORY_IMAGE_FILE])
Mike Frysingera0e6a282016-09-01 17:29:08 -0400633_AddCrOSArtifact(artifact_info.FACTORY_SHIM_IMAGE, BundledArtifact,
634 FACTORY_SHIM_FILE,
635 files_to_extract=[devserver_constants.FACTORY_SHIM_IMAGE_FILE])
Chris Sosa76e44b92013-01-31 12:11:38 -0800636
Chris Sosa968a1062013-08-02 17:42:50 -0700637# Add all the paygen_au artifacts in one go.
Gabe Black3b567202015-09-23 14:07:59 -0700638for c in devserver_constants.CHANNELS:
639 _AddCrOSArtifact(artifact_info.PAYGEN_AU_SUITE_TEMPLATE % {'channel': c},
640 BundledArtifact,
641 PAYGEN_AU_SUITE_FILE_TEMPLATE % {'channel': c})
642
Justin Giorgib7590522017-02-07 13:36:24 -0800643#### Libiota Artifacts ####
644_AddCrOSArtifact(artifact_info.LIBIOTA_TEST_BINARIES, Artifact,
645 LIBIOTA_TEST_BINARIES_FILE)
646_AddCrOSArtifact(artifact_info.LIBIOTA_BOARD_UTILS, Artifact,
647 LIBIOTA_BOARD_UTILS_FILE)
648
Gabe Black3b567202015-09-23 14:07:59 -0700649android_artifact_map = {}
Chris Sosa968a1062013-08-02 17:42:50 -0700650
Chris Sosa76e44b92013-01-31 12:11:38 -0800651
Gabe Black3b567202015-09-23 14:07:59 -0700652def _AddAndroidArtifact(tag, base, name, *fixed_args, **fixed_kwargs):
Don Garrettfb15e322016-06-21 19:12:08 -0700653 """Add a data wrapper for android artifacts.
654
655 Add a data wrapper that describes an Android artifact's implementation to
Gabe Black3b567202015-09-23 14:07:59 -0700656 android_artifact_map.
657 """
658 artifact = _CreateNewArtifact(tag, base, name, *fixed_args, **fixed_kwargs)
659 android_artifact_map.setdefault(tag, []).append(artifact)
660
661
Dan Shiba4e00f2015-10-27 12:03:53 -0700662_AddAndroidArtifact(artifact_info.ANDROID_ZIP_IMAGES, Artifact,
Gabe Black3b567202015-09-23 14:07:59 -0700663 ANDROID_IMAGE_ZIP, is_regex_name=True)
664_AddAndroidArtifact(artifact_info.ANDROID_RADIO_IMAGE, Artifact,
665 ANDROID_RADIO_IMAGE)
666_AddAndroidArtifact(artifact_info.ANDROID_BOOTLOADER_IMAGE, Artifact,
667 ANDROID_BOOTLOADER_IMAGE)
668_AddAndroidArtifact(artifact_info.ANDROID_FASTBOOT, Artifact, ANDROID_FASTBOOT)
669_AddAndroidArtifact(artifact_info.ANDROID_TEST_ZIP, BundledArtifact,
670 ANDROID_TEST_ZIP, is_regex_name=True)
Dan Shi74136ae2015-12-01 14:40:06 -0800671_AddAndroidArtifact(artifact_info.ANDROID_VENDOR_PARTITION_ZIP, Artifact,
672 ANDROID_VENDOR_PARTITION_ZIP, is_regex_name=True)
Dan Shi6c2b2a22016-03-04 15:52:19 -0800673_AddAndroidArtifact(artifact_info.AUTOTEST_SERVER_PACKAGE, Artifact,
674 ANDROID_AUTOTEST_SERVER_PACKAGE, is_regex_name=True)
675_AddAndroidArtifact(artifact_info.TEST_SUITES, BundledArtifact,
676 ANDROID_TEST_SUITES, is_regex_name=True)
677_AddAndroidArtifact(artifact_info.CONTROL_FILES, BundledArtifact,
678 ANDROID_CONTROL_FILES, is_regex_name=True)
Simran Basi05be7212016-03-16 13:08:23 -0700679_AddAndroidArtifact(artifact_info.ANDROID_NATIVETESTS_ZIP, BundledArtifact,
680 ANDROID_NATIVETESTS_FILE, is_regex_name=True)
Ralph Nathan6c7fe5e2016-06-22 15:50:10 -0700681_AddAndroidArtifact(artifact_info.ANDROID_CONTINUOUS_NATIVE_TESTS_ZIP,
682 BundledArtifact,
683 ANDROID_CONTINUOUS_NATIVE_TESTS_FILE,
684 is_regex_name=True)
Justin Giorgibb32ac42016-08-04 10:34:35 -0700685_AddAndroidArtifact(artifact_info.ANDROID_CONTINUOUS_INSTRUMENTATION_TESTS_ZIP,
686 BundledArtifact,
687 ANDROID_CONTINUOUS_INSTRUMENTATION_TESTS_FILE,
688 is_regex_name=True)
Justin Giorgi4faa8902016-08-19 19:54:30 -0700689_AddAndroidArtifact(artifact_info.ANDROID_CTS_ZIP, BundledArtifact,
690 ANDROID_CTS_FILE)
Justin Giorgi576c1542016-06-24 08:24:20 -0700691_AddAndroidArtifact(artifact_info.ANDROID_TARGET_FILES_ZIP, Artifact,
692 ANDROID_TARGET_FILES_ZIP, is_regex_name=True)
693_AddAndroidArtifact(artifact_info.ANDROID_DTB_ZIP, Artifact,
694 ANDROID_DTB_ZIP, is_regex_name=True)
Gabe Black3b567202015-09-23 14:07:59 -0700695
696class BaseArtifactFactory(object):
Chris Sosa76e44b92013-01-31 12:11:38 -0800697 """A factory class that generates build artifacts from artifact names."""
698
Dan Shi6c2b2a22016-03-04 15:52:19 -0800699 def __init__(self, artifact_map, download_dir, artifacts, files, build,
700 requested_to_optional_map):
Chris Sosa76e44b92013-01-31 12:11:38 -0800701 """Initalizes the member variables for the factory.
702
703 Args:
Gabe Black3b567202015-09-23 14:07:59 -0700704 artifact_map: A map from artifact names to ImplDescription objects.
Gilad Arnold950569b2013-08-27 14:38:01 -0700705 download_dir: A directory to which artifacts are downloaded.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700706 artifacts: List of artifacts to stage. These artifacts must be
707 defined in artifact_info.py and have a mapping in the
708 ARTIFACT_IMPLEMENTATION_MAP.
709 files: List of files to stage. These files are just downloaded and staged
710 as files into the download_dir.
Chris Sosa76e44b92013-01-31 12:11:38 -0800711 build: The name of the build.
Dan Shi6c2b2a22016-03-04 15:52:19 -0800712 requested_to_optional_map: A map between an artifact X to a list of
713 artifacts Y. If X is requested, all items in Y should also get
714 triggered for download.
Chris Sosa76e44b92013-01-31 12:11:38 -0800715 """
Gabe Black3b567202015-09-23 14:07:59 -0700716 self.artifact_map = artifact_map
joychen0a8e34e2013-06-24 17:58:36 -0700717 self.download_dir = download_dir
Chris Sosa6b0c6172013-08-05 17:01:33 -0700718 self.artifacts = artifacts
719 self.files = files
Chris Sosa76e44b92013-01-31 12:11:38 -0800720 self.build = build
Dan Shi6c2b2a22016-03-04 15:52:19 -0800721 self.requested_to_optional_map = requested_to_optional_map
Chris Sosa76e44b92013-01-31 12:11:38 -0800722
Chris Sosa6b0c6172013-08-05 17:01:33 -0700723 def _Artifacts(self, names, is_artifact):
Gabe Black3b567202015-09-23 14:07:59 -0700724 """Returns the Artifacts from |names|.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700725
726 If is_artifact is true, then these names define artifacts that must exist in
727 the ARTIFACT_IMPLEMENTATION_MAP. Otherwise, treat as filenames to stage as
728 basic BuildArtifacts.
729
Gilad Arnold950569b2013-08-27 14:38:01 -0700730 Args:
731 names: A sequence of artifact names.
732 is_artifact: Whether this is a named (True) or file (False) artifact.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800733
Gilad Arnold950569b2013-08-27 14:38:01 -0700734 Returns:
Gabe Black3b567202015-09-23 14:07:59 -0700735 An iterable of Artifacts.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800736
Gilad Arnold950569b2013-08-27 14:38:01 -0700737 Raises:
Gabe Black3b567202015-09-23 14:07:59 -0700738 KeyError: if artifact doesn't exist in the artifact map.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700739 """
Gabe Black3b567202015-09-23 14:07:59 -0700740 if is_artifact:
741 classes = itertools.chain(*(self.artifact_map[name] for name in names))
742 return list(cls(self.download_dir, self.build) for cls in classes)
743 else:
744 return list(Artifact(name, self.download_dir, self.build)
745 for name in names)
Chris Sosa76e44b92013-01-31 12:11:38 -0800746
747 def RequiredArtifacts(self):
Gilad Arnold950569b2013-08-27 14:38:01 -0700748 """Returns BuildArtifacts for the factory's artifacts.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700749
Gilad Arnold950569b2013-08-27 14:38:01 -0700750 Returns:
751 An iterable of BuildArtifacts.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800752
Gilad Arnold950569b2013-08-27 14:38:01 -0700753 Raises:
754 KeyError: if artifact doesn't exist in ARTIFACT_IMPLEMENTATION_MAP.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700755 """
756 artifacts = []
757 if self.artifacts:
758 artifacts.extend(self._Artifacts(self.artifacts, True))
759 if self.files:
760 artifacts.extend(self._Artifacts(self.files, False))
761
762 return artifacts
Chris Sosa76e44b92013-01-31 12:11:38 -0800763
764 def OptionalArtifacts(self):
Gilad Arnold950569b2013-08-27 14:38:01 -0700765 """Returns BuildArtifacts that should be cached.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700766
Gilad Arnold950569b2013-08-27 14:38:01 -0700767 Returns:
768 An iterable of BuildArtifacts.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800769
Gilad Arnold950569b2013-08-27 14:38:01 -0700770 Raises:
771 KeyError: if an optional artifact doesn't exist in
772 ARTIFACT_IMPLEMENTATION_MAP yet defined in
773 artifact_info.REQUESTED_TO_OPTIONAL_MAP.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700774 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800775 optional_names = set()
Dan Shi6c2b2a22016-03-04 15:52:19 -0800776 for artifact_name, optional_list in self.requested_to_optional_map.items():
Chris Sosa76e44b92013-01-31 12:11:38 -0800777 # We are already downloading it.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700778 if artifact_name in self.artifacts:
Chris Sosa76e44b92013-01-31 12:11:38 -0800779 optional_names = optional_names.union(optional_list)
780
Chris Sosa6b0c6172013-08-05 17:01:33 -0700781 return self._Artifacts(optional_names - set(self.artifacts), True)
Chris Sosa968a1062013-08-02 17:42:50 -0700782
783
Gabe Black3b567202015-09-23 14:07:59 -0700784class ChromeOSArtifactFactory(BaseArtifactFactory):
785 """A factory class that generates ChromeOS build artifacts from names."""
786
787 def __init__(self, download_dir, artifacts, files, build):
788 """Pass the ChromeOS artifact map to the base class."""
789 super(ChromeOSArtifactFactory, self).__init__(
Dan Shi6c2b2a22016-03-04 15:52:19 -0800790 chromeos_artifact_map, download_dir, artifacts, files, build,
791 artifact_info.CROS_REQUESTED_TO_OPTIONAL_MAP)
Gabe Black3b567202015-09-23 14:07:59 -0700792
793
794class AndroidArtifactFactory(BaseArtifactFactory):
795 """A factory class that generates Android build artifacts from names."""
796
797 def __init__(self, download_dir, artifacts, files, build):
798 """Pass the Android artifact map to the base class."""
799 super(AndroidArtifactFactory, self).__init__(
Dan Shi6c2b2a22016-03-04 15:52:19 -0800800 android_artifact_map, download_dir, artifacts, files, build,
801 artifact_info.ANDROID_REQUESTED_TO_OPTIONAL_MAP)
Gabe Black3b567202015-09-23 14:07:59 -0700802
803
Chris Sosa968a1062013-08-02 17:42:50 -0700804# A simple main to verify correctness of the artifact map when making simple
805# name changes.
806if __name__ == '__main__':
Gabe Black3b567202015-09-23 14:07:59 -0700807 print('ARTIFACT IMPLEMENTATION MAPs (for debugging)')
808 print('FORMAT: ARTIFACT -> IMPLEMENTATION (<type>_file)')
809 for label, mapping in (('CHROMEOS', chromeos_artifact_map),
810 ('ANDROID', android_artifact_map)):
811 print('%s:' % label)
812 for key, value in sorted(mapping.items()):
813 print(' %s -> %s' % (key, ', '.join(str(val) for val in value)))