blob: 3073cd9a27a28d9f78af0a2155d041c7b01b9673 [file] [log] [blame]
David Riley0655bab2017-11-02 10:44:26 -07001#!/usr/bin/env python2
Luis Hector Chavezdca9dd72018-06-12 12:56:30 -07002# -*- coding: utf-8 -*-
Chris Sosa47a7d4e2012-03-28 11:26:55 -07003# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Module containing classes that wrap artifact downloads."""
8
Gabe Black3b567202015-09-23 14:07:59 -07009from __future__ import print_function
10
11import itertools
Chris Sosa47a7d4e2012-03-28 11:26:55 -070012import os
Dan Shi6e50c722013-08-19 15:05:06 -070013import pickle
Gilad Arnold950569b2013-08-27 14:38:01 -070014import re
Chris Sosa47a7d4e2012-03-28 11:26:55 -070015import shutil
16import subprocess
xixuan56252ff2017-03-09 15:40:31 -080017import traceback
Chris Sosa47a7d4e2012-03-28 11:26:55 -070018
Chris Sosa76e44b92013-01-31 12:11:38 -080019import artifact_info
20import common_util
joychen3cb228e2013-06-12 12:13:13 -070021import devserver_constants
Gilad Arnoldc65330c2012-09-20 15:17:48 -070022import log_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070023
Don Garrettfb15e322016-06-21 19:12:08 -070024# We do a number of things with args/kwargs arguments that confuse pylint.
25# pylint: disable=docstring-misnamed-args
Chris Sosa47a7d4e2012-03-28 11:26:55 -070026
Chris Sosa76e44b92013-01-31 12:11:38 -080027_AU_BASE = 'au'
28_NTON_DIR_SUFFIX = '_nton'
29_MTON_DIR_SUFFIX = '_mton'
30
31############ Actual filenames of artifacts in Google Storage ############
32
33AU_SUITE_FILE = 'au_control.tar.bz2'
Chris Sosa968a1062013-08-02 17:42:50 -070034PAYGEN_AU_SUITE_FILE_TEMPLATE = 'paygen_au_%(channel)s_control.tar.bz2'
Chris Sosa76e44b92013-01-31 12:11:38 -080035AUTOTEST_FILE = 'autotest.tar'
Simran Basiea0590d2014-10-29 11:31:26 -070036CONTROL_FILES_FILE = 'control_files.tar'
37AUTOTEST_PACKAGES_FILE = 'autotest_packages.tar'
Chris Sosa76e44b92013-01-31 12:11:38 -080038AUTOTEST_COMPRESSED_FILE = 'autotest.tar.bz2'
Simran Basi6459bba2015-02-04 14:47:23 -080039AUTOTEST_SERVER_PACKAGE_FILE = 'autotest_server_package.tar.bz2'
Chris Sosa76e44b92013-01-31 12:11:38 -080040DEBUG_SYMBOLS_FILE = 'debug.tgz'
Dan Shi55d0f972016-10-04 11:45:00 -070041DEBUG_SYMBOLS_ONLY_FILE = 'debug_breakpad.tar.xz'
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -080042FACTORY_FILE = 'ChromeOS-factory*.zip'
Mike Frysingera0e6a282016-09-01 17:29:08 -040043FACTORY_SHIM_FILE = 'factory_image.zip'
Chris Sosa76e44b92013-01-31 12:11:38 -080044FIRMWARE_FILE = 'firmware_from_source.tar.bz2'
45IMAGE_FILE = 'image.zip'
Chris Sosa76e44b92013-01-31 12:11:38 -080046TEST_SUITES_FILE = 'test_suites.tar.bz2'
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -080047BASE_IMAGE_FILE = 'chromiumos_base_image.tar.xz'
48TEST_IMAGE_FILE = 'chromiumos_test_image.tar.xz'
49RECOVERY_IMAGE_FILE = 'recovery_image.tar.xz'
Luis Hector Chavezdca9dd72018-06-12 12:56:30 -070050SIGNED_IMAGE_FILE = 'chromeos_*mp*.bin'
Justin Giorgib7590522017-02-07 13:36:24 -080051LIBIOTA_TEST_BINARIES_FILE = 'test_binaries.tar.gz'
52LIBIOTA_BOARD_UTILS_FILE = 'board_utils.tar.gz'
David Riley0655bab2017-11-02 10:44:26 -070053QUICK_PROVISION_FILE = 'full_dev_part_*.bin.gz'
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -080054
Gabe Black3b567202015-09-23 14:07:59 -070055############ Actual filenames of Android build artifacts ############
56
Dan Shiba4e00f2015-10-27 12:03:53 -070057ANDROID_IMAGE_ZIP = r'.*-img-[^-]*\.zip'
Gabe Black3b567202015-09-23 14:07:59 -070058ANDROID_RADIO_IMAGE = 'radio.img'
59ANDROID_BOOTLOADER_IMAGE = 'bootloader.img'
60ANDROID_FASTBOOT = 'fastboot'
61ANDROID_TEST_ZIP = r'[^-]*-tests-.*\.zip'
Dan Shi74136ae2015-12-01 14:40:06 -080062ANDROID_VENDOR_PARTITION_ZIP = r'[^-]*-vendor_partitions-.*\.zip'
Dan Shi6c2b2a22016-03-04 15:52:19 -080063ANDROID_AUTOTEST_SERVER_PACKAGE = r'[^-]*-autotest_server_package-.*\.tar.bz2'
64ANDROID_TEST_SUITES = r'[^-]*-test_suites-.*\.tar.bz2'
65ANDROID_CONTROL_FILES = r'[^-]*-autotest_control_files-.*\.tar'
Simran Basi05be7212016-03-16 13:08:23 -070066ANDROID_NATIVETESTS_FILE = r'[^-]*-brillo-tests-.*\.zip'
Ralph Nathan6c7fe5e2016-06-22 15:50:10 -070067ANDROID_CONTINUOUS_NATIVE_TESTS_FILE = r'[^-]*-continuous_native_tests-.*\.zip'
Justin Giorgibb32ac42016-08-04 10:34:35 -070068ANDROID_CONTINUOUS_INSTRUMENTATION_TESTS_FILE = (
69 r'[^-]*-continuous_instrumentation_tests-.*\.zip')
Justin Giorgi4faa8902016-08-19 19:54:30 -070070ANDROID_CTS_FILE = 'android-cts.zip'
Justin Giorgi576c1542016-06-24 08:24:20 -070071ANDROID_TARGET_FILES_ZIP = r'[^-]*-target_files-.*\.zip'
72ANDROID_DTB_ZIP = r'[^-]*-dtb-.*\.zip'
Satoshi Niwa5e3ed5e2018-06-25 16:03:33 +090073ANDROID_PUSH_TO_DEVICE_ZIP = 'push_to_device.zip'
Chris Sosa76e44b92013-01-31 12:11:38 -080074
75_build_artifact_locks = common_util.LockDict()
Chris Sosa47a7d4e2012-03-28 11:26:55 -070076
77
78class ArtifactDownloadError(Exception):
79 """Error used to signify an issue processing an artifact."""
80 pass
81
82
Gabe Black3b567202015-09-23 14:07:59 -070083class ArtifactMeta(type):
84 """metaclass for an artifact type.
85
86 This metaclass is for class Artifact and its subclasses to have a meaningful
87 string composed of class name and the corresponding artifact name, e.g.,
88 `Artifact_full_payload`. This helps to better logging, refer to logging in
89 method Downloader.Download.
90 """
91
92 ARTIFACT_NAME = None
93
94 def __str__(cls):
95 return '%s_%s' % (cls.__name__, cls.ARTIFACT_NAME)
96
97 def __repr__(cls):
98 return str(cls)
99
100
101class Artifact(log_util.Loggable):
102 """Wrapper around an artifact to download using a fetcher.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700103
104 The purpose of this class is to download objects from Google Storage
105 and install them to a local directory. There are two main functions, one to
106 download/prepare the artifacts in to a temporary staging area and the second
107 to stage it into its final destination.
Chris Sosa76e44b92013-01-31 12:11:38 -0800108
Gilad Arnold950569b2013-08-27 14:38:01 -0700109 IMPORTANT! (i) `name' is a glob expression by default (and not a regex), be
110 attentive when adding new artifacts; (ii) name matching semantics differ
111 between a glob (full name string match) and a regex (partial match).
112
Chris Sosa76e44b92013-01-31 12:11:38 -0800113 Class members:
Gabe Black3b567202015-09-23 14:07:59 -0700114 fetcher: An object which knows how to fetch the artifact.
Gilad Arnold950569b2013-08-27 14:38:01 -0700115 name: Name given for artifact; in fact, it is a pattern that captures the
116 names of files contained in the artifact. This can either be an
117 ordinary shell-style glob (the default), or a regular expression (if
118 is_regex_name is True).
119 is_regex_name: Whether the name value is a regex (default: glob).
Chris Sosa76e44b92013-01-31 12:11:38 -0800120 build: The version of the build i.e. R26-2342.0.0.
121 marker_name: Name used to define the lock marker for the artifacts to
122 prevent it from being re-downloaded. By default based on name
123 but can be overriden by children.
Dan Shi6e50c722013-08-19 15:05:06 -0700124 exception_file_path: Path to a file containing the serialized exception,
125 which was raised in Process method. The file is located
126 in the parent folder of install_dir, since the
127 install_dir will be deleted if the build does not
128 existed.
joychen0a8e34e2013-06-24 17:58:36 -0700129 install_path: Path to artifact.
Gabe Black3b567202015-09-23 14:07:59 -0700130 install_subdir: Directory within install_path where the artifact is actually
131 stored.
Chris Sosa76e44b92013-01-31 12:11:38 -0800132 install_dir: The final location where the artifact should be staged to.
133 single_name: If True the name given should only match one item. Note, if not
134 True, self.name will become a list of items returned.
Gilad Arnold1638d822013-11-07 23:38:16 -0800135 installed_files: A list of files that were the final result of downloading
136 and setting up the artifact.
137 store_installed_files: Whether the list of installed files is stored in the
138 marker file.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700139 """
Gilad Arnold950569b2013-08-27 14:38:01 -0700140
Gabe Black3b567202015-09-23 14:07:59 -0700141 __metaclass__ = ArtifactMeta
142
143 def __init__(self, name, install_dir, build, install_subdir='',
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800144 is_regex_name=False, optional_name=None):
Gilad Arnold950569b2013-08-27 14:38:01 -0700145 """Constructor.
146
147 Args:
Chris Sosa76e44b92013-01-31 12:11:38 -0800148 install_dir: Where to install the artifact.
Chris Sosa76e44b92013-01-31 12:11:38 -0800149 name: Identifying name to be used to find/store the artifact.
150 build: The name of the build e.g. board/release.
Gabe Black3b567202015-09-23 14:07:59 -0700151 install_subdir: Directory within install_path where the artifact is
152 actually stored.
Gilad Arnold950569b2013-08-27 14:38:01 -0700153 is_regex_name: Whether the name pattern is a regex (default: glob).
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800154 optional_name: An alternative name to find the artifact, which can lead
155 to faster download. Unlike |name|, there is no guarantee that an
156 artifact named |optional_name| is/will be on Google Storage. If it
157 exists, we download it. Otherwise, we fall back to wait for |name|.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700158 """
Gabe Black3b567202015-09-23 14:07:59 -0700159 super(Artifact, self).__init__()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700160
Chris Sosa76e44b92013-01-31 12:11:38 -0800161 # In-memory lock to keep the devserver from colliding with itself while
162 # attempting to stage the same artifact.
163 self._process_lock = None
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700164
Chris Sosa76e44b92013-01-31 12:11:38 -0800165 self.name = name
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800166 self.optional_name = optional_name
Gilad Arnold950569b2013-08-27 14:38:01 -0700167 self.is_regex_name = is_regex_name
Chris Sosa76e44b92013-01-31 12:11:38 -0800168 self.build = build
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700169
Chris Sosa76e44b92013-01-31 12:11:38 -0800170 self.marker_name = '.' + self._SanitizeName(name)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700171
Dan Shi6e50c722013-08-19 15:05:06 -0700172 exception_file_name = ('.' + self._SanitizeName(build) + self.marker_name +
173 '.exception')
174 # The exception file needs to be located in parent folder, since the
175 # install_dir will be deleted is the build does not exist.
176 self.exception_file_path = os.path.join(os.path.dirname(install_dir),
Gilad Arnold950569b2013-08-27 14:38:01 -0700177 exception_file_name)
Dan Shi6e50c722013-08-19 15:05:06 -0700178
joychen0a8e34e2013-06-24 17:58:36 -0700179 self.install_path = None
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700180
Chris Sosa76e44b92013-01-31 12:11:38 -0800181 self.install_dir = install_dir
Gabe Black3b567202015-09-23 14:07:59 -0700182 self.install_subdir = install_subdir
Chris Sosa76e44b92013-01-31 12:11:38 -0800183
184 self.single_name = True
185
Gilad Arnold1638d822013-11-07 23:38:16 -0800186 self.installed_files = []
187 self.store_installed_files = True
188
Chris Sosa76e44b92013-01-31 12:11:38 -0800189 @staticmethod
190 def _SanitizeName(name):
191 """Sanitizes name to be used for creating a file on the filesystem.
192
193 '.','/' and '*' have special meaning in FS lingo. Replace them with words.
Gilad Arnold950569b2013-08-27 14:38:01 -0700194
195 Args:
196 name: A file name/path.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800197
Gilad Arnold950569b2013-08-27 14:38:01 -0700198 Returns:
199 The sanitized name/path.
Chris Sosa76e44b92013-01-31 12:11:38 -0800200 """
201 return name.replace('*', 'STAR').replace('.', 'DOT').replace('/', 'SLASH')
202
Dan Shif8eb0d12013-08-01 17:52:06 -0700203 def ArtifactStaged(self):
Gilad Arnold1638d822013-11-07 23:38:16 -0800204 """Returns True if artifact is already staged.
205
206 This checks for (1) presence of the artifact marker file, and (2) the
207 presence of each installed file listed in this marker. Both must hold for
208 the artifact to be considered staged. Note that this method is safe for use
209 even if the artifacts were not stageed by this instance, as it is assumed
Gabe Black3b567202015-09-23 14:07:59 -0700210 that any Artifact instance that did the staging wrote the list of
Gilad Arnold1638d822013-11-07 23:38:16 -0800211 files actually installed into the marker.
212 """
213 marker_file = os.path.join(self.install_dir, self.marker_name)
214
215 # If the marker is missing, it's definitely not staged.
216 if not os.path.exists(marker_file):
Aviv Keshet57d18172016-06-18 20:39:09 -0700217 self._Log('No marker file, %s is not staged.', self)
Gilad Arnold1638d822013-11-07 23:38:16 -0800218 return False
219
220 # We want to ensure that every file listed in the marker is actually there.
221 if self.store_installed_files:
222 with open(marker_file) as f:
223 files = [line.strip() for line in f]
224
225 # Check to see if any of the purportedly installed files are missing, in
226 # which case the marker is outdated and should be removed.
227 missing_files = [fname for fname in files if not os.path.exists(fname)]
228 if missing_files:
229 self._Log('***ATTENTION*** %s files listed in %s are missing:\n%s',
230 'All' if len(files) == len(missing_files) else 'Some',
231 marker_file, '\n'.join(missing_files))
232 os.remove(marker_file)
Aviv Keshet57d18172016-06-18 20:39:09 -0700233 self._Log('Missing files, %s is not staged.', self)
Gilad Arnold1638d822013-11-07 23:38:16 -0800234 return False
235
Aviv Keshet57d18172016-06-18 20:39:09 -0700236 self._Log('ArtifactStaged() -> yes, %s is staged.', self)
Gilad Arnold1638d822013-11-07 23:38:16 -0800237 return True
Chris Sosa76e44b92013-01-31 12:11:38 -0800238
239 def _MarkArtifactStaged(self):
240 """Marks the artifact as staged."""
241 with open(os.path.join(self.install_dir, self.marker_name), 'w') as f:
Gilad Arnold1638d822013-11-07 23:38:16 -0800242 f.write('\n'.join(self.installed_files))
Chris Sosa76e44b92013-01-31 12:11:38 -0800243
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800244 def _UpdateName(self, names):
245 if self.single_name and len(names) > 1:
246 raise ArtifactDownloadError('Too many artifacts match %s' % self.name)
Chris Sosa76e44b92013-01-31 12:11:38 -0800247
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800248 self.name = names[0]
Chris Sosa76e44b92013-01-31 12:11:38 -0800249
joychen0a8e34e2013-06-24 17:58:36 -0700250 def _Setup(self):
Gilad Arnold1638d822013-11-07 23:38:16 -0800251 """Process the downloaded content, update the list of installed files."""
252 # In this primitive case, what was downloaded (has to be a single file) is
253 # what's installed.
254 self.installed_files = [self.install_path]
joychen0a8e34e2013-06-24 17:58:36 -0700255
Dan Shi6e50c722013-08-19 15:05:06 -0700256 def _ClearException(self):
257 """Delete any existing exception saved for this artifact."""
258 if os.path.exists(self.exception_file_path):
259 os.remove(self.exception_file_path)
260
261 def _SaveException(self, e):
xixuan56252ff2017-03-09 15:40:31 -0800262 """Save the exception and traceback to a file for downloader.IsStaged.
Dan Shi6e50c722013-08-19 15:05:06 -0700263
Gilad Arnold950569b2013-08-27 14:38:01 -0700264 Args:
265 e: Exception object to be saved.
Dan Shi6e50c722013-08-19 15:05:06 -0700266 """
267 with open(self.exception_file_path, 'w') as f:
xixuan56252ff2017-03-09 15:40:31 -0800268 pickle.dump('%s\n%s' % (e, str(traceback.format_exc())), f)
Dan Shi6e50c722013-08-19 15:05:06 -0700269
270 def GetException(self):
271 """Retrieve any exception that was raised in Process method.
272
Gilad Arnold950569b2013-08-27 14:38:01 -0700273 Returns:
274 An Exception object that was raised when trying to process the artifact.
275 Return None if no exception was found.
Dan Shi6e50c722013-08-19 15:05:06 -0700276 """
277 if not os.path.exists(self.exception_file_path):
278 return None
279 with open(self.exception_file_path, 'r') as f:
280 return pickle.load(f)
Chris Sosa76e44b92013-01-31 12:11:38 -0800281
Gabe Black3b567202015-09-23 14:07:59 -0700282 def Process(self, downloader, no_wait):
Chris Sosa76e44b92013-01-31 12:11:38 -0800283 """Main call point to all artifacts. Downloads and Stages artifact.
284
285 Downloads and Stages artifact from Google Storage to the install directory
286 specified in the constructor. It multi-thread safe and does not overwrite
287 the artifact if it's already been downloaded or being downloaded. After
288 processing, leaves behind a marker to indicate to future invocations that
289 the artifact has already been staged based on the name of the artifact.
290
291 Do not override as it modifies important private variables, ensures thread
292 safety, and maintains cache semantics.
293
294 Note: this may be a blocking call when the artifact is already in the
295 process of being staged.
296
297 Args:
Gabe Black3b567202015-09-23 14:07:59 -0700298 downloader: A downloader instance containing the logic to download
299 artifacts.
Chris Sosa76e44b92013-01-31 12:11:38 -0800300 no_wait: If True, don't block waiting for artifact to exist if we fail to
301 immediately find it.
302
303 Raises:
304 ArtifactDownloadError: If the artifact fails to download from Google
305 Storage for any reason or that the regexp
306 defined by name is not specific enough.
307 """
308 if not self._process_lock:
309 self._process_lock = _build_artifact_locks.lock(
310 os.path.join(self.install_dir, self.name))
311
Gabe Black3b567202015-09-23 14:07:59 -0700312 real_install_dir = os.path.join(self.install_dir, self.install_subdir)
Chris Sosa76e44b92013-01-31 12:11:38 -0800313 with self._process_lock:
Gabe Black3b567202015-09-23 14:07:59 -0700314 common_util.MkDirP(real_install_dir)
Dan Shif8eb0d12013-08-01 17:52:06 -0700315 if not self.ArtifactStaged():
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800316 # Delete any existing exception saved for this artifact.
317 self._ClearException()
318 found_artifact = False
319 if self.optional_name:
320 try:
Gabe Black3b567202015-09-23 14:07:59 -0700321 # Check if the artifact named |optional_name| exists.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800322 # Because this artifact may not always exist, don't bother
323 # to wait for it (set timeout=1).
Gabe Black3b567202015-09-23 14:07:59 -0700324 new_names = downloader.Wait(
325 self.optional_name, self.is_regex_name, timeout=1)
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800326 self._UpdateName(new_names)
327
328 except ArtifactDownloadError:
329 self._Log('Unable to download %s; fall back to download %s',
330 self.optional_name, self.name)
331 else:
332 found_artifact = True
333
Dan Shi6e50c722013-08-19 15:05:06 -0700334 try:
Dan Shi6e50c722013-08-19 15:05:06 -0700335 # If the artifact should already have been uploaded, don't waste
336 # cycles waiting around for it to exist.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800337 if not found_artifact:
338 timeout = 1 if no_wait else 10
Gabe Black3b567202015-09-23 14:07:59 -0700339 new_names = downloader.Wait(
340 self.name, self.is_regex_name, timeout)
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800341 self._UpdateName(new_names)
342
David Rileye131a0f2017-11-02 10:42:34 -0700343
344 files = self.name if isinstance(self.name, list) else [self.name]
345 for filename in files:
346 self._Log('Downloading file %s', filename)
347 self.install_path = downloader.Fetch(filename, real_install_dir)
Dan Shi6e50c722013-08-19 15:05:06 -0700348 self._Setup()
349 self._MarkArtifactStaged()
350 except Exception as e:
351 # Save the exception to a file for downloader.IsStaged to retrieve.
352 self._SaveException(e)
Gilad Arnold02dc6552013-11-14 11:27:54 -0800353
354 # Convert an unknown exception into an ArtifactDownloadError.
355 if type(e) is ArtifactDownloadError:
356 raise
357 else:
358 raise ArtifactDownloadError('An error occurred: %s' % e)
Chris Sosa76e44b92013-01-31 12:11:38 -0800359 else:
360 self._Log('%s is already staged.', self)
361
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700362 def __str__(self):
363 """String representation for the download."""
Gabe Black3b567202015-09-23 14:07:59 -0700364 return '%s->%s' % (self.name, self.install_dir)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700365
Chris Sosab26b1202013-08-16 16:40:55 -0700366 def __repr__(self):
367 return str(self)
368
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700369
Gabe Black3b567202015-09-23 14:07:59 -0700370class AUTestPayload(Artifact):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700371 """Wrapper for AUTest delta payloads which need additional setup."""
Gilad Arnold950569b2013-08-27 14:38:01 -0700372
joychen0a8e34e2013-06-24 17:58:36 -0700373 def _Setup(self):
Gabe Black3b567202015-09-23 14:07:59 -0700374 super(AUTestPayload, self)._Setup()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700375
Chris Sosa76e44b92013-01-31 12:11:38 -0800376 # Rename to update.gz.
Gabe Black3b567202015-09-23 14:07:59 -0700377 install_path = os.path.join(self.install_dir, self.install_subdir,
378 self.name)
379 new_install_path = os.path.join(self.install_dir, self.install_subdir,
joychen7c2054a2013-07-25 11:14:07 -0700380 devserver_constants.UPDATE_FILE)
Chris Sosa76e44b92013-01-31 12:11:38 -0800381 shutil.move(install_path, new_install_path)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700382
Gilad Arnold1638d822013-11-07 23:38:16 -0800383 # Reflect the rename in the list of installed files.
384 self.installed_files.remove(install_path)
385 self.installed_files = [new_install_path]
386
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700387
David Rileye131a0f2017-11-02 10:42:34 -0700388class MultiArtifact(Artifact):
389 """Wrapper for artifacts where name matches multiple items.."""
390
391 def __init__(self, *args, **kwargs):
392 """Takes Artifact args.
393
394 Args:
395 *args: See Artifact documentation.
396 **kwargs: See Artifact documentation.
397 """
398 super(MultiArtifact, self).__init__(*args, **kwargs)
399 self.single_name = False
400
401 def _UpdateName(self, names):
402 self.name = names if isinstance(names, list) else [names]
403
404 def _Setup(self):
405 super(MultiArtifact, self)._Setup()
406
407 self.installed_files = [os.path.join(self.install_dir, self.install_subdir,
408 name) for name in self.name]
409
410
Gabe Black3b567202015-09-23 14:07:59 -0700411class DeltaPayloadBase(AUTestPayload):
Chris Sosa76e44b92013-01-31 12:11:38 -0800412 """Delta payloads from the archive_url.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700413
Gabe Black3b567202015-09-23 14:07:59 -0700414 These artifacts are super strange. They custom handle directories and
415 pull in all delta payloads. We can't specify exactly what we want
Chris Sosa76e44b92013-01-31 12:11:38 -0800416 because unlike other artifacts, this one does not conform to something a
417 client might know. The client doesn't know the version of n-1 or whether it
418 was even generated.
Gilad Arnold950569b2013-08-27 14:38:01 -0700419
420 IMPORTANT! Note that this artifact simply ignores the `name' argument because
Gabe Black3b567202015-09-23 14:07:59 -0700421 that name is derived internally.
Chris Sosa76e44b92013-01-31 12:11:38 -0800422 """
Gilad Arnold950569b2013-08-27 14:38:01 -0700423
joychen0a8e34e2013-06-24 17:58:36 -0700424 def _Setup(self):
Gabe Black3b567202015-09-23 14:07:59 -0700425 super(DeltaPayloadBase, self)._Setup()
426 # Setup symlink so that AU will work for this payload.
427 stateful_update_symlink = os.path.join(
428 self.install_dir, self.install_subdir,
429 devserver_constants.STATEFUL_FILE)
430 os.symlink(os.path.join(os.pardir, os.pardir,
431 devserver_constants.STATEFUL_FILE),
432 stateful_update_symlink)
433 self.installed_files.append(stateful_update_symlink)
Yu-Ju Honge61cbe92012-07-10 14:10:26 -0700434
Chris Sosa76e44b92013-01-31 12:11:38 -0800435
Gabe Black3b567202015-09-23 14:07:59 -0700436class BundledArtifact(Artifact):
Chris Sosa76e44b92013-01-31 12:11:38 -0800437 """A single build artifact bundle e.g. zip file or tar file."""
Chris Sosa76e44b92013-01-31 12:11:38 -0800438
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800439 def __init__(self, *args, **kwargs):
Gabe Black3b567202015-09-23 14:07:59 -0700440 """Takes Artifact args with some additional ones.
Gilad Arnold950569b2013-08-27 14:38:01 -0700441
442 Args:
Gabe Black3b567202015-09-23 14:07:59 -0700443 *args: See Artifact documentation.
444 **kwargs: See Artifact documentation.
Gilad Arnold950569b2013-08-27 14:38:01 -0700445 files_to_extract: A list of files to extract. If set to None, extract
446 all files.
447 exclude: A list of files to exclude. If None, no files are excluded.
Chris Sosa76e44b92013-01-31 12:11:38 -0800448 """
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800449 self._files_to_extract = kwargs.pop('files_to_extract', None)
450 self._exclude = kwargs.pop('exclude', None)
Gabe Black3b567202015-09-23 14:07:59 -0700451 super(BundledArtifact, self).__init__(*args, **kwargs)
Chris Sosa76e44b92013-01-31 12:11:38 -0800452
453 # We modify the marker so that it is unique to what was staged.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800454 if self._files_to_extract:
Chris Sosa76e44b92013-01-31 12:11:38 -0800455 self.marker_name = self._SanitizeName(
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800456 '_'.join(['.' + self.name] + self._files_to_extract))
Chris Sosa76e44b92013-01-31 12:11:38 -0800457
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800458 def _RunUnzip(self, list_only):
459 # Unzip is weird. It expects its args before any excludes and expects its
460 # excludes in a list following the -x.
461 cmd = ['unzip', '-qql' if list_only else '-o', self.install_path]
462 if not list_only:
463 cmd += ['-d', self.install_dir]
Chris Sosa76e44b92013-01-31 12:11:38 -0800464
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800465 if self._files_to_extract:
466 cmd.extend(self._files_to_extract)
467
468 if self._exclude:
469 cmd.append('-x')
470 cmd.extend(self._exclude)
471
472 try:
473 return subprocess.check_output(cmd).strip('\n').splitlines()
474 except subprocess.CalledProcessError, e:
475 raise ArtifactDownloadError(
476 'An error occurred when attempting to unzip %s:\n%s' %
477 (self.install_path, e))
Chris Sosa76e44b92013-01-31 12:11:38 -0800478
joychen0a8e34e2013-06-24 17:58:36 -0700479 def _Setup(self):
Gilad Arnold1638d822013-11-07 23:38:16 -0800480 extract_result = self._Extract()
481 if self.store_installed_files:
482 # List both the archive and the extracted files.
483 self.installed_files.append(self.install_path)
484 self.installed_files.extend(extract_result)
Chris Sosa76e44b92013-01-31 12:11:38 -0800485
Chris Sosa76e44b92013-01-31 12:11:38 -0800486 def _Extract(self):
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800487 """Extracts files into the install path."""
488 if self.name.endswith('.zip'):
489 return self._ExtractZipfile()
490 else:
491 return self._ExtractTarball()
492
493 def _ExtractZipfile(self):
494 """Extracts a zip file using unzip."""
495 file_list = [os.path.join(self.install_dir, line[30:].strip())
496 for line in self._RunUnzip(True)
497 if not line.endswith('/')]
498 if file_list:
499 self._RunUnzip(False)
500
501 return file_list
502
503 def _ExtractTarball(self):
Chris Sosa76e44b92013-01-31 12:11:38 -0800504 """Extracts a tarball using tar.
505
506 Detects whether the tarball is compressed or not based on the file
507 extension and extracts the tarball into the install_path.
508 """
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700509 try:
Gilad Arnold1638d822013-11-07 23:38:16 -0800510 return common_util.ExtractTarball(self.install_path, self.install_dir,
511 files_to_extract=self._files_to_extract,
512 excluded_files=self._exclude,
513 return_extracted_files=True)
Simran Basi4baad082013-02-14 13:39:18 -0800514 except common_util.CommonUtilError as e:
515 raise ArtifactDownloadError(str(e))
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700516
517
Gabe Black3b567202015-09-23 14:07:59 -0700518class AutotestTarball(BundledArtifact):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700519 """Wrapper around the autotest tarball to download from gsutil."""
520
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800521 def __init__(self, *args, **kwargs):
Gabe Black3b567202015-09-23 14:07:59 -0700522 super(AutotestTarball, self).__init__(*args, **kwargs)
Gilad Arnold1638d822013-11-07 23:38:16 -0800523 # We don't store/check explicit file lists in Autotest tarball markers;
524 # this can get huge and unwieldy, and generally make little sense.
525 self.store_installed_files = False
526
joychen0a8e34e2013-06-24 17:58:36 -0700527 def _Setup(self):
Chris Sosa76e44b92013-01-31 12:11:38 -0800528 """Extracts the tarball into the install path excluding test suites."""
Gabe Black3b567202015-09-23 14:07:59 -0700529 super(AutotestTarball, self)._Setup()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700530
Chris Sosa76e44b92013-01-31 12:11:38 -0800531 # Deal with older autotest packages that may not be bundled.
joychen3cb228e2013-06-12 12:13:13 -0700532 autotest_dir = os.path.join(self.install_dir,
533 devserver_constants.AUTOTEST_DIR)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700534 autotest_pkgs_dir = os.path.join(autotest_dir, 'packages')
535 if not os.path.exists(autotest_pkgs_dir):
536 os.makedirs(autotest_pkgs_dir)
537
538 if not os.path.exists(os.path.join(autotest_pkgs_dir, 'packages.checksum')):
Prashant Malani20e83712017-02-28 01:30:41 -0800539 cmd = ['autotest/utils/packager.py', '--action=upload', '--repository',
Chris Sosa76e44b92013-01-31 12:11:38 -0800540 autotest_pkgs_dir, '--all']
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700541 try:
joychen0a8e34e2013-06-24 17:58:36 -0700542 subprocess.check_call(cmd, cwd=self.install_dir)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700543 except subprocess.CalledProcessError, e:
Chris Sosa76e44b92013-01-31 12:11:38 -0800544 raise ArtifactDownloadError(
545 'Failed to create autotest packages!:\n%s' % e)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700546 else:
Gilad Arnoldf5843132012-09-25 00:31:20 -0700547 self._Log('Using pre-generated packages from autotest')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700548
Chris Masone816e38c2012-05-02 12:22:36 -0700549
Luis Hector Chavezdca9dd72018-06-12 12:56:30 -0700550class SignedArtifact(Artifact):
551 """Wrapper for signed artifacts which need a path translation."""
552
553 def _Setup(self):
554 super(SignedArtifact, self)._Setup()
555
556 # Rename to signed_image.bin.
557 install_path = os.path.join(self.install_dir, self.install_subdir,
558 self.name)
559 new_install_path = os.path.join(self.install_dir, self.install_subdir,
560 devserver_constants.SIGNED_IMAGE_FILE)
561 shutil.move(install_path, new_install_path)
562
563 # Reflect the rename in the list of installed files.
564 self.installed_files.remove(install_path)
565 self.installed_files = [new_install_path]
566
567
Gabe Black3b567202015-09-23 14:07:59 -0700568def _CreateNewArtifact(tag, base, name, *fixed_args, **fixed_kwargs):
569 """Get a data wrapper that describes an artifact's implementation.
Gilad Arnold950569b2013-08-27 14:38:01 -0700570
Gabe Black3b567202015-09-23 14:07:59 -0700571 Args:
572 tag: Tag of the artifact, defined in artifact_info.
573 base: Class of the artifact, e.g., BundledArtifact.
574 name: Name of the artifact, e.g., image.zip.
575 *fixed_args: Fixed arguments that are additional to the one used in base
576 class.
577 **fixed_kwargs: Fixed keyword arguments that are additional to the one used
578 in base class.
Chris Sosa76e44b92013-01-31 12:11:38 -0800579
Gabe Black3b567202015-09-23 14:07:59 -0700580 Returns:
581 A data wrapper that describes an artifact's implementation.
Gabe Black3b567202015-09-23 14:07:59 -0700582 """
Don Garrettfb15e322016-06-21 19:12:08 -0700583 # pylint: disable=super-on-old-class
Gabe Black3b567202015-09-23 14:07:59 -0700584 class NewArtifact(base):
585 """A data wrapper that describes an artifact's implementation."""
586 ARTIFACT_TAG = tag
587 ARTIFACT_NAME = name
588
589 def __init__(self, *args, **kwargs):
590 all_args = fixed_args + args
591 all_kwargs = {}
592 all_kwargs.update(fixed_kwargs)
593 all_kwargs.update(kwargs)
594 super(NewArtifact, self).__init__(self.ARTIFACT_NAME,
595 *all_args, **all_kwargs)
596
597 NewArtifact.__name__ = base.__name__
598 return NewArtifact
Chris Sosa968a1062013-08-02 17:42:50 -0700599
Chris Sosa76e44b92013-01-31 12:11:38 -0800600
Gabe Black3b567202015-09-23 14:07:59 -0700601# TODO(dshi): Refactor the code here to split out the logic of creating the
602# artifacts mapping to a different module.
603chromeos_artifact_map = {}
Chris Sosa76e44b92013-01-31 12:11:38 -0800604
Chris Sosa76e44b92013-01-31 12:11:38 -0800605
Gabe Black3b567202015-09-23 14:07:59 -0700606def _AddCrOSArtifact(tag, base, name, *fixed_args, **fixed_kwargs):
Don Garrettfb15e322016-06-21 19:12:08 -0700607 """Add a data wrapper for ChromeOS artifacts.
608
609 Add a data wrapper that describes a ChromeOS artifact's implementation to
Gabe Black3b567202015-09-23 14:07:59 -0700610 chromeos_artifact_map.
611 """
612 artifact = _CreateNewArtifact(tag, base, name, *fixed_args, **fixed_kwargs)
613 chromeos_artifact_map.setdefault(tag, []).append(artifact)
Chris Sosa76e44b92013-01-31 12:11:38 -0800614
beepsc3d0f872013-07-31 21:50:40 -0700615
Gabe Black3b567202015-09-23 14:07:59 -0700616_AddCrOSArtifact(artifact_info.FULL_PAYLOAD, AUTestPayload, '*_full_*')
617
618
619class DeltaPayloadNtoN(DeltaPayloadBase):
620 """ChromeOS Delta payload artifact for updating from version N to N."""
621 ARTIFACT_TAG = artifact_info.DELTA_PAYLOADS
622 ARTIFACT_NAME = 'NOT_APPLICABLE'
623
624 def __init__(self, install_dir, build, *args, **kwargs):
625 name = 'chromeos_%s*_delta_*' % build
626 install_subdir = os.path.join(_AU_BASE, build + _NTON_DIR_SUFFIX)
627 super(DeltaPayloadNtoN, self).__init__(name, install_dir, build, *args,
628 install_subdir=install_subdir,
629 **kwargs)
630
631
632class DeltaPayloadMtoN(DeltaPayloadBase):
633 """ChromeOS Delta payload artifact for updating from version M to N."""
634 ARTIFACT_TAG = artifact_info.DELTA_PAYLOADS
635 ARTIFACT_NAME = 'NOT_APPLICABLE'
636
637 def __init__(self, install_dir, build, *args, **kwargs):
638 name = ('chromeos_(?!%s).*_delta_.*' % re.escape(build))
639 install_subdir = os.path.join(_AU_BASE, build + _MTON_DIR_SUFFIX)
640 super(DeltaPayloadMtoN, self).__init__(name, install_dir, build, *args,
641 install_subdir=install_subdir,
642 is_regex_name=True, **kwargs)
643
644
645chromeos_artifact_map[artifact_info.DELTA_PAYLOADS] = [DeltaPayloadNtoN,
646 DeltaPayloadMtoN]
647
648
649_AddCrOSArtifact(artifact_info.STATEFUL_PAYLOAD, Artifact,
650 devserver_constants.STATEFUL_FILE)
651_AddCrOSArtifact(artifact_info.BASE_IMAGE, BundledArtifact, IMAGE_FILE,
652 optional_name=BASE_IMAGE_FILE,
653 files_to_extract=[devserver_constants.BASE_IMAGE_FILE])
654_AddCrOSArtifact(artifact_info.RECOVERY_IMAGE, BundledArtifact, IMAGE_FILE,
655 optional_name=RECOVERY_IMAGE_FILE,
656 files_to_extract=[devserver_constants.RECOVERY_IMAGE_FILE])
Luis Hector Chavezdca9dd72018-06-12 12:56:30 -0700657_AddCrOSArtifact(artifact_info.SIGNED_IMAGE, SignedArtifact, SIGNED_IMAGE_FILE)
Gabe Black3b567202015-09-23 14:07:59 -0700658_AddCrOSArtifact(artifact_info.DEV_IMAGE, BundledArtifact, IMAGE_FILE,
659 files_to_extract=[devserver_constants.IMAGE_FILE])
660_AddCrOSArtifact(artifact_info.TEST_IMAGE, BundledArtifact, IMAGE_FILE,
661 optional_name=TEST_IMAGE_FILE,
662 files_to_extract=[devserver_constants.TEST_IMAGE_FILE])
663_AddCrOSArtifact(artifact_info.AUTOTEST, AutotestTarball, AUTOTEST_FILE,
664 files_to_extract=None, exclude=['autotest/test_suites'])
665_AddCrOSArtifact(artifact_info.CONTROL_FILES, BundledArtifact,
666 CONTROL_FILES_FILE)
667_AddCrOSArtifact(artifact_info.AUTOTEST_PACKAGES, AutotestTarball,
668 AUTOTEST_PACKAGES_FILE)
669_AddCrOSArtifact(artifact_info.TEST_SUITES, BundledArtifact, TEST_SUITES_FILE)
670_AddCrOSArtifact(artifact_info.AU_SUITE, BundledArtifact, AU_SUITE_FILE)
671_AddCrOSArtifact(artifact_info.AUTOTEST_SERVER_PACKAGE, Artifact,
672 AUTOTEST_SERVER_PACKAGE_FILE)
673_AddCrOSArtifact(artifact_info.FIRMWARE, Artifact, FIRMWARE_FILE)
674_AddCrOSArtifact(artifact_info.SYMBOLS, BundledArtifact, DEBUG_SYMBOLS_FILE,
675 files_to_extract=['debug/breakpad'])
Dan Shi55d0f972016-10-04 11:45:00 -0700676_AddCrOSArtifact(artifact_info.SYMBOLS_ONLY, BundledArtifact,
677 DEBUG_SYMBOLS_ONLY_FILE,
678 files_to_extract=['debug/breakpad'])
Gabe Black3b567202015-09-23 14:07:59 -0700679_AddCrOSArtifact(artifact_info.FACTORY_IMAGE, BundledArtifact, FACTORY_FILE,
680 files_to_extract=[devserver_constants.FACTORY_IMAGE_FILE])
Mike Frysingera0e6a282016-09-01 17:29:08 -0400681_AddCrOSArtifact(artifact_info.FACTORY_SHIM_IMAGE, BundledArtifact,
682 FACTORY_SHIM_FILE,
683 files_to_extract=[devserver_constants.FACTORY_SHIM_IMAGE_FILE])
David Riley0655bab2017-11-02 10:44:26 -0700684_AddCrOSArtifact(artifact_info.QUICK_PROVISION, MultiArtifact,
685 QUICK_PROVISION_FILE)
Chris Sosa76e44b92013-01-31 12:11:38 -0800686
Chris Sosa968a1062013-08-02 17:42:50 -0700687# Add all the paygen_au artifacts in one go.
Gabe Black3b567202015-09-23 14:07:59 -0700688for c in devserver_constants.CHANNELS:
689 _AddCrOSArtifact(artifact_info.PAYGEN_AU_SUITE_TEMPLATE % {'channel': c},
690 BundledArtifact,
691 PAYGEN_AU_SUITE_FILE_TEMPLATE % {'channel': c})
692
Justin Giorgib7590522017-02-07 13:36:24 -0800693#### Libiota Artifacts ####
694_AddCrOSArtifact(artifact_info.LIBIOTA_TEST_BINARIES, Artifact,
695 LIBIOTA_TEST_BINARIES_FILE)
696_AddCrOSArtifact(artifact_info.LIBIOTA_BOARD_UTILS, Artifact,
697 LIBIOTA_BOARD_UTILS_FILE)
698
Gabe Black3b567202015-09-23 14:07:59 -0700699android_artifact_map = {}
Chris Sosa968a1062013-08-02 17:42:50 -0700700
Chris Sosa76e44b92013-01-31 12:11:38 -0800701
Gabe Black3b567202015-09-23 14:07:59 -0700702def _AddAndroidArtifact(tag, base, name, *fixed_args, **fixed_kwargs):
Don Garrettfb15e322016-06-21 19:12:08 -0700703 """Add a data wrapper for android artifacts.
704
705 Add a data wrapper that describes an Android artifact's implementation to
Gabe Black3b567202015-09-23 14:07:59 -0700706 android_artifact_map.
707 """
708 artifact = _CreateNewArtifact(tag, base, name, *fixed_args, **fixed_kwargs)
709 android_artifact_map.setdefault(tag, []).append(artifact)
710
711
Dan Shiba4e00f2015-10-27 12:03:53 -0700712_AddAndroidArtifact(artifact_info.ANDROID_ZIP_IMAGES, Artifact,
Gabe Black3b567202015-09-23 14:07:59 -0700713 ANDROID_IMAGE_ZIP, is_regex_name=True)
714_AddAndroidArtifact(artifact_info.ANDROID_RADIO_IMAGE, Artifact,
715 ANDROID_RADIO_IMAGE)
716_AddAndroidArtifact(artifact_info.ANDROID_BOOTLOADER_IMAGE, Artifact,
717 ANDROID_BOOTLOADER_IMAGE)
718_AddAndroidArtifact(artifact_info.ANDROID_FASTBOOT, Artifact, ANDROID_FASTBOOT)
719_AddAndroidArtifact(artifact_info.ANDROID_TEST_ZIP, BundledArtifact,
720 ANDROID_TEST_ZIP, is_regex_name=True)
Dan Shi74136ae2015-12-01 14:40:06 -0800721_AddAndroidArtifact(artifact_info.ANDROID_VENDOR_PARTITION_ZIP, Artifact,
722 ANDROID_VENDOR_PARTITION_ZIP, is_regex_name=True)
Dan Shi6c2b2a22016-03-04 15:52:19 -0800723_AddAndroidArtifact(artifact_info.AUTOTEST_SERVER_PACKAGE, Artifact,
724 ANDROID_AUTOTEST_SERVER_PACKAGE, is_regex_name=True)
725_AddAndroidArtifact(artifact_info.TEST_SUITES, BundledArtifact,
726 ANDROID_TEST_SUITES, is_regex_name=True)
727_AddAndroidArtifact(artifact_info.CONTROL_FILES, BundledArtifact,
728 ANDROID_CONTROL_FILES, is_regex_name=True)
Simran Basi05be7212016-03-16 13:08:23 -0700729_AddAndroidArtifact(artifact_info.ANDROID_NATIVETESTS_ZIP, BundledArtifact,
730 ANDROID_NATIVETESTS_FILE, is_regex_name=True)
Ralph Nathan6c7fe5e2016-06-22 15:50:10 -0700731_AddAndroidArtifact(artifact_info.ANDROID_CONTINUOUS_NATIVE_TESTS_ZIP,
732 BundledArtifact,
733 ANDROID_CONTINUOUS_NATIVE_TESTS_FILE,
734 is_regex_name=True)
Justin Giorgibb32ac42016-08-04 10:34:35 -0700735_AddAndroidArtifact(artifact_info.ANDROID_CONTINUOUS_INSTRUMENTATION_TESTS_ZIP,
736 BundledArtifact,
737 ANDROID_CONTINUOUS_INSTRUMENTATION_TESTS_FILE,
738 is_regex_name=True)
Justin Giorgi4faa8902016-08-19 19:54:30 -0700739_AddAndroidArtifact(artifact_info.ANDROID_CTS_ZIP, BundledArtifact,
740 ANDROID_CTS_FILE)
Justin Giorgi576c1542016-06-24 08:24:20 -0700741_AddAndroidArtifact(artifact_info.ANDROID_TARGET_FILES_ZIP, Artifact,
742 ANDROID_TARGET_FILES_ZIP, is_regex_name=True)
743_AddAndroidArtifact(artifact_info.ANDROID_DTB_ZIP, Artifact,
744 ANDROID_DTB_ZIP, is_regex_name=True)
Satoshi Niwa5e3ed5e2018-06-25 16:03:33 +0900745_AddAndroidArtifact(artifact_info.ANDROID_PUSH_TO_DEVICE_ZIP,
746 Artifact, ANDROID_PUSH_TO_DEVICE_ZIP)
Gabe Black3b567202015-09-23 14:07:59 -0700747
748class BaseArtifactFactory(object):
Chris Sosa76e44b92013-01-31 12:11:38 -0800749 """A factory class that generates build artifacts from artifact names."""
750
Dan Shi6c2b2a22016-03-04 15:52:19 -0800751 def __init__(self, artifact_map, download_dir, artifacts, files, build,
752 requested_to_optional_map):
Chris Sosa76e44b92013-01-31 12:11:38 -0800753 """Initalizes the member variables for the factory.
754
755 Args:
Gabe Black3b567202015-09-23 14:07:59 -0700756 artifact_map: A map from artifact names to ImplDescription objects.
Gilad Arnold950569b2013-08-27 14:38:01 -0700757 download_dir: A directory to which artifacts are downloaded.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700758 artifacts: List of artifacts to stage. These artifacts must be
759 defined in artifact_info.py and have a mapping in the
760 ARTIFACT_IMPLEMENTATION_MAP.
761 files: List of files to stage. These files are just downloaded and staged
762 as files into the download_dir.
Chris Sosa76e44b92013-01-31 12:11:38 -0800763 build: The name of the build.
Dan Shi6c2b2a22016-03-04 15:52:19 -0800764 requested_to_optional_map: A map between an artifact X to a list of
765 artifacts Y. If X is requested, all items in Y should also get
766 triggered for download.
Chris Sosa76e44b92013-01-31 12:11:38 -0800767 """
Gabe Black3b567202015-09-23 14:07:59 -0700768 self.artifact_map = artifact_map
joychen0a8e34e2013-06-24 17:58:36 -0700769 self.download_dir = download_dir
Chris Sosa6b0c6172013-08-05 17:01:33 -0700770 self.artifacts = artifacts
771 self.files = files
Chris Sosa76e44b92013-01-31 12:11:38 -0800772 self.build = build
Dan Shi6c2b2a22016-03-04 15:52:19 -0800773 self.requested_to_optional_map = requested_to_optional_map
Chris Sosa76e44b92013-01-31 12:11:38 -0800774
Chris Sosa6b0c6172013-08-05 17:01:33 -0700775 def _Artifacts(self, names, is_artifact):
Gabe Black3b567202015-09-23 14:07:59 -0700776 """Returns the Artifacts from |names|.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700777
778 If is_artifact is true, then these names define artifacts that must exist in
779 the ARTIFACT_IMPLEMENTATION_MAP. Otherwise, treat as filenames to stage as
780 basic BuildArtifacts.
781
Gilad Arnold950569b2013-08-27 14:38:01 -0700782 Args:
783 names: A sequence of artifact names.
784 is_artifact: Whether this is a named (True) or file (False) artifact.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800785
Gilad Arnold950569b2013-08-27 14:38:01 -0700786 Returns:
Gabe Black3b567202015-09-23 14:07:59 -0700787 An iterable of Artifacts.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800788
Gilad Arnold950569b2013-08-27 14:38:01 -0700789 Raises:
Gabe Black3b567202015-09-23 14:07:59 -0700790 KeyError: if artifact doesn't exist in the artifact map.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700791 """
Gabe Black3b567202015-09-23 14:07:59 -0700792 if is_artifact:
793 classes = itertools.chain(*(self.artifact_map[name] for name in names))
794 return list(cls(self.download_dir, self.build) for cls in classes)
795 else:
796 return list(Artifact(name, self.download_dir, self.build)
797 for name in names)
Chris Sosa76e44b92013-01-31 12:11:38 -0800798
799 def RequiredArtifacts(self):
Gilad Arnold950569b2013-08-27 14:38:01 -0700800 """Returns BuildArtifacts for the factory's artifacts.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700801
Gilad Arnold950569b2013-08-27 14:38:01 -0700802 Returns:
803 An iterable of BuildArtifacts.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800804
Gilad Arnold950569b2013-08-27 14:38:01 -0700805 Raises:
806 KeyError: if artifact doesn't exist in ARTIFACT_IMPLEMENTATION_MAP.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700807 """
808 artifacts = []
809 if self.artifacts:
810 artifacts.extend(self._Artifacts(self.artifacts, True))
811 if self.files:
812 artifacts.extend(self._Artifacts(self.files, False))
813
814 return artifacts
Chris Sosa76e44b92013-01-31 12:11:38 -0800815
816 def OptionalArtifacts(self):
Gilad Arnold950569b2013-08-27 14:38:01 -0700817 """Returns BuildArtifacts that should be cached.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700818
Gilad Arnold950569b2013-08-27 14:38:01 -0700819 Returns:
820 An iterable of BuildArtifacts.
Yu-Ju Hong5d5bf0d2014-02-11 21:38:20 -0800821
Gilad Arnold950569b2013-08-27 14:38:01 -0700822 Raises:
823 KeyError: if an optional artifact doesn't exist in
824 ARTIFACT_IMPLEMENTATION_MAP yet defined in
825 artifact_info.REQUESTED_TO_OPTIONAL_MAP.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700826 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800827 optional_names = set()
Dan Shi6c2b2a22016-03-04 15:52:19 -0800828 for artifact_name, optional_list in self.requested_to_optional_map.items():
Chris Sosa76e44b92013-01-31 12:11:38 -0800829 # We are already downloading it.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700830 if artifact_name in self.artifacts:
Chris Sosa76e44b92013-01-31 12:11:38 -0800831 optional_names = optional_names.union(optional_list)
832
Chris Sosa6b0c6172013-08-05 17:01:33 -0700833 return self._Artifacts(optional_names - set(self.artifacts), True)
Chris Sosa968a1062013-08-02 17:42:50 -0700834
835
Gabe Black3b567202015-09-23 14:07:59 -0700836class ChromeOSArtifactFactory(BaseArtifactFactory):
837 """A factory class that generates ChromeOS build artifacts from names."""
838
839 def __init__(self, download_dir, artifacts, files, build):
840 """Pass the ChromeOS artifact map to the base class."""
841 super(ChromeOSArtifactFactory, self).__init__(
Dan Shi6c2b2a22016-03-04 15:52:19 -0800842 chromeos_artifact_map, download_dir, artifacts, files, build,
843 artifact_info.CROS_REQUESTED_TO_OPTIONAL_MAP)
Gabe Black3b567202015-09-23 14:07:59 -0700844
845
846class AndroidArtifactFactory(BaseArtifactFactory):
847 """A factory class that generates Android build artifacts from names."""
848
849 def __init__(self, download_dir, artifacts, files, build):
850 """Pass the Android artifact map to the base class."""
851 super(AndroidArtifactFactory, self).__init__(
Dan Shi6c2b2a22016-03-04 15:52:19 -0800852 android_artifact_map, download_dir, artifacts, files, build,
853 artifact_info.ANDROID_REQUESTED_TO_OPTIONAL_MAP)
Gabe Black3b567202015-09-23 14:07:59 -0700854
855
Chris Sosa968a1062013-08-02 17:42:50 -0700856# A simple main to verify correctness of the artifact map when making simple
857# name changes.
858if __name__ == '__main__':
Gabe Black3b567202015-09-23 14:07:59 -0700859 print('ARTIFACT IMPLEMENTATION MAPs (for debugging)')
860 print('FORMAT: ARTIFACT -> IMPLEMENTATION (<type>_file)')
861 for label, mapping in (('CHROMEOS', chromeos_artifact_map),
862 ('ANDROID', android_artifact_map)):
863 print('%s:' % label)
864 for key, value in sorted(mapping.items()):
865 print(' %s -> %s' % (key, ', '.join(str(val) for val in value)))