blob: 02c33756ad993279e1c690a1291303243f1a9737 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2012 The ChromiumOS Authors
David James8c846492011-01-25 17:07:29 -08002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Mike Frysingerc6824f62014-02-03 11:09:44 -05005"""Unittests for upload_prebuilts.py."""
6
David James8c846492011-01-25 17:07:29 -08007import copy
David James8fa34ea2011-04-15 13:00:20 -07008import multiprocessing
Mike Frysinger40ffb532021-02-12 07:36:08 -05009import os
David James8c846492011-01-25 17:07:29 -080010import tempfile
Mike Frysinger166fea02021-02-12 05:30:33 -050011from unittest import mock
Chris Sosa471532a2011-02-01 15:10:06 -080012
Mike Frysinger40ffb532021-02-12 07:36:08 -050013from chromite.lib import binpkg
Brian Harringc92788f2012-09-21 18:07:15 -070014from chromite.lib import cros_test_lib
David James2c7ccb42012-11-04 15:34:28 -080015from chromite.lib import gs
Zdenek Behanc0e18762012-09-22 04:06:17 +020016from chromite.lib import osutils
Mike Frysinger36870f92015-04-12 02:59:55 -040017from chromite.lib import parallel_unittest
Prathmesh Prabhu421eef22014-10-16 17:13:19 -070018from chromite.lib import portage_util
Mike Frysinger40ffb532021-02-12 07:36:08 -050019from chromite.scripts import upload_prebuilts as prebuilt
David James8c846492011-01-25 17:07:29 -080020
Mike Frysinger68182472014-11-05 22:38:39 -050021
Mike Frysinger27e21b72018-07-12 14:20:21 -040022# pylint: disable=protected-access
23
24
Alex Klein1699fab2022-09-08 08:46:06 -060025PUBLIC_PACKAGES = [
26 {"CPV": "gtk+/public1", "SHA1": "1", "MTIME": "1"},
27 {"CPV": "gtk+/public2", "SHA1": "2", "PATH": "gtk+/foo.tgz", "MTIME": "2"},
28]
29PRIVATE_PACKAGES = [{"CPV": "private", "SHA1": "3", "MTIME": "3"}]
David James8c846492011-01-25 17:07:29 -080030
31
32def SimplePackageIndex(header=True, packages=True):
Alex Klein1699fab2022-09-08 08:46:06 -060033 pkgindex = binpkg.PackageIndex()
34 if header:
35 pkgindex.header["URI"] = "gs://example"
36 if packages:
37 pkgindex.packages = copy.deepcopy(PUBLIC_PACKAGES + PRIVATE_PACKAGES)
38 return pkgindex
David James8c846492011-01-25 17:07:29 -080039
40
Brian Harringc92788f2012-09-21 18:07:15 -070041class TestUpdateFile(cros_test_lib.TempDirTestCase):
Alex Klein1699fab2022-09-08 08:46:06 -060042 """Tests for the UpdateLocalFile function."""
David James8c846492011-01-25 17:07:29 -080043
Alex Klein1699fab2022-09-08 08:46:06 -060044 def setUp(self):
45 self.contents_str = [
46 "# comment that should be skipped",
47 'PKGDIR="/var/lib/portage/pkgs"',
48 'PORTAGE_BINHOST="http://no.thanks.com"',
49 "portage portage-20100310.tar.bz2",
50 'COMPILE_FLAGS="some_value=some_other"',
51 ]
52 self.version_file = os.path.join(self.tempdir, "version")
53 osutils.WriteFile(self.version_file, "\n".join(self.contents_str))
David James8c846492011-01-25 17:07:29 -080054
Alex Klein1699fab2022-09-08 08:46:06 -060055 def _read_version_file(self, version_file=None):
56 """Read the contents of self.version_file and return as a list."""
57 if not version_file:
58 version_file = self.version_file
David James8c846492011-01-25 17:07:29 -080059
Alex Klein1699fab2022-09-08 08:46:06 -060060 with open(version_file) as version_fh:
61 return [line.strip() for line in version_fh.readlines()]
Sergey Frolov89d28242022-06-06 17:49:20 -060062
Alex Klein1699fab2022-09-08 08:46:06 -060063 def _verify_key_pair(self, key, val):
64 file_contents = self._read_version_file()
David James8c846492011-01-25 17:07:29 -080065
Alex Klein1699fab2022-09-08 08:46:06 -060066 # Make sure 'key' entry is only found once.
67 entry_found = False
Sergey Frolov09280f12022-06-06 17:47:32 -060068
Alex Klein1699fab2022-09-08 08:46:06 -060069 # Ensure value is wrapped on quotes.
70 if '"' not in val:
71 val = '"%s"' % val
Sergey Frolov09280f12022-06-06 17:47:32 -060072
Alex Klein1699fab2022-09-08 08:46:06 -060073 # Inspect file contents.
74 for entry in file_contents:
75 if "=" not in entry:
76 continue
77 file_key, file_val = entry.split("=", maxsplit=1)
78 if file_key == key:
79 if val == file_val:
80 if entry_found:
81 self.fail(f"Variable {file_key} appears twice")
82 else:
83 entry_found = True
Sergey Frolov09280f12022-06-06 17:47:32 -060084
Alex Klein1699fab2022-09-08 08:46:06 -060085 if not entry_found:
86 self.fail('Could not find "%s=%s" in version file' % (key, val))
Sergey Frolov09280f12022-06-06 17:47:32 -060087
Alex Klein1699fab2022-09-08 08:46:06 -060088 def testAddVariableThatDoesNotExist(self):
89 """Add in a new variable that was no present in the file."""
90 key = "PORTAGE_BINHOST"
91 value = "1234567"
92 prebuilt.UpdateLocalFile(self.version_file, value)
93 print(self.version_file)
94 self._read_version_file()
95 self._verify_key_pair(key, value)
96 print(self.version_file)
David James8c846492011-01-25 17:07:29 -080097
Alex Klein1699fab2022-09-08 08:46:06 -060098 def testUpdateVariable(self):
99 """Test updating a variable that already exists."""
100 binhost_key, binhost_val = self.contents_str[2].split("=")
101 if binhost_key != "PORTAGE_BINHOST":
102 self.fail(
103 "unexpected test input: expected PORTAGE_BINHOST at line[2]"
104 )
105 self._verify_key_pair(binhost_key, binhost_val)
David James8c846492011-01-25 17:07:29 -0800106
Alex Klein1699fab2022-09-08 08:46:06 -0600107 # Confirm that unrelated variable 'PKGDIR' does not change.
108 pkgdir_key, pkgdir_val = self.contents_str[1].split("=")
109 if pkgdir_key != "PKGDIR":
110 self.fail("unexpected test input: expected PKGDIR at line[1]")
111 self._verify_key_pair(pkgdir_key, pkgdir_val)
Sergey Frolov09280f12022-06-06 17:47:32 -0600112
Alex Klein1699fab2022-09-08 08:46:06 -0600113 binhost_new_val = "test_update"
114 prebuilt.UpdateLocalFile(self.version_file, binhost_new_val)
115 self._verify_key_pair(binhost_key, binhost_new_val)
116 self._verify_key_pair(pkgdir_key, pkgdir_val)
Sergey Frolov09280f12022-06-06 17:47:32 -0600117
Alex Klein1699fab2022-09-08 08:46:06 -0600118 binhost_new_val = "test_update2"
119 prebuilt.UpdateLocalFile(self.version_file, binhost_new_val)
120 self._verify_key_pair(binhost_key, binhost_new_val)
121 self._verify_key_pair(pkgdir_key, pkgdir_val)
Sergey Frolov09280f12022-06-06 17:47:32 -0600122
Alex Klein1699fab2022-09-08 08:46:06 -0600123 def testUpdateNonExistentFile(self):
124 key = "PORTAGE_BINHOST"
125 value = "1234567"
126 non_existent_file = tempfile.mktemp()
127 try:
128 prebuilt.UpdateLocalFile(non_existent_file, value)
129 file_contents = self._read_version_file(non_existent_file)
130 self.assertEqual(file_contents, ['%s="%s"' % (key, value)])
131 finally:
132 if os.path.exists(non_existent_file):
133 os.remove(non_existent_file)
David James8c846492011-01-25 17:07:29 -0800134
135
Mike Frysinger68182472014-11-05 22:38:39 -0500136class TestPrebuilt(cros_test_lib.MockTestCase):
Alex Klein1699fab2022-09-08 08:46:06 -0600137 """Tests for Prebuilt logic."""
David James8c846492011-01-25 17:07:29 -0800138
Alex Klein1699fab2022-09-08 08:46:06 -0600139 def setUp(self):
140 self._base_local_path = "/b/cbuild/build/chroot/build/x86-dogfood/"
141 self._gs_bucket_path = "gs://chromeos-prebuilt/host/version"
142 self._local_path = os.path.join(self._base_local_path, "public1.tbz2")
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800143
Alex Klein1699fab2022-09-08 08:46:06 -0600144 def testGenerateUploadDict(self):
145 self.PatchObject(prebuilt.os.path, "exists", return_true=True)
146 pkgs = [{"CPV": "public1"}]
147 result = prebuilt.GenerateUploadDict(
148 self._base_local_path, self._gs_bucket_path, pkgs
149 )
150 expected = {
151 self._local_path: self._gs_bucket_path + "/public1.tbz2",
152 }
153 self.assertEqual(result, expected)
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800154
Alex Klein1699fab2022-09-08 08:46:06 -0600155 def testGenerateUploadDictWithDebug(self):
156 self.PatchObject(prebuilt.os.path, "exists", return_true=True)
157 pkgs = [{"CPV": "public1", "DEBUG_SYMBOLS": "yes"}]
158 result = prebuilt.GenerateUploadDict(
159 self._base_local_path, self._gs_bucket_path, pkgs
160 )
161 expected = {
162 self._local_path: self._gs_bucket_path + "/public1.tbz2",
163 self._local_path.replace(
164 ".tbz2", ".debug.tbz2"
165 ): self._gs_bucket_path
166 + "/public1.debug.tbz2",
167 }
168 self.assertEqual(result, expected)
David James8c846492011-01-25 17:07:29 -0800169
Alex Klein1699fab2022-09-08 08:46:06 -0600170 def testDeterminePrebuiltConfHost(self):
171 """Test that the host prebuilt path comes back properly."""
172 expected_path = os.path.join(prebuilt._PREBUILT_MAKE_CONF["amd64"])
173 self.assertEqual(
174 prebuilt.DeterminePrebuiltConfFile("fake_path", "amd64"),
175 expected_path,
176 )
David James8c846492011-01-25 17:07:29 -0800177
David James8c846492011-01-25 17:07:29 -0800178
David James2c7ccb42012-11-04 15:34:28 -0800179class TestPkgIndex(cros_test_lib.TestCase):
Alex Klein1699fab2022-09-08 08:46:06 -0600180 """Helper for tests that update the Packages index file."""
David James2c7ccb42012-11-04 15:34:28 -0800181
Alex Klein1699fab2022-09-08 08:46:06 -0600182 def setUp(self):
183 self.db = {}
184 self.pkgindex = SimplePackageIndex()
185 self.empty = SimplePackageIndex(packages=False)
David James2c7ccb42012-11-04 15:34:28 -0800186
Alex Klein1699fab2022-09-08 08:46:06 -0600187 def assertURIs(self, uris):
188 """Verify that the duplicate DB has the specified URLs."""
189 expected = [v.uri for _, v in sorted(self.db.items())]
190 self.assertEqual(expected, uris)
David James2c7ccb42012-11-04 15:34:28 -0800191
192
193class TestPackagesFileFiltering(TestPkgIndex):
Alex Klein1699fab2022-09-08 08:46:06 -0600194 """Tests for Packages filtering behavior."""
David James8c846492011-01-25 17:07:29 -0800195
Alex Klein1699fab2022-09-08 08:46:06 -0600196 def testFilterPkgIndex(self):
197 """Test filtering out of private packages."""
198 self.pkgindex.RemoveFilteredPackages(
199 lambda pkg: pkg in PRIVATE_PACKAGES
200 )
201 self.assertEqual(self.pkgindex.packages, PUBLIC_PACKAGES)
202 self.assertEqual(self.pkgindex.modified, True)
David James8c846492011-01-25 17:07:29 -0800203
204
David James2c7ccb42012-11-04 15:34:28 -0800205class TestPopulateDuplicateDB(TestPkgIndex):
Alex Klein1699fab2022-09-08 08:46:06 -0600206 """Tests for the _PopulateDuplicateDB function."""
David James8c846492011-01-25 17:07:29 -0800207
Alex Klein1699fab2022-09-08 08:46:06 -0600208 def testEmptyIndex(self):
209 """Test population of the duplicate DB with an empty index."""
210 self.empty._PopulateDuplicateDB(self.db, 0)
211 self.assertEqual(self.db, {})
David James8c846492011-01-25 17:07:29 -0800212
Alex Klein1699fab2022-09-08 08:46:06 -0600213 def testNormalIndex(self):
214 """Test population of the duplicate DB with a full index."""
215 self.pkgindex._PopulateDuplicateDB(self.db, 0)
216 self.assertURIs(
217 [
218 "gs://example/gtk+/public1.tbz2",
219 "gs://example/gtk+/foo.tgz",
220 "gs://example/private.tbz2",
221 ]
222 )
David James8c846492011-01-25 17:07:29 -0800223
Alex Klein1699fab2022-09-08 08:46:06 -0600224 def testMissingSHA1(self):
225 """Test population of the duplicate DB with a missing SHA1."""
226 del self.pkgindex.packages[0]["SHA1"]
227 self.pkgindex._PopulateDuplicateDB(self.db, 0)
228 self.assertURIs(
229 ["gs://example/gtk+/foo.tgz", "gs://example/private.tbz2"]
230 )
David James8c846492011-01-25 17:07:29 -0800231
Alex Klein1699fab2022-09-08 08:46:06 -0600232 def testFailedPopulate(self):
233 """Test failure conditions for the populate method."""
234 headerless = SimplePackageIndex(header=False)
235 self.assertRaises(KeyError, headerless._PopulateDuplicateDB, self.db, 0)
236 del self.pkgindex.packages[0]["CPV"]
237 self.assertRaises(
238 KeyError, self.pkgindex._PopulateDuplicateDB, self.db, 0
239 )
David James8c846492011-01-25 17:07:29 -0800240
241
Mike Frysinger68182472014-11-05 22:38:39 -0500242class TestResolveDuplicateUploads(cros_test_lib.MockTestCase, TestPkgIndex):
Alex Klein1699fab2022-09-08 08:46:06 -0600243 """Tests for the ResolveDuplicateUploads function."""
David James8c846492011-01-25 17:07:29 -0800244
Alex Klein1699fab2022-09-08 08:46:06 -0600245 def setUp(self):
246 self.PatchObject(binpkg.time, "time", return_value=binpkg.TWO_WEEKS)
247 self.db = {}
248 self.dup = SimplePackageIndex()
249 self.expected_pkgindex = SimplePackageIndex()
David James2c7ccb42012-11-04 15:34:28 -0800250
Alex Klein1699fab2022-09-08 08:46:06 -0600251 def assertNoDuplicates(self, candidates):
252 """Verify no duplicates are found with the specified candidates."""
253 uploads = self.pkgindex.ResolveDuplicateUploads(candidates)
254 self.assertEqual(uploads, self.pkgindex.packages)
255 self.assertEqual(
256 len(self.pkgindex.packages), len(self.expected_pkgindex.packages)
257 )
258 for pkg1, pkg2 in zip(
259 self.pkgindex.packages, self.expected_pkgindex.packages
260 ):
261 self.assertNotEqual(pkg1["MTIME"], pkg2["MTIME"])
262 del pkg1["MTIME"]
263 del pkg2["MTIME"]
264 self.assertEqual(self.pkgindex.modified, False)
265 self.assertEqual(
266 self.pkgindex.packages, self.expected_pkgindex.packages
267 )
David James2c7ccb42012-11-04 15:34:28 -0800268
Alex Klein1699fab2022-09-08 08:46:06 -0600269 def assertAllDuplicates(self, candidates):
270 """Verify every package is a duplicate in the specified list."""
271 for pkg in self.expected_pkgindex.packages:
272 pkg.setdefault("PATH", pkg["CPV"] + ".tbz2")
273 self.pkgindex.ResolveDuplicateUploads(candidates)
274 self.assertEqual(
275 self.pkgindex.packages, self.expected_pkgindex.packages
276 )
David James615e5b52011-06-03 11:10:15 -0700277
Alex Klein1699fab2022-09-08 08:46:06 -0600278 def testEmptyList(self):
279 """If no candidates are supplied, no duplicates should be found."""
280 self.assertNoDuplicates([])
David James8c846492011-01-25 17:07:29 -0800281
Alex Klein1699fab2022-09-08 08:46:06 -0600282 def testEmptyIndex(self):
283 """If no packages are supplied, no duplicates should be found."""
284 self.assertNoDuplicates([self.empty])
David James8c846492011-01-25 17:07:29 -0800285
Alex Klein1699fab2022-09-08 08:46:06 -0600286 def testDifferentURI(self):
287 """If the URI differs, no duplicates should be found."""
288 self.dup.header["URI"] = "gs://example2"
289 self.assertNoDuplicates([self.dup])
David James2c7ccb42012-11-04 15:34:28 -0800290
Alex Klein1699fab2022-09-08 08:46:06 -0600291 def testUpdateModificationTime(self):
292 """When duplicates are found, we should use the latest mtime."""
293 for pkg in self.expected_pkgindex.packages:
294 pkg["MTIME"] = "10"
295 for pkg in self.dup.packages:
296 pkg["MTIME"] = "4"
297 self.assertAllDuplicates([self.expected_pkgindex, self.dup])
David James2c7ccb42012-11-04 15:34:28 -0800298
Alex Klein1699fab2022-09-08 08:46:06 -0600299 def testCanonicalUrl(self):
300 """If the URL is in a different format, we should still find duplicates."""
301 self.dup.header["URI"] = gs.PUBLIC_BASE_HTTPS_URL + "example"
302 self.assertAllDuplicates([self.dup])
David James8c846492011-01-25 17:07:29 -0800303
Alex Klein1699fab2022-09-08 08:46:06 -0600304 def testMissingSHA1(self):
305 """We should not find duplicates if there is no SHA1."""
306 del self.pkgindex.packages[0]["SHA1"]
307 del self.expected_pkgindex.packages[0]["SHA1"]
308 for pkg in self.expected_pkgindex.packages[1:]:
309 pkg.setdefault("PATH", pkg["CPV"] + ".tbz2")
310 self.pkgindex.ResolveDuplicateUploads([self.dup])
311 self.assertNotEqual(
312 self.pkgindex.packages[0]["MTIME"],
313 self.expected_pkgindex.packages[0]["MTIME"],
314 )
315 del self.pkgindex.packages[0]["MTIME"]
316 del self.expected_pkgindex.packages[0]["MTIME"]
317 self.assertEqual(
318 self.pkgindex.packages, self.expected_pkgindex.packages
319 )
David James8c846492011-01-25 17:07:29 -0800320
Alex Klein1699fab2022-09-08 08:46:06 -0600321 def testSymbolsAvailable(self):
322 """If symbols are available remotely, re-use them and set DEBUG_SYMBOLS."""
323 self.dup.packages[0]["DEBUG_SYMBOLS"] = "yes"
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800324
Alex Klein1699fab2022-09-08 08:46:06 -0600325 uploads = self.pkgindex.ResolveDuplicateUploads([self.dup])
326 self.assertEqual(uploads, [])
327 self.assertEqual(self.pkgindex.packages[0].get("DEBUG_SYMBOLS"), "yes")
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800328
Alex Klein1699fab2022-09-08 08:46:06 -0600329 def testSymbolsAvailableLocallyOnly(self):
330 """If the symbols are only available locally, reupload them."""
331 self.pkgindex.packages[0]["DEBUG_SYMBOLS"] = "yes"
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800332
Alex Klein1699fab2022-09-08 08:46:06 -0600333 uploads = self.pkgindex.ResolveDuplicateUploads([self.dup])
334 self.assertEqual(uploads, [self.pkgindex.packages[0]])
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800335
David James8c846492011-01-25 17:07:29 -0800336
Mike Frysinger68182472014-11-05 22:38:39 -0500337class TestWritePackageIndex(cros_test_lib.MockTestCase, TestPkgIndex):
Alex Klein1699fab2022-09-08 08:46:06 -0600338 """Tests for the WriteToNamedTemporaryFile function."""
David James8c846492011-01-25 17:07:29 -0800339
Alex Klein1699fab2022-09-08 08:46:06 -0600340 def testSimple(self):
341 """Test simple call of WriteToNamedTemporaryFile()"""
342 self.PatchObject(self.pkgindex, "Write")
343 f = self.pkgindex.WriteToNamedTemporaryFile()
344 self.assertEqual(f.read(), "")
David James8c846492011-01-25 17:07:29 -0800345
346
Mike Frysinger36870f92015-04-12 02:59:55 -0400347class TestUploadPrebuilt(cros_test_lib.MockTempDirTestCase):
Alex Klein1699fab2022-09-08 08:46:06 -0600348 """Tests for the _UploadPrebuilt function."""
David James05bcb2b2011-02-09 09:25:47 -0800349
Alex Klein1699fab2022-09-08 08:46:06 -0600350 def setUp(self):
351 class MockTemporaryFile(object):
352 """Mock out the temporary file logic."""
David James05bcb2b2011-02-09 09:25:47 -0800353
Alex Klein1699fab2022-09-08 08:46:06 -0600354 def __init__(self, name):
355 self.name = name
356
357 self.pkgindex = SimplePackageIndex()
358 self.PatchObject(
359 binpkg, "GrabLocalPackageIndex", return_value=self.pkgindex
360 )
361 self.PatchObject(
362 self.pkgindex,
363 "ResolveDuplicateUploads",
364 return_value=PRIVATE_PACKAGES,
365 )
366 self.PatchObject(
367 self.pkgindex,
368 "WriteToNamedTemporaryFile",
369 return_value=MockTemporaryFile("fake"),
370 )
371 self.remote_up_mock = self.PatchObject(prebuilt, "RemoteUpload")
372 self.gs_up_mock = self.PatchObject(prebuilt, "_GsUpload")
373
374 def testSuccessfulGsUpload(self):
375 uploads = {
376 os.path.join(self.tempdir, "private.tbz2"): "gs://foo/private.tbz2"
377 }
378 dev_extras = os.path.join(self.tempdir, "dev-only-extras.tar.xz")
379 osutils.Touch(dev_extras)
380 self.PatchObject(prebuilt, "GenerateUploadDict", return_value=uploads)
381 uploads = uploads.copy()
382 uploads["fake"] = "gs://foo/suffix/Packages"
383 uploads[dev_extras] = "gs://foo/suffix/dev-only-extras.tar.xz"
384 acl = "public-read"
385 uri = self.pkgindex.header["URI"]
386 uploader = prebuilt.PrebuiltUploader(
387 "gs://foo",
388 acl,
389 uri,
390 [],
391 "/",
392 [],
393 False,
394 "foo",
395 False,
396 "x86-foo",
397 [],
398 "",
399 )
400 uploader._UploadPrebuilt(self.tempdir, "suffix")
401 self.remote_up_mock.assert_called_once_with(mock.ANY, acl, uploads)
402 self.assertTrue(self.gs_up_mock.called)
David James05bcb2b2011-02-09 09:25:47 -0800403
David James05bcb2b2011-02-09 09:25:47 -0800404
Mike Frysinger36870f92015-04-12 02:59:55 -0400405class TestSyncPrebuilts(cros_test_lib.MockTestCase):
Alex Klein1699fab2022-09-08 08:46:06 -0600406 """Tests for the SyncHostPrebuilts function."""
David James05bcb2b2011-02-09 09:25:47 -0800407
Alex Klein1699fab2022-09-08 08:46:06 -0600408 def setUp(self):
409 self.rev_mock = self.PatchObject(
410 prebuilt, "RevGitFile", return_value=None
411 )
412 self.update_binhost_mock = self.PatchObject(
413 prebuilt, "UpdateBinhostConfFile", return_value=None
414 )
415 self.build_path = "/trunk"
416 self.upload_location = "gs://upload/"
417 self.version = "1"
418 self.binhost = "http://prebuilt/"
419 self.key = "PORTAGE_BINHOST"
420 self.upload_mock = self.PatchObject(
421 prebuilt.PrebuiltUploader, "_UploadPrebuilt", return_value=True
422 )
David James05bcb2b2011-02-09 09:25:47 -0800423
Alex Klein1699fab2022-09-08 08:46:06 -0600424 def _testSyncHostPrebuilts(self, chroot):
425 board = "x86-foo"
426 target = prebuilt.BuildTarget(board, "aura")
427 slave_targets = [prebuilt.BuildTarget("x86-bar", "aura")]
428 if chroot is None:
429 package_path = os.path.join(
430 self.build_path, "chroot", prebuilt._HOST_PACKAGES_PATH
431 )
432 else:
433 package_path = os.path.join(chroot, prebuilt._HOST_PACKAGES_PATH)
434 url_suffix = prebuilt._REL_HOST_PATH % {
435 "version": self.version,
436 "host_arch": prebuilt._HOST_ARCH,
437 "target": target,
438 }
439 packages_url_suffix = "%s/packages" % url_suffix.rstrip("/")
440 url_value = "%s/%s/" % (
441 self.binhost.rstrip("/"),
442 packages_url_suffix.rstrip("/"),
443 )
444 urls = [url_value.replace("foo", "bar"), url_value]
445 binhost = " ".join(urls)
446 uploader = prebuilt.PrebuiltUploader(
447 self.upload_location,
448 "public-read",
449 self.binhost,
450 [],
451 self.build_path,
452 [],
453 False,
454 "foo",
455 False,
456 target,
457 slave_targets,
458 self.version,
459 chroot=chroot,
460 )
461 uploader.SyncHostPrebuilts(self.key, True, True)
462 self.upload_mock.assert_called_once_with(
463 package_path, packages_url_suffix
464 )
465 self.rev_mock.assert_called_once_with(
466 mock.ANY, {self.key: binhost}, dryrun=False
467 )
468 self.update_binhost_mock.assert_called_once_with(
469 mock.ANY, self.key, binhost
470 )
David James05bcb2b2011-02-09 09:25:47 -0800471
Alex Klein1699fab2022-09-08 08:46:06 -0600472 def testSyncHostPrebuilts(self):
473 self._testSyncHostPrebuilts(chroot=None)
Bob Haarmand1225ea2022-01-19 22:01:42 +0000474
Alex Klein1699fab2022-09-08 08:46:06 -0600475 def testSyncHostPrebuiltsWithChroot(self):
476 self._testSyncHostPrebuilts("/test/chroot")
Bob Haarmand1225ea2022-01-19 22:01:42 +0000477
Alex Klein1699fab2022-09-08 08:46:06 -0600478 def testSyncBoardPrebuilts(self):
479 board = "x86-foo"
480 target = prebuilt.BuildTarget(board, "aura")
481 slave_targets = [prebuilt.BuildTarget("x86-bar", "aura")]
482 board_path = os.path.join(
483 self.build_path, prebuilt._BOARD_PATH % {"board": board}
484 )
485 package_path = os.path.join(board_path, "packages")
486 url_suffix = prebuilt._REL_BOARD_PATH % {
487 "version": self.version,
488 "target": target,
489 }
490 packages_url_suffix = "%s/packages" % url_suffix.rstrip("/")
491 url_value = "%s/%s/" % (
492 self.binhost.rstrip("/"),
493 packages_url_suffix.rstrip("/"),
494 )
495 bar_binhost = url_value.replace("foo", "bar")
496 determine_mock = self.PatchObject(
497 prebuilt, "DeterminePrebuiltConfFile", side_effect=("bar", "foo")
498 )
499 self.PatchObject(prebuilt.PrebuiltUploader, "_UploadSdkTarball")
500 with parallel_unittest.ParallelMock():
501 multiprocessing.Process.exitcode = 0
502 uploader = prebuilt.PrebuiltUploader(
503 self.upload_location,
504 "public-read",
505 self.binhost,
506 [],
507 self.build_path,
508 [],
509 False,
510 "foo",
511 False,
512 target,
513 slave_targets,
514 self.version,
515 )
516 uploader.SyncBoardPrebuilts(
517 self.key, True, True, True, None, None, None, None, None
518 )
519 determine_mock.assert_has_calls(
520 [
521 mock.call(self.build_path, slave_targets[0]),
522 mock.call(self.build_path, target),
523 ]
524 )
525 self.upload_mock.assert_called_once_with(
526 package_path, packages_url_suffix
527 )
528 self.rev_mock.assert_has_calls(
529 [
530 mock.call("bar", {self.key: bar_binhost}, dryrun=False),
531 mock.call("foo", {self.key: url_value}, dryrun=False),
532 ]
533 )
534 self.update_binhost_mock.assert_has_calls(
535 [
536 mock.call(mock.ANY, self.key, bar_binhost),
537 mock.call(mock.ANY, self.key, url_value),
538 ]
539 )
David James05bcb2b2011-02-09 09:25:47 -0800540
541
Mike Frysinger36870f92015-04-12 02:59:55 -0400542class TestMain(cros_test_lib.MockTestCase):
Alex Klein1699fab2022-09-08 08:46:06 -0600543 """Tests for the main() function."""
David Jamesc0f158a2011-02-22 16:07:29 -0800544
Alex Klein1699fab2022-09-08 08:46:06 -0600545 def testMain(self):
546 """Test that the main function works."""
547 options = mock.MagicMock()
548 old_binhost = "http://prebuilt/1"
549 options.previous_binhost_url = [old_binhost]
550 options.board = "x86-foo"
551 options.profile = None
552 target = prebuilt.BuildTarget(options.board, options.profile)
553 options.build_path = "/trunk"
554 options.chroot = None
555 options.dryrun = False
556 options.private = True
557 options.packages = []
558 options.sync_host = True
559 options.git_sync = True
560 options.upload_board_tarball = True
561 options.prepackaged_tarball = None
562 options.toolchains_overlay_tarballs = []
563 options.toolchains_overlay_upload_path = ""
564 options.toolchain_tarballs = []
565 options.toolchain_upload_path = ""
566 options.upload = "gs://upload/"
567 options.binhost_base_url = options.upload
568 options.prepend_version = True
569 options.set_version = None
570 options.skip_upload = False
571 options.filters = True
572 options.key = "PORTAGE_BINHOST"
573 options.binhost_conf_dir = None
574 options.sync_binhost_conf = True
575 options.slave_targets = [prebuilt.BuildTarget("x86-bar", "aura")]
576 self.PatchObject(
577 prebuilt, "ParseOptions", return_value=tuple([options, target])
578 )
579 self.PatchObject(binpkg, "GrabRemotePackageIndex", return_value=True)
580 init_mock = self.PatchObject(
581 prebuilt.PrebuiltUploader, "__init__", return_value=None
582 )
583 expected_gs_acl_path = os.path.join(
584 "/fake_path", prebuilt._GOOGLESTORAGE_GSUTIL_FILE
585 )
586 self.PatchObject(
587 portage_util, "FindOverlayFile", return_value=expected_gs_acl_path
588 )
589 host_mock = self.PatchObject(
590 prebuilt.PrebuiltUploader, "SyncHostPrebuilts", return_value=None
591 )
592 board_mock = self.PatchObject(
593 prebuilt.PrebuiltUploader, "SyncBoardPrebuilts", return_value=None
594 )
Mike Frysinger36870f92015-04-12 02:59:55 -0400595
Alex Klein1699fab2022-09-08 08:46:06 -0600596 prebuilt.main([])
Mike Frysinger36870f92015-04-12 02:59:55 -0400597
Alex Klein1699fab2022-09-08 08:46:06 -0600598 init_mock.assert_called_once_with(
599 options.upload,
600 expected_gs_acl_path,
601 options.upload,
602 mock.ANY,
603 options.build_path,
604 options.packages,
605 False,
606 None,
607 False,
608 target,
609 options.slave_targets,
610 mock.ANY,
611 None,
612 )
613 board_mock.assert_called_once_with(
614 options.key,
615 options.git_sync,
616 options.sync_binhost_conf,
617 options.upload_board_tarball,
618 None,
619 [],
620 "",
621 [],
622 "",
623 )
624 host_mock.assert_called_once_with(
625 options.key, options.git_sync, options.sync_binhost_conf
626 )
David Jamesc0f158a2011-02-22 16:07:29 -0800627
Mike Frysinger9e979b92012-11-29 02:55:09 -0500628
Mike Frysinger68182472014-11-05 22:38:39 -0500629class TestSdk(cros_test_lib.MockTestCase):
Alex Klein1699fab2022-09-08 08:46:06 -0600630 """Test logic related to uploading SDK binaries"""
Mike Frysinger9e979b92012-11-29 02:55:09 -0500631
Alex Klein1699fab2022-09-08 08:46:06 -0600632 def setUp(self):
633 self.PatchObject(
634 prebuilt,
635 "_GsUpload",
636 side_effect=Exception("should not get called"),
637 )
638 self.PatchObject(
639 prebuilt,
640 "UpdateBinhostConfFile",
641 side_effect=Exception("should not get called"),
642 )
643 self.upload_mock = self.PatchObject(
644 prebuilt.PrebuiltUploader, "_Upload"
645 )
Mike Frysinger9e979b92012-11-29 02:55:09 -0500646
Alex Klein1699fab2022-09-08 08:46:06 -0600647 self.acl = "magic-acl"
Mike Frysinger9e979b92012-11-29 02:55:09 -0500648
Alex Klein1699fab2022-09-08 08:46:06 -0600649 # All these args pretty much get ignored. Whee.
650 self.uploader = prebuilt.PrebuiltUploader(
651 "gs://foo",
652 self.acl,
653 "prebuilt",
654 [],
655 "/",
656 [],
657 False,
658 "foo",
659 False,
660 "x86-foo",
661 [],
662 "chroot-1234",
663 )
Mike Frysinger9e979b92012-11-29 02:55:09 -0500664
Alex Klein1699fab2022-09-08 08:46:06 -0600665 def testSdkUpload(
666 self,
667 to_tarballs=(),
668 to_upload_path=None,
669 tc_tarballs=(),
670 tc_upload_path=None,
671 ):
672 """Make sure we can upload just an SDK tarball"""
673 tar = "sdk.tar.xz"
674 ver = "1234"
675 vtar = "cros-sdk-%s.tar.xz" % ver
Mike Frysinger9e979b92012-11-29 02:55:09 -0500676
Alex Klein1699fab2022-09-08 08:46:06 -0600677 calls = [
678 mock.call(
679 "%s.Manifest" % tar, "gs://chromiumos-sdk/%s.Manifest" % vtar
680 ),
681 mock.call(tar, "gs://chromiumos-sdk/%s" % vtar),
682 ]
683 for to in to_tarballs:
684 to = to.split(":")
685 calls.append(
686 mock.call(
687 to[1],
688 ("gs://chromiumos-sdk/" + to_upload_path)
689 % {"toolchains": to[0]},
690 )
691 )
692 for tc in tc_tarballs:
693 tc = tc.split(":")
694 calls.append(
695 mock.call(
696 tc[1],
697 ("gs://chromiumos-sdk/" + tc_upload_path)
698 % {"target": tc[0]},
699 )
700 )
701 calls.append(
702 mock.call(mock.ANY, "gs://chromiumos-sdk/cros-sdk-latest.conf")
703 )
Mike Frysinger9e979b92012-11-29 02:55:09 -0500704
Alex Klein1699fab2022-09-08 08:46:06 -0600705 self.uploader._UploadSdkTarball(
706 "amd64-host",
707 "",
708 tar,
709 to_tarballs,
710 to_upload_path,
711 tc_tarballs,
712 tc_upload_path,
713 )
714 self.upload_mock.assert_has_calls(calls)
Mike Frysinger9e979b92012-11-29 02:55:09 -0500715
Alex Klein1699fab2022-09-08 08:46:06 -0600716 def testBoardOverlayTarballUpload(self):
717 """Make sure processing of board-specific overlay tarballs works."""
718 to_tarballs = (
719 (
720 "i686-pc-linux-gnu:"
721 "/some/path/built-sdk-overlay-toolchains-i686-pc-linux-gnu.tar.xz"
722 ),
723 (
724 "armv7a-cros-linux-gnueabi-arm-none-eabi:"
725 "/some/path/built-sdk-overlay-toolchains-armv7a-cros-linux-gnueabi-"
726 "arm-none-eabi"
727 ),
728 )
729 to_upload_path = "1994/04/cros-sdk-overlay-toolchains-%(toolchains)s-1994.04.02.tar.xz"
730 self.testSdkUpload(
731 to_tarballs=to_tarballs, to_upload_path=to_upload_path
732 )
Gilad Arnoldad333182015-05-27 15:50:41 -0700733
Alex Klein1699fab2022-09-08 08:46:06 -0600734 def testToolchainTarballUpload(self):
735 """Make sure processing of toolchain tarballs works."""
736 tc_tarballs = (
737 "i686:/some/i686.tar.xz",
738 "arm-none:/some/arm.tar.xz",
739 )
740 tc_upload_path = "1994/04/%(target)s-1994.04.02.tar.xz"
741 self.testSdkUpload(
742 tc_tarballs=tc_tarballs, tc_upload_path=tc_upload_path
743 )