blob: 8ba8fbd0fd4c9cbf8662f620cbb1487a6cce75d3 [file] [log] [blame]
Frank Farzan37761d12011-12-01 14:29:08 -08001#!/usr/bin/python
2#
Chris Sosa781ba6d2012-04-11 12:44:43 -07003# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Frank Farzan37761d12011-12-01 14:29:08 -08004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Gilad Arnoldc65330c2012-09-20 15:17:48 -07007"""Unit tests for common_util module."""
Frank Farzan37761d12011-12-01 14:29:08 -08008
9import os
10import shutil
Chris Sosaea148d92012-03-06 16:22:04 -080011import subprocess
Frank Farzan37761d12011-12-01 14:29:08 -080012import tempfile
13import unittest
14
Gilad Arnoldabb352e2012-09-23 01:24:27 -070015import mox
16
Gilad Arnoldc65330c2012-09-20 15:17:48 -070017import build_artifact
18import common_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070019import gsutil_util
Frank Farzan37761d12011-12-01 14:29:08 -080020
21
22# Fake Dev Server Layout:
23TEST_LAYOUT = {
24 'test-board-1': ['R17-1413.0.0-a1-b1346', 'R17-18.0.0-a1-b1346'],
Scott Zawalski16954532012-03-20 15:31:36 -040025 'test-board-2': ['R16-2241.0.0-a0-b2', 'R17-2.0.0-a1-b1346'],
26 'test-board-3': []
Frank Farzan37761d12011-12-01 14:29:08 -080027}
28
29
Gilad Arnold17fe03d2012-10-02 10:05:01 -070030class CommonUtilTest(mox.MoxTestBase):
Frank Farzan37761d12011-12-01 14:29:08 -080031
32 def setUp(self):
Chris Sosaea148d92012-03-06 16:22:04 -080033 mox.MoxTestBase.setUp(self)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070034 self._static_dir = tempfile.mkdtemp('common_util_unittest')
35 self._outside_sandbox_dir = tempfile.mkdtemp('common_util_unittest')
36 self._install_dir = tempfile.mkdtemp('common_util_unittest')
Frank Farzan37761d12011-12-01 14:29:08 -080037
38 for board, builds in TEST_LAYOUT.iteritems():
39 board_path = os.path.join(self._static_dir, board)
40 os.mkdir(board_path)
41 for build in builds:
42 build_path = os.path.join(board_path, build)
43 os.mkdir(build_path)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070044 with open(os.path.join(
45 build_path, build_artifact.TEST_IMAGE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080046 f.write('TEST_IMAGE')
Chris Sosa47a7d4e2012-03-28 11:26:55 -070047 with open(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -070048 build_path, build_artifact.STATEFUL_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080049 f.write('STATEFUL_UPDATE')
Gilad Arnoldc65330c2012-09-20 15:17:48 -070050 with open(os.path.join(
51 build_path, build_artifact.ROOT_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080052 f.write('ROOT_UPDATE')
53 # AU payloads.
Gilad Arnoldc65330c2012-09-20 15:17:48 -070054 au_dir = os.path.join(build_path, common_util.AU_BASE)
55 nton_dir = os.path.join(au_dir, build + common_util.NTON_DIR_SUFFIX)
Frank Farzan37761d12011-12-01 14:29:08 -080056 os.makedirs(nton_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070057 with open(os.path.join(nton_dir, build_artifact.ROOT_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080058 f.write('ROOT_UPDATE')
Gilad Arnoldc65330c2012-09-20 15:17:48 -070059 mton_dir = os.path.join(au_dir, build + common_util.MTON_DIR_SUFFIX)
Frank Farzan37761d12011-12-01 14:29:08 -080060 os.makedirs(mton_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070061 with open(os.path.join(mton_dir, build_artifact.ROOT_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080062 f.write('ROOT_UPDATE')
63
Chris Sosaea148d92012-03-06 16:22:04 -080064 self._good_mock_process = self.mox.CreateMock(subprocess.Popen)
65 self._good_mock_process.returncode = 0
66 self._bad_mock_process = self.mox.CreateMock(subprocess.Popen)
67 self._bad_mock_process.returncode = 1
68
Frank Farzan37761d12011-12-01 14:29:08 -080069 def tearDown(self):
70 shutil.rmtree(self._static_dir)
71 shutil.rmtree(self._outside_sandbox_dir)
Chris Sosa47a7d4e2012-03-28 11:26:55 -070072 shutil.rmtree(self._install_dir)
Frank Farzan37761d12011-12-01 14:29:08 -080073
74 def testParsePayloadList(self):
Chris Sosa1228a1a2012-05-22 17:12:13 -070075 """Tests we can parse the payload list into urls."""
Frank Farzan37761d12011-12-01 14:29:08 -080076 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -070077 'R17-1413.0.0-a1-b1346')
78 mton_basename = ('chromeos_R17-1412.0.0-a1-b1345_R17-1413.0.0-a1_'
79 'x86-mario_delta_dev.bin')
80 nton_basename = ('chromeos_R17-1413.0.0-a1_R17-1413.0.0-a1_'
81 'x86-mario_delta_dev.bin')
82 full_basename = ('chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
83
84 mton_url = '/'.join([archive_url_prefix, mton_basename])
85 nton_url = '/'.join([archive_url_prefix, nton_basename])
86 full_url = '/'.join([archive_url_prefix, full_basename])
87
Frank Farzan37761d12011-12-01 14:29:08 -080088 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -070089 common_util.ParsePayloadList(archive_url_prefix,
90 [full_basename, nton_basename,
91 mton_basename]))
Frank Farzan37761d12011-12-01 14:29:08 -080092 self.assertEqual([full_url, nton_url, mton_url],
93 [full_url_out, nton_url_out, mton_url_out])
94
95 archive_url_prefix = ('gs://chromeos-image-archive/x86-alex_he-release/'
96 'R18-1420.0.0-a1-b541')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -070097
98 mton_basename = ('chromeos_R18-1418.0.0-a1-b54a0_R18-1420.0.0-a1'
99 '_x86-alex_he_delta_dev.bin')
100 nton_basename = ('chromeos_R18-1420.0.0-a1_R18-1420.0.0-a1_'
101 'x86-alex_he_delta_dev.bin')
102 full_basename = ('chromeos_R18-1420.0.0-a1_x86-alex_he_full_dev.bin')
103
104 mton_url = '/'.join([archive_url_prefix, mton_basename])
105 nton_url = '/'.join([archive_url_prefix, nton_basename])
106 full_url = '/'.join([archive_url_prefix, full_basename])
107
Frank Farzan37761d12011-12-01 14:29:08 -0800108 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700109 common_util.ParsePayloadList(archive_url_prefix,
110 [full_basename, nton_basename,
111 mton_basename]))
Frank Farzan37761d12011-12-01 14:29:08 -0800112 self.assertEqual([full_url, nton_url, mton_url],
113 [full_url_out, nton_url_out, mton_url_out])
114
Chris Sosa1228a1a2012-05-22 17:12:13 -0700115 def testParsePayloadListWithoutDeltas(self):
116 """Tests we can parse the payload list when no delta updates exist."""
117 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700118 'R17-1413.0.0-a1-b1346')
119 full_basename = ('chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
120 full_url = '/'.join([archive_url_prefix, full_basename])
Chris Sosa1228a1a2012-05-22 17:12:13 -0700121 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700122 common_util.ParsePayloadList(
123 archive_url_prefix, [full_basename, '', '']))
Chris Sosa1228a1a2012-05-22 17:12:13 -0700124 self.assertEqual([full_url, None, None],
125 [full_url_out, nton_url_out, mton_url_out])
126
Chris Sosa781ba6d2012-04-11 12:44:43 -0700127 def testParsePartialPayloadList(self):
128 """Tests that we can parse a payload list with missing optional payload."""
129 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/'
130 'R17-1413.0.0-a1-b1346/')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700131 nton_basename = ('chromeos_R17-1413.0.0-a1_R17-1413.0.0-a1_x86-'
132 'mario_delta_dev.bin')
133 full_basename = ('chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
134
135 nton_url = '/'.join([archive_url_prefix, nton_basename])
136 full_url = '/'.join([archive_url_prefix, full_basename])
137
Chris Sosa781ba6d2012-04-11 12:44:43 -0700138 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700139 common_util.ParsePayloadList(archive_url_prefix,
140 [full_basename, nton_basename]))
Chris Sosa781ba6d2012-04-11 12:44:43 -0700141 self.assertEqual([full_url, nton_url, None],
142 [full_url_out, nton_url_out, mton_url_out])
143
Frank Farzan37761d12011-12-01 14:29:08 -0800144 def testInstallBuild(self):
Simon Glassf5019de2012-03-20 12:14:41 -0700145 # TODO(frankf): Implement this test
146 # self.fail('Not implemented.')
147 pass
Frank Farzan37761d12011-12-01 14:29:08 -0800148
149 def testPrepareAutotestPkgs(self):
Simon Glassf5019de2012-03-20 12:14:41 -0700150 # TODO(frankf): Implement this test
151 # self.fail('Not implemented.')
Scott Zawalski16954532012-03-20 15:31:36 -0400152 # TODO: implement
Simon Glassf5019de2012-03-20 12:14:41 -0700153 pass
Frank Farzan37761d12011-12-01 14:29:08 -0800154
155 def testSafeSandboxAccess(self):
156 # Path is in sandbox.
157 self.assertTrue(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700158 common_util.SafeSandboxAccess(
Frank Farzan37761d12011-12-01 14:29:08 -0800159 self._static_dir, os.path.join(self._static_dir, 'some-board')))
160
161 # Path is sandbox.
162 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700163 common_util.SafeSandboxAccess(self._static_dir, self._static_dir))
Frank Farzan37761d12011-12-01 14:29:08 -0800164
165 # Path is outside the sandbox.
166 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700167 common_util.SafeSandboxAccess(
168 self._static_dir, self._outside_sandbox_dir))
Frank Farzan37761d12011-12-01 14:29:08 -0800169
170 # Path contains '..'.
171 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700172 common_util.SafeSandboxAccess(
Frank Farzan37761d12011-12-01 14:29:08 -0800173 self._static_dir, os.path.join(self._static_dir, os.pardir)))
174
175 # Path contains symbolic link references.
176 os.chdir(self._static_dir)
177 os.symlink(os.pardir, 'parent')
178 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700179 common_util.SafeSandboxAccess(
Frank Farzan37761d12011-12-01 14:29:08 -0800180 self._static_dir, os.path.join(self._static_dir, os.pardir)))
181
182 def testAcquireReleaseLocks(self):
Gilad Arnold5174ca22012-09-12 10:49:09 -0700183 # Successful lock and unlock, removing the newly created directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700184 lock_file = common_util.AcquireLock(self._static_dir, 'test-lock')
Frank Farzan37761d12011-12-01 14:29:08 -0800185 self.assertTrue(os.path.exists(lock_file))
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700186 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Frank Farzan37761d12011-12-01 14:29:08 -0800187 self.assertFalse(os.path.exists(lock_file))
188
Gilad Arnold5174ca22012-09-12 10:49:09 -0700189 # Attempt to freshly create and lock an existing directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700190 common_util.AcquireLock(self._static_dir, 'test-lock')
191 common_util.ReleaseLock(self._static_dir, 'test-lock')
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700192 self.assertRaises(common_util.CommonUtilError, common_util.AcquireLock,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700193 self._static_dir, 'test-lock')
194 common_util.AcquireLock(self._static_dir, 'test-lock', create_once=False)
195 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Gilad Arnold5174ca22012-09-12 10:49:09 -0700196
197 # Sucessfully re-lock a pre-existing directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700198 common_util.AcquireLock(self._static_dir, 'test-lock')
199 common_util.ReleaseLock(self._static_dir, 'test-lock')
200 common_util.AcquireLock(self._static_dir, 'test-lock', create_once=False)
201 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Gilad Arnold5174ca22012-09-12 10:49:09 -0700202
203 # Attempt to lock an already locked directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700204 common_util.AcquireLock(self._static_dir, 'test-lock')
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700205 self.assertRaises(common_util.CommonUtilError, common_util.AcquireLock,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700206 self._static_dir, 'test-lock')
207 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Frank Farzan37761d12011-12-01 14:29:08 -0800208
209 def testFindMatchingBoards(self):
210 for key in TEST_LAYOUT:
211 # Partial match with multiple boards.
212 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700213 set(common_util.FindMatchingBoards(self._static_dir, key[:-5])),
Frank Farzan37761d12011-12-01 14:29:08 -0800214 set(TEST_LAYOUT.keys()))
215
216 # Absolute match.
217 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700218 common_util.FindMatchingBoards(self._static_dir, key), [key])
Frank Farzan37761d12011-12-01 14:29:08 -0800219
220 # Invalid partial match.
221 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700222 common_util.FindMatchingBoards(self._static_dir, 'asdfsadf'), [])
Frank Farzan37761d12011-12-01 14:29:08 -0800223
224 def testFindMatchingBuilds(self):
225 # Try a partial board and build match with single match.
226 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700227 common_util.FindMatchingBuilds(self._static_dir, 'test-board',
228 'R17-1413'),
Frank Farzan37761d12011-12-01 14:29:08 -0800229 [('test-board-1', 'R17-1413.0.0-a1-b1346')])
230
231 # Try a partial board and build match with multiple match.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700232 actual = set(common_util.FindMatchingBuilds(
Frank Farzan37761d12011-12-01 14:29:08 -0800233 self._static_dir, 'test-board', 'R17'))
234 expected = set([('test-board-1', 'R17-1413.0.0-a1-b1346'),
235 ('test-board-1', 'R17-18.0.0-a1-b1346'),
236 ('test-board-2', 'R17-2.0.0-a1-b1346')])
237 self.assertEqual(actual, expected)
238
239 def testGetLatestBuildVersion(self):
240 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700241 common_util.GetLatestBuildVersion(self._static_dir, 'test-board-1'),
Frank Farzan37761d12011-12-01 14:29:08 -0800242 'R17-1413.0.0-a1-b1346')
243
Scott Zawalski16954532012-03-20 15:31:36 -0400244 def testGetLatestBuildVersionLatest(self):
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700245 """Test that we raise CommonUtilError when a build dir is empty."""
246 self.assertRaises(common_util.CommonUtilError,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700247 common_util.GetLatestBuildVersion,
Scott Zawalski16954532012-03-20 15:31:36 -0400248 self._static_dir, 'test-board-3')
Frank Farzan37761d12011-12-01 14:29:08 -0800249
Scott Zawalski16954532012-03-20 15:31:36 -0400250 def testGetLatestBuildVersionUnknownBuild(self):
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700251 """Test that we raise CommonUtilError when a build dir does not exist."""
252 self.assertRaises(common_util.CommonUtilError,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700253 common_util.GetLatestBuildVersion,
Scott Zawalski16954532012-03-20 15:31:36 -0400254 self._static_dir, 'bad-dir')
Frank Farzan37761d12011-12-01 14:29:08 -0800255
Scott Zawalski16954532012-03-20 15:31:36 -0400256 def testGetLatestBuildVersionMilestone(self):
257 """Test that we can get builds based on milestone."""
258 expected_build_str = 'R16-2241.0.0-a0-b2'
259 milestone = 'R16'
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700260 build_str = common_util.GetLatestBuildVersion(
Scott Zawalski16954532012-03-20 15:31:36 -0400261 self._static_dir, 'test-board-2', milestone)
262 self.assertEqual(expected_build_str, build_str)
Frank Farzan37761d12011-12-01 14:29:08 -0800263
264 def testCloneBuild(self):
265 test_prefix = 'abc'
266 test_tag = test_prefix + '/123'
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700267 abc_path = os.path.join(
268 self._static_dir, common_util.DEV_BUILD_PREFIX, test_tag)
Frank Farzan37761d12011-12-01 14:29:08 -0800269
270 os.mkdir(os.path.join(self._static_dir, test_prefix))
271
272 # Verify leaf path is created and proper values returned.
273 board, builds = TEST_LAYOUT.items()[0]
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700274 dev_build = common_util.CloneBuild(
275 self._static_dir, board, builds[0], test_tag)
Frank Farzan37761d12011-12-01 14:29:08 -0800276 self.assertEquals(dev_build, abc_path)
277 self.assertTrue(os.path.exists(abc_path))
278 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700279 abc_path, build_artifact.TEST_IMAGE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800280 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700281 abc_path, build_artifact.ROOT_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800282 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700283 abc_path, build_artifact.STATEFUL_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800284
285 # Verify force properly removes the old directory.
286 junk_path = os.path.join(dev_build, 'junk')
287 with open(junk_path, 'w') as f:
288 f.write('hello!')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700289 remote_dir = common_util.CloneBuild(
Frank Farzan37761d12011-12-01 14:29:08 -0800290 self._static_dir, board, builds[0], test_tag, force=True)
291 self.assertEquals(remote_dir, abc_path)
292 self.assertTrue(os.path.exists(abc_path))
293 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700294 abc_path, build_artifact.TEST_IMAGE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800295 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700296 abc_path, build_artifact.ROOT_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800297 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700298 abc_path, build_artifact.STATEFUL_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800299 self.assertFalse(os.path.exists(junk_path))
300
301 def testGetControlFile(self):
302 control_file_dir = os.path.join(
Chris Sosaea148d92012-03-06 16:22:04 -0800303 self._static_dir, 'test-board-1', 'R17-1413.0.0-a1-b1346', 'autotest',
304 'server', 'site_tests', 'network_VPN')
Frank Farzan37761d12011-12-01 14:29:08 -0800305 os.makedirs(control_file_dir)
306 with open(os.path.join(control_file_dir, 'control'), 'w') as f:
307 f.write('hello!')
308
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700309 control_content = common_util.GetControlFile(
Chris Sosaea148d92012-03-06 16:22:04 -0800310 self._static_dir, 'test-board-1/R17-1413.0.0-a1-b1346',
Frank Farzan37761d12011-12-01 14:29:08 -0800311 os.path.join('server', 'site_tests', 'network_VPN', 'control'))
312 self.assertEqual(control_content, 'hello!')
313
314 def testListAutoupdateTargets(self):
315 for board, builds in TEST_LAYOUT.iteritems():
316 for build in builds:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700317 au_targets = common_util.ListAutoupdateTargets(
318 self._static_dir, board, build)
Frank Farzan37761d12011-12-01 14:29:08 -0800319 self.assertEqual(set(au_targets),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700320 set([build + common_util.NTON_DIR_SUFFIX,
321 build + common_util.MTON_DIR_SUFFIX]))
Frank Farzan37761d12011-12-01 14:29:08 -0800322
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700323 def testGatherArtifactDownloads(self):
324 """Tests that we can gather the correct download requirements."""
325 build = 'R17-1413.0.0-a1-b1346'
326 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
327 build)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700328 mock_data = 'mock data\nmock_data'
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700329 payloads = map(lambda x: '/'.join([archive_url_prefix, x]),
330 ['p1', 'p2', 'p3'])
331 expected_payloads = payloads + map(
332 lambda x: '/'.join([archive_url_prefix, x]),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700333 [build_artifact.STATEFUL_UPDATE,
334 build_artifact.AUTOTEST_ZIPPED_PACKAGE,
335 build_artifact.TEST_SUITES_PACKAGE])
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700336 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700337 self.mox.StubOutWithMock(common_util, 'IsAvailable')
338 self.mox.StubOutWithMock(common_util, 'ParsePayloadList')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700339
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700340 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700341 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700342 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700343 common_util.IsAvailable(
344 mox.IgnoreArg(), mock_data.splitlines()).AndReturn(True)
345 common_util.ParsePayloadList(archive_url_prefix,
346 mock_data.splitlines()).AndReturn(payloads)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700347
348 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700349 artifacts = common_util.GatherArtifactDownloads(
Gilad Arnold6f99b982012-09-12 10:49:40 -0700350 self._static_dir, archive_url_prefix, self._install_dir, build)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700351 for index, artifact in enumerate(artifacts):
352 self.assertEqual(artifact._gs_path, expected_payloads[index])
353 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700354
355 self.mox.VerifyAll()
356
Chris Sosa781ba6d2012-04-11 12:44:43 -0700357 def testGatherArtifactDownloadsWithoutMton(self):
358 """Gather the correct download requirements without mton delta."""
359 build = 'R17-1413.0.0-a1-b1346'
360 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
361 build)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700362 mock_data = 'mock data\nmock_data\nmock_data'
Chris Sosa781ba6d2012-04-11 12:44:43 -0700363 payloads = map(lambda x: '/'.join([archive_url_prefix, x]),
364 ['p1', 'p2'])
365 expected_payloads = payloads + map(
366 lambda x: '/'.join([archive_url_prefix, x]),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700367 [build_artifact.STATEFUL_UPDATE,
368 build_artifact.AUTOTEST_ZIPPED_PACKAGE,
369 build_artifact.TEST_SUITES_PACKAGE])
Chris Sosa781ba6d2012-04-11 12:44:43 -0700370 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700371 self.mox.StubOutWithMock(common_util, 'IsAvailable')
372 self.mox.StubOutWithMock(common_util, 'ParsePayloadList')
Chris Sosa781ba6d2012-04-11 12:44:43 -0700373
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700374 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700375 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Sosa781ba6d2012-04-11 12:44:43 -0700376 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700377 common_util.IsAvailable(
378 mox.IgnoreArg(), mock_data.splitlines()).AndReturn(True)
379 common_util.ParsePayloadList(archive_url_prefix,
380 mock_data.splitlines()
381 ).AndReturn(payloads + [None])
Chris Sosa781ba6d2012-04-11 12:44:43 -0700382
383 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700384 artifacts = common_util.GatherArtifactDownloads(
Gilad Arnold6f99b982012-09-12 10:49:40 -0700385 self._static_dir, archive_url_prefix, self._install_dir, build)
Chris Sosa781ba6d2012-04-11 12:44:43 -0700386 for index, artifact in enumerate(artifacts):
387 self.assertEqual(artifact._gs_path, expected_payloads[index])
388 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
Chris Sosa1228a1a2012-05-22 17:12:13 -0700389
390 self.mox.VerifyAll()
391
392 def testGatherArtifactDownloadsWithoutMtonOrNton(self):
393 """Gather the correct download requirements without delta payloads."""
394 build = 'R17-1413.0.0-a1-b1346'
395 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
396 build)
397 mock_data = 'mock data\nmock_data'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700398
Chris Sosa1228a1a2012-05-22 17:12:13 -0700399 payloads = map(lambda x: '/'.join([archive_url_prefix, x]),
400 ['p1'])
401 expected_payloads = payloads + map(
402 lambda x: '/'.join([archive_url_prefix, x]),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700403 [build_artifact.STATEFUL_UPDATE,
404 build_artifact.AUTOTEST_ZIPPED_PACKAGE,
405 build_artifact.TEST_SUITES_PACKAGE])
Chris Sosa1228a1a2012-05-22 17:12:13 -0700406 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700407 self.mox.StubOutWithMock(common_util, 'IsAvailable')
408 self.mox.StubOutWithMock(common_util, 'ParsePayloadList')
Chris Sosa1228a1a2012-05-22 17:12:13 -0700409
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700410 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700411 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Sosa1228a1a2012-05-22 17:12:13 -0700412 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700413 common_util.IsAvailable(
414 mox.IgnoreArg(), mock_data.splitlines()).AndReturn(True)
415 common_util.ParsePayloadList(archive_url_prefix,
416 mock_data.splitlines()
417 ).AndReturn(payloads + [None, None])
Chris Sosa1228a1a2012-05-22 17:12:13 -0700418
419 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700420 artifacts = common_util.GatherArtifactDownloads(
Gilad Arnold6f99b982012-09-12 10:49:40 -0700421 self._static_dir, archive_url_prefix, self._install_dir, build)
Chris Sosa1228a1a2012-05-22 17:12:13 -0700422 for index, artifact in enumerate(artifacts):
423 self.assertEqual(artifact._gs_path, expected_payloads[index])
424 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
Chris Sosa781ba6d2012-04-11 12:44:43 -0700425
426 self.mox.VerifyAll()
427
Chris Masone816e38c2012-05-02 12:22:36 -0700428 def testGatherSymbolArtifactDownloads(self):
429 """Tests that we can find debug symbol artifacts to download."""
430 build = 'R17-1413.0.0-a1-b1346'
431 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
432 build)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700433 symbol_url = '/'.join([archive_url_prefix,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700434 build_artifact.DEBUG_SYMBOLS])
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700435 uploaded_list_url = '/'.join([archive_url_prefix,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700436 common_util.UPLOADED_LIST])
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700437 mock_data = 'mock-tarball.tgz\nmock-debug.tgz'
Chris Masone816e38c2012-05-02 12:22:36 -0700438 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Chris Masone816e38c2012-05-02 12:22:36 -0700439
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700440 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700441 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Masone816e38c2012-05-02 12:22:36 -0700442 mox.IgnoreArg()).AndReturn(mock_data)
443
444 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700445 artifacts = common_util.GatherSymbolArtifactDownloads(
Chris Masone816e38c2012-05-02 12:22:36 -0700446 self._static_dir, archive_url_prefix, self._install_dir)
447 for index, artifact in enumerate(artifacts):
448 self.assertEqual(artifact._gs_path, symbol_url)
449 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
450
451 self.mox.VerifyAll()
452
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700453 def testIsAvailable(self):
454 """Test that we can detect whether the target artifacts are avaialble."""
455 # Test when the all target files are available
456 pattern_list = ['_full_', 'autotest.tar']
457 uploaded_list = ['chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin',
458 'debug.tgz',
459 'autotest.tar.bz2']
460
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700461 available = common_util.IsAvailable(pattern_list, uploaded_list)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700462 self.assertTrue(available)
463
464 # Test when some target files are missing
465 pattern_list = ['_full_', 'autotest.tar']
466 uploaded_list = ['chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin',
467 'debug.tgz']
468
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700469 available = common_util.IsAvailable(pattern_list, uploaded_list)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700470 self.assertFalse(available)
471
472 def testWaitUntilAvailable(self):
473 """Test that we can poll until all target artifacts are available."""
Chris Masone816e38c2012-05-02 12:22:36 -0700474 build = 'R17-1413.0.0-a1-b1346'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700475 archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
476 'R17-1413.0.0-a1-b1346')
477 to_wait_list = ['_full_']
Chris Masone816e38c2012-05-02 12:22:36 -0700478 mock_data = 'mock data\nmock_data\nmock_data'
Chris Masone816e38c2012-05-02 12:22:36 -0700479
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700480 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700481 self.mox.StubOutWithMock(common_util, 'IsAvailable')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700482
483 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700484 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Masone816e38c2012-05-02 12:22:36 -0700485 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700486 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True)
Chris Masone816e38c2012-05-02 12:22:36 -0700487
488 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700489 uploaded_list = common_util.WaitUntilAvailable(
490 to_wait_list, archive_url, 'UNIT TEST', delay=1)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700491 self.assertEqual(uploaded_list, mock_data.splitlines())
Chris Masone816e38c2012-05-02 12:22:36 -0700492 self.mox.VerifyAll()
493
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700494 def testWaitUntilAvailableWithRetry(self):
495 """Test that we can poll until all target artifacts are available."""
Chris Masone816e38c2012-05-02 12:22:36 -0700496 build = 'R17-1413.0.0-a1-b1346'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700497 archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
498 'R17-1413.0.0-a1-b1346')
499 to_wait_list = ['_full_']
500 mock_data = 'mock data\nmock_data\nmock_data'
Chris Masone816e38c2012-05-02 12:22:36 -0700501
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700502 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700503 self.mox.StubOutWithMock(common_util, 'IsAvailable')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700504
505 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700506 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700507 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700508 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700509
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700510 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700511 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700512 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True)
Chris Masone816e38c2012-05-02 12:22:36 -0700513
514 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700515 uploaded_list = common_util.WaitUntilAvailable(
516 to_wait_list, archive_url, 'UNIT TEST', delay=1)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700517 self.assertEqual(uploaded_list, mock_data.splitlines())
Chris Masone816e38c2012-05-02 12:22:36 -0700518 self.mox.VerifyAll()
519
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700520 def testWaitUntilAvailableTimeout(self):
521 """Test that we wait for the target artifacts until timeout occurs."""
522 build = 'R17-1413.0.0-a1-b1346'
523 archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
524 'R17-1413.0.0-a1-b1346')
525 to_wait_list = ['_full_']
526 mock_data = 'mock data\nmock_data\nmock_data'
527
528 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700529 self.mox.StubOutWithMock(common_util, 'IsAvailable')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700530
531 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700532 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700533 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700534 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700535
536 self.mox.ReplayAll()
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700537 self.assertRaises(common_util.CommonUtilError,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700538 common_util.WaitUntilAvailable,
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700539 to_wait_list,
540 archive_url,
541 'UNIT TEST',
542 delay=2,
543 timeout=1)
544 self.mox.VerifyAll()
Frank Farzan37761d12011-12-01 14:29:08 -0800545
546if __name__ == '__main__':
547 unittest.main()