blob: 5a9d0132254764e65ff59cf28e3def947e958ba4 [file] [log] [blame]
Frank Farzan37761d12011-12-01 14:29:08 -08001#!/usr/bin/python
2#
Chris Sosa781ba6d2012-04-11 12:44:43 -07003# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Frank Farzan37761d12011-12-01 14:29:08 -08004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Gilad Arnoldc65330c2012-09-20 15:17:48 -07007"""Unit tests for common_util module."""
Frank Farzan37761d12011-12-01 14:29:08 -08008
Chris Sosaea148d92012-03-06 16:22:04 -08009import mox
Frank Farzan37761d12011-12-01 14:29:08 -080010import os
11import shutil
Chris Sosaea148d92012-03-06 16:22:04 -080012import subprocess
Frank Farzan37761d12011-12-01 14:29:08 -080013import tempfile
14import unittest
15
Gilad Arnoldc65330c2012-09-20 15:17:48 -070016import build_artifact
17import common_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070018import gsutil_util
Frank Farzan37761d12011-12-01 14:29:08 -080019
20
21# Fake Dev Server Layout:
22TEST_LAYOUT = {
23 'test-board-1': ['R17-1413.0.0-a1-b1346', 'R17-18.0.0-a1-b1346'],
Scott Zawalski16954532012-03-20 15:31:36 -040024 'test-board-2': ['R16-2241.0.0-a0-b2', 'R17-2.0.0-a1-b1346'],
25 'test-board-3': []
Frank Farzan37761d12011-12-01 14:29:08 -080026}
27
28
Chris Sosaea148d92012-03-06 16:22:04 -080029class DevServerUtilTest(mox.MoxTestBase):
Frank Farzan37761d12011-12-01 14:29:08 -080030
31 def setUp(self):
Chris Sosaea148d92012-03-06 16:22:04 -080032 mox.MoxTestBase.setUp(self)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070033 self._static_dir = tempfile.mkdtemp('common_util_unittest')
34 self._outside_sandbox_dir = tempfile.mkdtemp('common_util_unittest')
35 self._install_dir = tempfile.mkdtemp('common_util_unittest')
Frank Farzan37761d12011-12-01 14:29:08 -080036
37 for board, builds in TEST_LAYOUT.iteritems():
38 board_path = os.path.join(self._static_dir, board)
39 os.mkdir(board_path)
40 for build in builds:
41 build_path = os.path.join(board_path, build)
42 os.mkdir(build_path)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070043 with open(os.path.join(
44 build_path, build_artifact.TEST_IMAGE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080045 f.write('TEST_IMAGE')
Chris Sosa47a7d4e2012-03-28 11:26:55 -070046 with open(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -070047 build_path, build_artifact.STATEFUL_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080048 f.write('STATEFUL_UPDATE')
Gilad Arnoldc65330c2012-09-20 15:17:48 -070049 with open(os.path.join(
50 build_path, build_artifact.ROOT_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080051 f.write('ROOT_UPDATE')
52 # AU payloads.
Gilad Arnoldc65330c2012-09-20 15:17:48 -070053 au_dir = os.path.join(build_path, common_util.AU_BASE)
54 nton_dir = os.path.join(au_dir, build + common_util.NTON_DIR_SUFFIX)
Frank Farzan37761d12011-12-01 14:29:08 -080055 os.makedirs(nton_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070056 with open(os.path.join(nton_dir, build_artifact.ROOT_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080057 f.write('ROOT_UPDATE')
Gilad Arnoldc65330c2012-09-20 15:17:48 -070058 mton_dir = os.path.join(au_dir, build + common_util.MTON_DIR_SUFFIX)
Frank Farzan37761d12011-12-01 14:29:08 -080059 os.makedirs(mton_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070060 with open(os.path.join(mton_dir, build_artifact.ROOT_UPDATE), 'w') as f:
Frank Farzan37761d12011-12-01 14:29:08 -080061 f.write('ROOT_UPDATE')
62
Chris Sosaea148d92012-03-06 16:22:04 -080063 self._good_mock_process = self.mox.CreateMock(subprocess.Popen)
64 self._good_mock_process.returncode = 0
65 self._bad_mock_process = self.mox.CreateMock(subprocess.Popen)
66 self._bad_mock_process.returncode = 1
67
Frank Farzan37761d12011-12-01 14:29:08 -080068 def tearDown(self):
69 shutil.rmtree(self._static_dir)
70 shutil.rmtree(self._outside_sandbox_dir)
Chris Sosa47a7d4e2012-03-28 11:26:55 -070071 shutil.rmtree(self._install_dir)
Frank Farzan37761d12011-12-01 14:29:08 -080072
73 def testParsePayloadList(self):
Chris Sosa1228a1a2012-05-22 17:12:13 -070074 """Tests we can parse the payload list into urls."""
Frank Farzan37761d12011-12-01 14:29:08 -080075 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -070076 'R17-1413.0.0-a1-b1346')
77 mton_basename = ('chromeos_R17-1412.0.0-a1-b1345_R17-1413.0.0-a1_'
78 'x86-mario_delta_dev.bin')
79 nton_basename = ('chromeos_R17-1413.0.0-a1_R17-1413.0.0-a1_'
80 'x86-mario_delta_dev.bin')
81 full_basename = ('chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
82
83 mton_url = '/'.join([archive_url_prefix, mton_basename])
84 nton_url = '/'.join([archive_url_prefix, nton_basename])
85 full_url = '/'.join([archive_url_prefix, full_basename])
86
Frank Farzan37761d12011-12-01 14:29:08 -080087 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -070088 common_util.ParsePayloadList(archive_url_prefix,
89 [full_basename, nton_basename,
90 mton_basename]))
Frank Farzan37761d12011-12-01 14:29:08 -080091 self.assertEqual([full_url, nton_url, mton_url],
92 [full_url_out, nton_url_out, mton_url_out])
93
94 archive_url_prefix = ('gs://chromeos-image-archive/x86-alex_he-release/'
95 'R18-1420.0.0-a1-b541')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -070096
97 mton_basename = ('chromeos_R18-1418.0.0-a1-b54a0_R18-1420.0.0-a1'
98 '_x86-alex_he_delta_dev.bin')
99 nton_basename = ('chromeos_R18-1420.0.0-a1_R18-1420.0.0-a1_'
100 'x86-alex_he_delta_dev.bin')
101 full_basename = ('chromeos_R18-1420.0.0-a1_x86-alex_he_full_dev.bin')
102
103 mton_url = '/'.join([archive_url_prefix, mton_basename])
104 nton_url = '/'.join([archive_url_prefix, nton_basename])
105 full_url = '/'.join([archive_url_prefix, full_basename])
106
Frank Farzan37761d12011-12-01 14:29:08 -0800107 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700108 common_util.ParsePayloadList(archive_url_prefix,
109 [full_basename, nton_basename,
110 mton_basename]))
Frank Farzan37761d12011-12-01 14:29:08 -0800111 self.assertEqual([full_url, nton_url, mton_url],
112 [full_url_out, nton_url_out, mton_url_out])
113
Chris Sosa1228a1a2012-05-22 17:12:13 -0700114 def testParsePayloadListWithoutDeltas(self):
115 """Tests we can parse the payload list when no delta updates exist."""
116 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700117 'R17-1413.0.0-a1-b1346')
118 full_basename = ('chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
119 full_url = '/'.join([archive_url_prefix, full_basename])
Chris Sosa1228a1a2012-05-22 17:12:13 -0700120 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700121 common_util.ParsePayloadList(
122 archive_url_prefix, [full_basename, '', '']))
Chris Sosa1228a1a2012-05-22 17:12:13 -0700123 self.assertEqual([full_url, None, None],
124 [full_url_out, nton_url_out, mton_url_out])
125
Chris Sosa781ba6d2012-04-11 12:44:43 -0700126 def testParsePartialPayloadList(self):
127 """Tests that we can parse a payload list with missing optional payload."""
128 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/'
129 'R17-1413.0.0-a1-b1346/')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700130 nton_basename = ('chromeos_R17-1413.0.0-a1_R17-1413.0.0-a1_x86-'
131 'mario_delta_dev.bin')
132 full_basename = ('chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
133
134 nton_url = '/'.join([archive_url_prefix, nton_basename])
135 full_url = '/'.join([archive_url_prefix, full_basename])
136
Chris Sosa781ba6d2012-04-11 12:44:43 -0700137 full_url_out, nton_url_out, mton_url_out = (
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700138 common_util.ParsePayloadList(archive_url_prefix,
139 [full_basename, nton_basename]))
Chris Sosa781ba6d2012-04-11 12:44:43 -0700140 self.assertEqual([full_url, nton_url, None],
141 [full_url_out, nton_url_out, mton_url_out])
142
Frank Farzan37761d12011-12-01 14:29:08 -0800143 def testInstallBuild(self):
Simon Glassf5019de2012-03-20 12:14:41 -0700144 # TODO(frankf): Implement this test
145 # self.fail('Not implemented.')
146 pass
Frank Farzan37761d12011-12-01 14:29:08 -0800147
148 def testPrepareAutotestPkgs(self):
Simon Glassf5019de2012-03-20 12:14:41 -0700149 # TODO(frankf): Implement this test
150 # self.fail('Not implemented.')
Scott Zawalski16954532012-03-20 15:31:36 -0400151 # TODO: implement
Simon Glassf5019de2012-03-20 12:14:41 -0700152 pass
Frank Farzan37761d12011-12-01 14:29:08 -0800153
154 def testSafeSandboxAccess(self):
155 # Path is in sandbox.
156 self.assertTrue(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700157 common_util.SafeSandboxAccess(
Frank Farzan37761d12011-12-01 14:29:08 -0800158 self._static_dir, os.path.join(self._static_dir, 'some-board')))
159
160 # Path is sandbox.
161 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700162 common_util.SafeSandboxAccess(self._static_dir, self._static_dir))
Frank Farzan37761d12011-12-01 14:29:08 -0800163
164 # Path is outside the sandbox.
165 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700166 common_util.SafeSandboxAccess(
167 self._static_dir, self._outside_sandbox_dir))
Frank Farzan37761d12011-12-01 14:29:08 -0800168
169 # Path contains '..'.
170 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700171 common_util.SafeSandboxAccess(
Frank Farzan37761d12011-12-01 14:29:08 -0800172 self._static_dir, os.path.join(self._static_dir, os.pardir)))
173
174 # Path contains symbolic link references.
175 os.chdir(self._static_dir)
176 os.symlink(os.pardir, 'parent')
177 self.assertFalse(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700178 common_util.SafeSandboxAccess(
Frank Farzan37761d12011-12-01 14:29:08 -0800179 self._static_dir, os.path.join(self._static_dir, os.pardir)))
180
181 def testAcquireReleaseLocks(self):
Gilad Arnold5174ca22012-09-12 10:49:09 -0700182 # Successful lock and unlock, removing the newly created directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700183 lock_file = common_util.AcquireLock(self._static_dir, 'test-lock')
Frank Farzan37761d12011-12-01 14:29:08 -0800184 self.assertTrue(os.path.exists(lock_file))
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700185 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Frank Farzan37761d12011-12-01 14:29:08 -0800186 self.assertFalse(os.path.exists(lock_file))
187
Gilad Arnold5174ca22012-09-12 10:49:09 -0700188 # Attempt to freshly create and lock an existing directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700189 common_util.AcquireLock(self._static_dir, 'test-lock')
190 common_util.ReleaseLock(self._static_dir, 'test-lock')
191 self.assertRaises(common_util.DevServerUtilError, common_util.AcquireLock,
192 self._static_dir, 'test-lock')
193 common_util.AcquireLock(self._static_dir, 'test-lock', create_once=False)
194 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Gilad Arnold5174ca22012-09-12 10:49:09 -0700195
196 # Sucessfully re-lock a pre-existing directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700197 common_util.AcquireLock(self._static_dir, 'test-lock')
198 common_util.ReleaseLock(self._static_dir, 'test-lock')
199 common_util.AcquireLock(self._static_dir, 'test-lock', create_once=False)
200 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Gilad Arnold5174ca22012-09-12 10:49:09 -0700201
202 # Attempt to lock an already locked directory.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700203 common_util.AcquireLock(self._static_dir, 'test-lock')
204 self.assertRaises(common_util.DevServerUtilError, common_util.AcquireLock,
205 self._static_dir, 'test-lock')
206 common_util.ReleaseLock(self._static_dir, 'test-lock', destroy=True)
Frank Farzan37761d12011-12-01 14:29:08 -0800207
208 def testFindMatchingBoards(self):
209 for key in TEST_LAYOUT:
210 # Partial match with multiple boards.
211 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700212 set(common_util.FindMatchingBoards(self._static_dir, key[:-5])),
Frank Farzan37761d12011-12-01 14:29:08 -0800213 set(TEST_LAYOUT.keys()))
214
215 # Absolute match.
216 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700217 common_util.FindMatchingBoards(self._static_dir, key), [key])
Frank Farzan37761d12011-12-01 14:29:08 -0800218
219 # Invalid partial match.
220 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700221 common_util.FindMatchingBoards(self._static_dir, 'asdfsadf'), [])
Frank Farzan37761d12011-12-01 14:29:08 -0800222
223 def testFindMatchingBuilds(self):
224 # Try a partial board and build match with single match.
225 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700226 common_util.FindMatchingBuilds(self._static_dir, 'test-board',
227 'R17-1413'),
Frank Farzan37761d12011-12-01 14:29:08 -0800228 [('test-board-1', 'R17-1413.0.0-a1-b1346')])
229
230 # Try a partial board and build match with multiple match.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700231 actual = set(common_util.FindMatchingBuilds(
Frank Farzan37761d12011-12-01 14:29:08 -0800232 self._static_dir, 'test-board', 'R17'))
233 expected = set([('test-board-1', 'R17-1413.0.0-a1-b1346'),
234 ('test-board-1', 'R17-18.0.0-a1-b1346'),
235 ('test-board-2', 'R17-2.0.0-a1-b1346')])
236 self.assertEqual(actual, expected)
237
238 def testGetLatestBuildVersion(self):
239 self.assertEqual(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700240 common_util.GetLatestBuildVersion(self._static_dir, 'test-board-1'),
Frank Farzan37761d12011-12-01 14:29:08 -0800241 'R17-1413.0.0-a1-b1346')
242
Scott Zawalski16954532012-03-20 15:31:36 -0400243 def testGetLatestBuildVersionLatest(self):
244 """Test that we raise DevServerUtilError when a build dir is empty."""
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700245 self.assertRaises(common_util.DevServerUtilError,
246 common_util.GetLatestBuildVersion,
Scott Zawalski16954532012-03-20 15:31:36 -0400247 self._static_dir, 'test-board-3')
Frank Farzan37761d12011-12-01 14:29:08 -0800248
Scott Zawalski16954532012-03-20 15:31:36 -0400249 def testGetLatestBuildVersionUnknownBuild(self):
250 """Test that we raise DevServerUtilError when a build dir does not exist."""
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700251 self.assertRaises(common_util.DevServerUtilError,
252 common_util.GetLatestBuildVersion,
Scott Zawalski16954532012-03-20 15:31:36 -0400253 self._static_dir, 'bad-dir')
Frank Farzan37761d12011-12-01 14:29:08 -0800254
Scott Zawalski16954532012-03-20 15:31:36 -0400255 def testGetLatestBuildVersionMilestone(self):
256 """Test that we can get builds based on milestone."""
257 expected_build_str = 'R16-2241.0.0-a0-b2'
258 milestone = 'R16'
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700259 build_str = common_util.GetLatestBuildVersion(
Scott Zawalski16954532012-03-20 15:31:36 -0400260 self._static_dir, 'test-board-2', milestone)
261 self.assertEqual(expected_build_str, build_str)
Frank Farzan37761d12011-12-01 14:29:08 -0800262
263 def testCloneBuild(self):
264 test_prefix = 'abc'
265 test_tag = test_prefix + '/123'
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700266 abc_path = os.path.join(
267 self._static_dir, common_util.DEV_BUILD_PREFIX, test_tag)
Frank Farzan37761d12011-12-01 14:29:08 -0800268
269 os.mkdir(os.path.join(self._static_dir, test_prefix))
270
271 # Verify leaf path is created and proper values returned.
272 board, builds = TEST_LAYOUT.items()[0]
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700273 dev_build = common_util.CloneBuild(
274 self._static_dir, board, builds[0], test_tag)
Frank Farzan37761d12011-12-01 14:29:08 -0800275 self.assertEquals(dev_build, abc_path)
276 self.assertTrue(os.path.exists(abc_path))
277 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700278 abc_path, build_artifact.TEST_IMAGE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800279 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700280 abc_path, build_artifact.ROOT_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800281 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700282 abc_path, build_artifact.STATEFUL_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800283
284 # Verify force properly removes the old directory.
285 junk_path = os.path.join(dev_build, 'junk')
286 with open(junk_path, 'w') as f:
287 f.write('hello!')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700288 remote_dir = common_util.CloneBuild(
Frank Farzan37761d12011-12-01 14:29:08 -0800289 self._static_dir, board, builds[0], test_tag, force=True)
290 self.assertEquals(remote_dir, abc_path)
291 self.assertTrue(os.path.exists(abc_path))
292 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700293 abc_path, build_artifact.TEST_IMAGE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800294 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700295 abc_path, build_artifact.ROOT_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800296 self.assertTrue(os.path.isfile(os.path.join(
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700297 abc_path, build_artifact.STATEFUL_UPDATE)))
Frank Farzan37761d12011-12-01 14:29:08 -0800298 self.assertFalse(os.path.exists(junk_path))
299
300 def testGetControlFile(self):
301 control_file_dir = os.path.join(
Chris Sosaea148d92012-03-06 16:22:04 -0800302 self._static_dir, 'test-board-1', 'R17-1413.0.0-a1-b1346', 'autotest',
303 'server', 'site_tests', 'network_VPN')
Frank Farzan37761d12011-12-01 14:29:08 -0800304 os.makedirs(control_file_dir)
305 with open(os.path.join(control_file_dir, 'control'), 'w') as f:
306 f.write('hello!')
307
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700308 control_content = common_util.GetControlFile(
Chris Sosaea148d92012-03-06 16:22:04 -0800309 self._static_dir, 'test-board-1/R17-1413.0.0-a1-b1346',
Frank Farzan37761d12011-12-01 14:29:08 -0800310 os.path.join('server', 'site_tests', 'network_VPN', 'control'))
311 self.assertEqual(control_content, 'hello!')
312
313 def testListAutoupdateTargets(self):
314 for board, builds in TEST_LAYOUT.iteritems():
315 for build in builds:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700316 au_targets = common_util.ListAutoupdateTargets(
317 self._static_dir, board, build)
Frank Farzan37761d12011-12-01 14:29:08 -0800318 self.assertEqual(set(au_targets),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700319 set([build + common_util.NTON_DIR_SUFFIX,
320 build + common_util.MTON_DIR_SUFFIX]))
Frank Farzan37761d12011-12-01 14:29:08 -0800321
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700322 def testGatherArtifactDownloads(self):
323 """Tests that we can gather the correct download requirements."""
324 build = 'R17-1413.0.0-a1-b1346'
325 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
326 build)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700327 mock_data = 'mock data\nmock_data'
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700328 payloads = map(lambda x: '/'.join([archive_url_prefix, x]),
329 ['p1', 'p2', 'p3'])
330 expected_payloads = payloads + map(
331 lambda x: '/'.join([archive_url_prefix, x]),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700332 [build_artifact.STATEFUL_UPDATE,
333 build_artifact.AUTOTEST_ZIPPED_PACKAGE,
334 build_artifact.TEST_SUITES_PACKAGE])
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700335 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700336 self.mox.StubOutWithMock(common_util, 'IsAvailable')
337 self.mox.StubOutWithMock(common_util, 'ParsePayloadList')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700338
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700339 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700340 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700341 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700342 common_util.IsAvailable(
343 mox.IgnoreArg(), mock_data.splitlines()).AndReturn(True)
344 common_util.ParsePayloadList(archive_url_prefix,
345 mock_data.splitlines()).AndReturn(payloads)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700346
347 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700348 artifacts = common_util.GatherArtifactDownloads(
Gilad Arnold6f99b982012-09-12 10:49:40 -0700349 self._static_dir, archive_url_prefix, self._install_dir, build)
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700350 for index, artifact in enumerate(artifacts):
351 self.assertEqual(artifact._gs_path, expected_payloads[index])
352 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700353
354 self.mox.VerifyAll()
355
Chris Sosa781ba6d2012-04-11 12:44:43 -0700356 def testGatherArtifactDownloadsWithoutMton(self):
357 """Gather the correct download requirements without mton delta."""
358 build = 'R17-1413.0.0-a1-b1346'
359 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
360 build)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700361 mock_data = 'mock data\nmock_data\nmock_data'
Chris Sosa781ba6d2012-04-11 12:44:43 -0700362 payloads = map(lambda x: '/'.join([archive_url_prefix, x]),
363 ['p1', 'p2'])
364 expected_payloads = payloads + map(
365 lambda x: '/'.join([archive_url_prefix, x]),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700366 [build_artifact.STATEFUL_UPDATE,
367 build_artifact.AUTOTEST_ZIPPED_PACKAGE,
368 build_artifact.TEST_SUITES_PACKAGE])
Chris Sosa781ba6d2012-04-11 12:44:43 -0700369 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700370 self.mox.StubOutWithMock(common_util, 'IsAvailable')
371 self.mox.StubOutWithMock(common_util, 'ParsePayloadList')
Chris Sosa781ba6d2012-04-11 12:44:43 -0700372
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700373 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700374 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Sosa781ba6d2012-04-11 12:44:43 -0700375 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700376 common_util.IsAvailable(
377 mox.IgnoreArg(), mock_data.splitlines()).AndReturn(True)
378 common_util.ParsePayloadList(archive_url_prefix,
379 mock_data.splitlines()
380 ).AndReturn(payloads + [None])
Chris Sosa781ba6d2012-04-11 12:44:43 -0700381
382 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700383 artifacts = common_util.GatherArtifactDownloads(
Gilad Arnold6f99b982012-09-12 10:49:40 -0700384 self._static_dir, archive_url_prefix, self._install_dir, build)
Chris Sosa781ba6d2012-04-11 12:44:43 -0700385 for index, artifact in enumerate(artifacts):
386 self.assertEqual(artifact._gs_path, expected_payloads[index])
387 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
Chris Sosa1228a1a2012-05-22 17:12:13 -0700388
389 self.mox.VerifyAll()
390
391 def testGatherArtifactDownloadsWithoutMtonOrNton(self):
392 """Gather the correct download requirements without delta payloads."""
393 build = 'R17-1413.0.0-a1-b1346'
394 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
395 build)
396 mock_data = 'mock data\nmock_data'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700397
Chris Sosa1228a1a2012-05-22 17:12:13 -0700398 payloads = map(lambda x: '/'.join([archive_url_prefix, x]),
399 ['p1'])
400 expected_payloads = payloads + map(
401 lambda x: '/'.join([archive_url_prefix, x]),
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700402 [build_artifact.STATEFUL_UPDATE,
403 build_artifact.AUTOTEST_ZIPPED_PACKAGE,
404 build_artifact.TEST_SUITES_PACKAGE])
Chris Sosa1228a1a2012-05-22 17:12:13 -0700405 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700406 self.mox.StubOutWithMock(common_util, 'IsAvailable')
407 self.mox.StubOutWithMock(common_util, 'ParsePayloadList')
Chris Sosa1228a1a2012-05-22 17:12:13 -0700408
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700409 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700410 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Sosa1228a1a2012-05-22 17:12:13 -0700411 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700412 common_util.IsAvailable(
413 mox.IgnoreArg(), mock_data.splitlines()).AndReturn(True)
414 common_util.ParsePayloadList(archive_url_prefix,
415 mock_data.splitlines()
416 ).AndReturn(payloads + [None, None])
Chris Sosa1228a1a2012-05-22 17:12:13 -0700417
418 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700419 artifacts = common_util.GatherArtifactDownloads(
Gilad Arnold6f99b982012-09-12 10:49:40 -0700420 self._static_dir, archive_url_prefix, self._install_dir, build)
Chris Sosa1228a1a2012-05-22 17:12:13 -0700421 for index, artifact in enumerate(artifacts):
422 self.assertEqual(artifact._gs_path, expected_payloads[index])
423 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
Chris Sosa781ba6d2012-04-11 12:44:43 -0700424
425 self.mox.VerifyAll()
426
Chris Masone816e38c2012-05-02 12:22:36 -0700427 def testGatherSymbolArtifactDownloads(self):
428 """Tests that we can find debug symbol artifacts to download."""
429 build = 'R17-1413.0.0-a1-b1346'
430 archive_url_prefix = ('gs://chromeos-image-archive/x86-mario-release/' +
431 build)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700432 symbol_url = '/'.join([archive_url_prefix,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700433 build_artifact.DEBUG_SYMBOLS])
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700434 uploaded_list_url = '/'.join([archive_url_prefix,
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700435 common_util.UPLOADED_LIST])
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700436 mock_data = 'mock-tarball.tgz\nmock-debug.tgz'
Chris Masone816e38c2012-05-02 12:22:36 -0700437 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Chris Masone816e38c2012-05-02 12:22:36 -0700438
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700439 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700440 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Masone816e38c2012-05-02 12:22:36 -0700441 mox.IgnoreArg()).AndReturn(mock_data)
442
443 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700444 artifacts = common_util.GatherSymbolArtifactDownloads(
Chris Masone816e38c2012-05-02 12:22:36 -0700445 self._static_dir, archive_url_prefix, self._install_dir)
446 for index, artifact in enumerate(artifacts):
447 self.assertEqual(artifact._gs_path, symbol_url)
448 self.assertTrue(artifact._tmp_staging_dir.startswith(self._static_dir))
449
450 self.mox.VerifyAll()
451
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700452 def testIsAvailable(self):
453 """Test that we can detect whether the target artifacts are avaialble."""
454 # Test when the all target files are available
455 pattern_list = ['_full_', 'autotest.tar']
456 uploaded_list = ['chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin',
457 'debug.tgz',
458 'autotest.tar.bz2']
459
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700460 available = common_util.IsAvailable(pattern_list, uploaded_list)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700461 self.assertTrue(available)
462
463 # Test when some target files are missing
464 pattern_list = ['_full_', 'autotest.tar']
465 uploaded_list = ['chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin',
466 'debug.tgz']
467
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700468 available = common_util.IsAvailable(pattern_list, uploaded_list)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700469 self.assertFalse(available)
470
471 def testWaitUntilAvailable(self):
472 """Test that we can poll until all target artifacts are available."""
Chris Masone816e38c2012-05-02 12:22:36 -0700473 build = 'R17-1413.0.0-a1-b1346'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700474 archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
475 'R17-1413.0.0-a1-b1346')
476 to_wait_list = ['_full_']
Chris Masone816e38c2012-05-02 12:22:36 -0700477 mock_data = 'mock data\nmock_data\nmock_data'
Chris Masone816e38c2012-05-02 12:22:36 -0700478
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700479 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700480 self.mox.StubOutWithMock(common_util, 'IsAvailable')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700481
482 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700483 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Chris Masone816e38c2012-05-02 12:22:36 -0700484 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700485 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True)
Chris Masone816e38c2012-05-02 12:22:36 -0700486
487 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700488 uploaded_list = common_util.WaitUntilAvailable(
489 to_wait_list, archive_url, 'UNIT TEST', delay=1)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700490 self.assertEqual(uploaded_list, mock_data.splitlines())
Chris Masone816e38c2012-05-02 12:22:36 -0700491 self.mox.VerifyAll()
492
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700493 def testWaitUntilAvailableWithRetry(self):
494 """Test that we can poll until all target artifacts are available."""
Chris Masone816e38c2012-05-02 12:22:36 -0700495 build = 'R17-1413.0.0-a1-b1346'
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700496 archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
497 'R17-1413.0.0-a1-b1346')
498 to_wait_list = ['_full_']
499 mock_data = 'mock data\nmock_data\nmock_data'
Chris Masone816e38c2012-05-02 12:22:36 -0700500
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700501 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700502 self.mox.StubOutWithMock(common_util, 'IsAvailable')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700503
504 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700505 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700506 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700507 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700508
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700509 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700510 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700511 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True)
Chris Masone816e38c2012-05-02 12:22:36 -0700512
513 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700514 uploaded_list = common_util.WaitUntilAvailable(
515 to_wait_list, archive_url, 'UNIT TEST', delay=1)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700516 self.assertEqual(uploaded_list, mock_data.splitlines())
Chris Masone816e38c2012-05-02 12:22:36 -0700517 self.mox.VerifyAll()
518
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700519 def testWaitUntilAvailableTimeout(self):
520 """Test that we wait for the target artifacts until timeout occurs."""
521 build = 'R17-1413.0.0-a1-b1346'
522 archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
523 'R17-1413.0.0-a1-b1346')
524 to_wait_list = ['_full_']
525 mock_data = 'mock data\nmock_data\nmock_data'
526
527 self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700528 self.mox.StubOutWithMock(common_util, 'IsAvailable')
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700529
530 # GSUtil cat gs://archive_url_prefix/UPLOADED.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700531 gsutil_util.GSUtilRun(mox.StrContains(common_util.UPLOADED_LIST),
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700532 mox.IgnoreArg()).AndReturn(mock_data)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700533 common_util.IsAvailable(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False)
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700534
535 self.mox.ReplayAll()
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700536 self.assertRaises(common_util.DevServerUtilError,
537 common_util.WaitUntilAvailable,
Yu-Ju Hong825ddc32012-08-13 18:47:10 -0700538 to_wait_list,
539 archive_url,
540 'UNIT TEST',
541 delay=2,
542 timeout=1)
543 self.mox.VerifyAll()
Frank Farzan37761d12011-12-01 14:29:08 -0800544
545if __name__ == '__main__':
546 unittest.main()