devserver: make artifact names globs by default
What this means is that, by default, we'll be using a glob expression
(i.e. a path that may contain shell-style wildcards) instead of a regex
to identify artifacts. We are still allowing artifacts to be defined by
a regex, as this is useful for some purposes such as disallowing certain
file name patterns; this semantics, however, needs to be specifically
selected via a flag.
Some additional changes included:
* Eliminate enforcing a single file in gsutil_util.GetGSNamesWithWait();
this is already done in the caller function and there's no need to
complicate the otherwise generic logic in this module.
* Have DeltaPayloadsArtifact override various name filtering parameters
internally; letting the user define these may cause inconsistencies
(especially given the glob vs regex semantics), makes it less robust
and is not needed in practice.
* Fixes passing of dictionary args (as opposed to list args) to artifact
objects via artifact factory.
* Fixes a great many gpylint errors/warnings.
BUG=chromium:280220
TEST=Unit tests
TEST=gpylint
Change-Id: I9ea59b7f8962a71f7b387b99b0577932e753c2eb
Reviewed-on: https://chromium-review.googlesource.com/167434
Reviewed-by: Gilad Arnold <garnold@chromium.org>
Tested-by: Gilad Arnold <garnold@chromium.org>
Commit-Queue: Gilad Arnold <garnold@chromium.org>
diff --git a/gsutil_util_unittest.py b/gsutil_util_unittest.py
index 638c5fe..0a493fc 100755
--- a/gsutil_util_unittest.py
+++ b/gsutil_util_unittest.py
@@ -70,40 +70,21 @@
'from', 'to')
self.mox.VerifyAll()
- def testGSNamesFromList(self):
- """Test that we can detect whether the target artifacts are available."""
- # Test when the all target files are available
- pattern = '.*_full_.*'
- uploaded_list = ['chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin',
- 'debug.tgz',
- 'autotest.tar.bz2']
-
- names = gsutil_util._GetGSNamesFromList(uploaded_list, pattern)
- self.assertEqual(names[0],
- 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
-
- bad_pattern = '_delta_'
- # Test when some target files are missing
- names = gsutil_util._GetGSNamesFromList(uploaded_list, bad_pattern)
- self.assertEqual(names, [])
-
def testGetGSNamesWithWait(self):
"""Test that we get the target artifact that is available."""
archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
'R17-1413.0.0-a1-b1346')
name = 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'
- pattern = '_full_'
+ pattern = '*_full_*'
mock_data = 'mock data\nmock_data\nmock_data'
msg = 'UNIT TEST'
self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
- self.mox.StubOutWithMock(gsutil_util, '_GetGSNamesFromList')
# GSUtil cat gs://archive_url_prefix/UPLOADED.
gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
- mox.IgnoreArg()).AndReturn(mock_data)
- gsutil_util._GetGSNamesFromList(mock_data.split('\n'),
- pattern).AndReturn([name])
+ mox.IgnoreArg()).AndReturn(
+ '%s\n%s' % (mock_data, name))
self.mox.ReplayAll()
# Timeout explicitly set to 0 to test that we always run at least once.
@@ -117,23 +98,19 @@
archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
'R17-1413.0.0-a1-b1346')
name = 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'
- pattern = '_full_'
+ pattern = '*_full_*'
mock_data = 'mock data\nmock_data\nmock_data'
msg = 'UNIT TEST'
self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
- self.mox.StubOutWithMock(gsutil_util, '_GetGSNamesFromList')
# GSUtil cat gs://archive_url_prefix/UPLOADED.
gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
mox.IgnoreArg()).AndReturn(mock_data)
- gsutil_util._GetGSNamesFromList(mock_data.split('\n'),
- pattern).AndReturn(None)
gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
- mox.IgnoreArg()).AndReturn(mock_data)
- gsutil_util._GetGSNamesFromList(mox.IgnoreArg(),
- mox.IgnoreArg()).AndReturn([name])
+ mox.IgnoreArg()).AndReturn(
+ '%s\n%s' % (mock_data, name))
self.mox.ReplayAll()
returned_names = gsutil_util.GetGSNamesWithWait(
@@ -145,18 +122,15 @@
"""Test that we wait for the target artifacts until timeout occurs."""
archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
'R17-1413.0.0-a1-b1346')
- pattern = '_full_'
+ pattern = '*_full_*'
mock_data = 'mock data\nmock_data\nmock_data'
msg = 'UNIT TEST'
self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
- self.mox.StubOutWithMock(gsutil_util, '_GetGSNamesFromList')
# GSUtil cat gs://archive_url_prefix/UPLOADED.
gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
mox.IgnoreArg()).AndReturn(mock_data)
- gsutil_util._GetGSNamesFromList(mock_data.split('\n'),
- pattern).AndReturn(None)
self.mox.ReplayAll()
returned_name = gsutil_util.GetGSNamesWithWait(
@@ -184,10 +158,12 @@
mox.IgnoreArg()).AndReturn(mock_data2)
self.mox.ReplayAll()
url = ''
- self.assertEqual(gsutil_util.GetLatestVersionFromGSDir(url),
- '3912.101.0')
- self.assertEqual(gsutil_util.GetLatestVersionFromGSDir(url),
- 'R28-3912.101.0')
+ self.assertEqual(
+ gsutil_util.GetLatestVersionFromGSDir(url, with_release=False),
+ '3912.101.0')
+ self.assertEqual(
+ gsutil_util.GetLatestVersionFromGSDir(url, with_release=True),
+ 'R28-3912.101.0')
self.mox.VerifyAll()
if __name__ == '__main__':