xz_auto: disable pixz's special tar indexing bits
At the moment, running `xz_auto` on an empty file with `pixz` on `$PATH`
breaks. It seems this related to pixz's tar file autodetection, which
isn't something we really make use of at the moment. Disable it, since
it seems to have no impact on pixz's ability to decompress things in
parallel.
Unfortunately, disabling this on both ends (compression and
decompression) seems like it leads to significant badness if we try to
decompress tar'ed files which have the index (...of which, we have a few
scattered throughout). The simplest solution to this seems to be hacking
around the place where badness exists.
BUG=b:202735786
TEST=Unittests pass
Change-Id: I0ba50ec8094a881d0926307717e9e7020c9d842d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/3248615
Tested-by: George Burgess <gbiv@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Commit-Queue: George Burgess <gbiv@chromium.org>
diff --git a/scripts/xz_auto_unittest.py b/scripts/xz_auto_unittest.py
index a652095..76cfd62 100644
--- a/scripts/xz_auto_unittest.py
+++ b/scripts/xz_auto_unittest.py
@@ -26,6 +26,8 @@
class XzAutoTests(cros_test_lib.MockTempDirTestCase):
"""Various tests for xz_auto."""
+ TEST_FILE_CONTENTS = (b'', b'some random file contents')
+
def DisablePixzForCurrentTest(self):
"""Disables the use of pixz for the current test."""
# This will be cleaned up by cros_test_lib, so no need to addCleanup.
@@ -75,105 +77,116 @@
with self.assertRaises(ExecvpStopError):
xz_auto.ExecDecompressCommand(stdout=False, argv=[])
- def _TestFileCompressionImpl(self):
+ def _TestFileCompressionImpl(self, test_empty_file=True):
"""Tests that compressing a file with xz_auto WAI."""
- file_contents = b'some random file contents'
- file_location = os.path.join(self.tempdir, 'file.txt')
- osutils.WriteFile(file_location, file_contents, mode='wb')
-
xz_auto_script = str(FindXzAutoLocation())
- cros_build_lib.run(
- [
- xz_auto_script,
- file_location,
- ],
- check=True,
- )
- xz_location = file_location + '.xz'
- self.assertExists(xz_location)
- self.assertNotExists(file_location)
- cros_build_lib.run([
- xz_auto_script,
- '--decompress',
- xz_location,
- ])
- self.assertNotExists(xz_location)
- self.assertExists(file_location)
- self.assertEqual(
- osutils.ReadFile(file_location, mode='rb'),
- file_contents,
- )
+ test_file_contents = self.TEST_FILE_CONTENTS
+ if not test_empty_file:
+ test_file_contents = (x for x in test_file_contents if x)
+
+ for file_contents in test_file_contents:
+ file_location = os.path.join(self.tempdir, 'file.txt')
+ osutils.WriteFile(file_location, file_contents, mode='wb')
+
+ cros_build_lib.run(
+ [
+ xz_auto_script,
+ file_location,
+ ],
+ )
+
+ xz_location = file_location + '.xz'
+ self.assertExists(xz_location)
+ self.assertNotExists(file_location)
+ cros_build_lib.run([
+ xz_auto_script,
+ '--decompress',
+ xz_location,
+ ])
+ self.assertNotExists(xz_location)
+ self.assertExists(file_location)
+ self.assertEqual(
+ osutils.ReadFile(file_location, mode='rb'),
+ file_contents,
+ )
def _TestStdoutCompressionImpl(self):
"""Tests that compressing stdstreams with xz_auto WAI."""
- file_contents = b'some random file contents'
xz_auto_script = str(FindXzAutoLocation())
+ for file_contents in self.TEST_FILE_CONTENTS:
+ run_result = cros_build_lib.run(
+ [
+ xz_auto_script,
+ '-c',
+ ],
+ capture_output=True,
+ input=file_contents,
+ )
- run_result = cros_build_lib.run(
- [
- xz_auto_script,
- '-c',
- ],
- capture_output=True,
- input=file_contents,
- )
+ compressed_file = run_result.stdout
+ self.assertNotEqual(compressed_file, file_contents)
- compressed_file = run_result.stdout
- self.assertNotEqual(compressed_file, file_contents)
-
- run_result = cros_build_lib.run(
- [
- xz_auto_script,
- '--decompress',
- '-c',
- ],
- input=compressed_file,
- capture_output=True,
- )
- uncompressed_file = run_result.stdout
- self.assertEqual(file_contents, uncompressed_file)
+ run_result = cros_build_lib.run(
+ [
+ xz_auto_script,
+ '--decompress',
+ '-c',
+ ],
+ input=compressed_file,
+ capture_output=True,
+ )
+ uncompressed_file = run_result.stdout
+ self.assertEqual(file_contents, uncompressed_file)
def _TestStdoutCompressionFromFileImpl(self):
"""Tests that compression of a file & outputting to stdout works.
Pixz has some semi-weird behavior here (b/202735786).
"""
- file_contents = b'some random file contents'
xz_auto_script = str(FindXzAutoLocation())
- file_location = os.path.join(self.tempdir, 'file.txt')
- osutils.WriteFile(file_location, file_contents, mode='wb')
+ for file_contents in self.TEST_FILE_CONTENTS:
+ file_location = os.path.join(self.tempdir, 'file.txt')
+ osutils.WriteFile(file_location, file_contents, mode='wb')
- run_result = cros_build_lib.run(
- [
- xz_auto_script,
- '-c',
- file_location,
- ],
- capture_output=True,
- )
+ run_result = cros_build_lib.run(
+ [
+ xz_auto_script,
+ '-c',
+ file_location,
+ ],
+ capture_output=True,
+ )
- compressed_file = run_result.stdout
- self.assertExists(file_location)
- self.assertNotEqual(compressed_file, file_contents)
+ compressed_file = run_result.stdout
+ self.assertExists(file_location)
+ self.assertNotEqual(compressed_file, file_contents)
- run_result = cros_build_lib.run(
- [
- xz_auto_script,
- '--decompress',
- '-c',
- ],
- capture_output=True,
- input=compressed_file,
- )
- uncompressed_file = run_result.stdout
- self.assertEqual(file_contents, uncompressed_file)
+ run_result = cros_build_lib.run(
+ [
+ xz_auto_script,
+ '--decompress',
+ '-c',
+ ],
+ capture_output=True,
+ input=compressed_file,
+ )
+ uncompressed_file = run_result.stdout
+ self.assertEqual(file_contents, uncompressed_file)
@unittest.skipIf(not xz_auto.HasPixz(), 'need pixz for this test')
def testFileCompressionWithPixzWorks(self):
"""Tests that compressing a file with pixz WAI."""
self._TestFileCompressionImpl()
+ # We fall back to `xz` with small files. Make sure we actually cover the
+ # pixz case, too. We disable testing of empty inputs, since pixz breaks
+ # with those.
+ #
+ # cros_test_lib will clean this var up at the end of the test.
+ os.environ[xz_auto.XZ_DISABLE_VAR] = '1'
+ self._TestFileCompressionImpl(test_empty_file=False)
+
@unittest.skipIf(not xz_auto.HasPixz(), 'need pixz for this test')
def testStdoutCompressionWithPixzWorks(self):
"""Tests that compressing `stdout` with pixz WAI."""