Reland "cbuildbot_launch: Clean distfiles cache when too old"
This is a reland of d2cb40af59d51077daef3adca2692d3c00cdef45
diff: Changed to using unix time instead of stip time because
cbuildbot_launch runs in an environment that does not have pytz.
Original change's description:
> cbuildbot_launch: Clean distfiles cache when too old
>
> - Use the cbuildbot_launch to persist time since last cleanup of
> distfiles.
> - Do this in a way that old format state without the timestamp is
> simply updated with a timestamp.
> - Clean distfiles cache when it is more than 8 days old.
>
> + make BuildrootCleanup tests more behavioural.
>
> BUG=chromium:814989
> TEST=unittests.
>
> Change-Id: I0f1c74993f3dd59a16da333fb1ff49056c63086b
> Reviewed-on: https://chromium-review.googlesource.com/993759
> Commit-Ready: Prathmesh Prabhu <pprabhu@chromium.org>
> Tested-by: Prathmesh Prabhu <pprabhu@chromium.org>
> Reviewed-by: Don Garrett <dgarrett@chromium.org>
BUG=chromium:814989
TEST=unittests.
Change-Id: I9a784091684ea10e3316e543a5cb0121bc331490
Reviewed-on: https://chromium-review.googlesource.com/998392
Reviewed-by: Don Garrett <dgarrett@chromium.org>
Tested-by: Prathmesh Prabhu <pprabhu@chromium.org>
diff --git a/scripts/cbuildbot_launch_unittest.py b/scripts/cbuildbot_launch_unittest.py
index 3fdbc4d..e5f2acd 100644
--- a/scripts/cbuildbot_launch_unittest.py
+++ b/scripts/cbuildbot_launch_unittest.py
@@ -3,12 +3,13 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-"""Unit tests for chromite.lib.git and helpers for testing that module."""
+"""Unit tests for chromite.scripts.cbuildbot_launch."""
from __future__ import print_function
import mock
import os
+import time
from chromite.cbuildbot import repository
from chromite.lib import constants
@@ -18,7 +19,6 @@
from chromite.lib import osutils
from chromite.scripts import cbuildbot_launch
-
EXPECTED_MANIFEST_URL = 'https://chrome-internal-review.googlesource.com/chromeos/manifest-internal' # pylint: disable=line-too-long
@@ -281,7 +281,8 @@
self.repo = os.path.join(self.buildroot, '.repo/repo')
self.chroot = os.path.join(self.buildroot, 'chroot/chroot')
self.general = os.path.join(self.buildroot, 'general/general')
- # TODO: Add .cache, and distfiles.
+ self.cache = os.path.join(self.buildroot, '.cache')
+ self.distfiles = os.path.join(self.cache, 'distfiles')
self.mock_repo = mock.MagicMock()
self.mock_repo.directory = self.buildroot
@@ -295,7 +296,7 @@
osutils.WriteFile(self.state, state)
# Create files.
- for f in (self.repo, self.chroot, self.general):
+ for f in (self.repo, self.chroot, self.general, self.distfiles):
osutils.Touch(f, makedirs=True)
def testNoBuildroot(self):
@@ -305,7 +306,10 @@
cbuildbot_launch.CleanBuildRoot(
self.root, self.mock_repo, self.metrics)
- self.assertEqual(osutils.ReadFile(self.state), '2 master')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'master')
+ self.assertIsNotNone(distfiles_ts)
def testBuildrootNoState(self):
"""Test CleanBuildRoot with no state information."""
@@ -315,10 +319,15 @@
cbuildbot_launch.CleanBuildRoot(
self.root, self.mock_repo, self.metrics)
- self.assertEqual(osutils.ReadFile(self.state), '2 master')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'master')
+ self.assertIsNotNone(distfiles_ts)
+
self.assertNotExists(self.repo)
self.assertNotExists(self.chroot)
self.assertNotExists(self.general)
+ self.assertNotExists(self.distfiles)
def testBuildrootFormatMismatch(self):
"""Test CleanBuildRoot with no state information."""
@@ -328,10 +337,15 @@
cbuildbot_launch.CleanBuildRoot(
self.root, self.mock_repo, self.metrics)
- self.assertEqual(osutils.ReadFile(self.state), '2 master')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'master')
+ self.assertIsNotNone(distfiles_ts)
+
self.assertNotExists(self.repo)
self.assertNotExists(self.chroot)
self.assertNotExists(self.general)
+ self.assertNotExists(self.distfiles)
def testBuildrootBranchChange(self):
"""Test CleanBuildRoot with a change in branches."""
@@ -342,10 +356,15 @@
cbuildbot_launch.CleanBuildRoot(
self.root, self.mock_repo, self.metrics)
- self.assertEqual(osutils.ReadFile(self.state), '2 branchB')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'branchB')
+ self.assertIsNotNone(distfiles_ts)
+
self.assertExists(self.repo)
self.assertNotExists(self.chroot)
self.assertExists(self.general)
+ self.assertNotExists(self.distfiles)
m.assert_called()
def testBuildrootBranchMatch(self):
@@ -356,10 +375,53 @@
cbuildbot_launch.CleanBuildRoot(
self.root, self.mock_repo, self.metrics)
- self.assertEqual(osutils.ReadFile(self.state), '2 branchA')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'branchA')
+ self.assertIsNotNone(distfiles_ts)
+
self.assertExists(self.repo)
self.assertExists(self.chroot)
self.assertExists(self.general)
+ self.assertExists(self.distfiles)
+
+ def testBuildrootDistfilesRecentCache(self):
+ """Test CleanBuildRoot does not delete distfiles when cache is recent."""
+ seed_distfiles_ts = time.time() - 60
+ self.populateBuildroot('2 branchA %f' % seed_distfiles_ts)
+ self.mock_repo.branch = 'branchA'
+
+ cbuildbot_launch.CleanBuildRoot(
+ self.root, self.mock_repo, self.metrics)
+
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'branchA')
+ # Same cache creation timestamp is rewritten to state.
+ self.assertEqual(distfiles_ts, seed_distfiles_ts)
+
+ self.assertExists(self.repo)
+ self.assertExists(self.chroot)
+ self.assertExists(self.general)
+ self.assertExists(self.distfiles)
+
+ def testBuildrootDistfilesCacheExpired(self):
+ """Test CleanBuildRoot when the distfiles cache is too old."""
+ self.populateBuildroot('2 branchA 100.000000')
+ self.mock_repo.branch = 'branchA'
+
+ cbuildbot_launch.CleanBuildRoot(
+ self.root, self.mock_repo, self.metrics)
+
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'branchA')
+ self.assertIsNotNone(distfiles_ts)
+
+ self.assertExists(self.repo)
+ self.assertExists(self.chroot)
+ self.assertExists(self.general)
+ self.assertNotExists(self.distfiles)
def testBuildrootRepoCleanFailure(self):
"""Test CleanBuildRoot with repo checkout failure."""
@@ -370,55 +432,84 @@
cbuildbot_launch.CleanBuildRoot(
self.root, self.mock_repo, self.metrics)
- self.assertEqual(osutils.ReadFile(self.state), '2 branchA')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'branchA')
+ self.assertIsNotNone(distfiles_ts)
+
self.assertNotExists(self.repo)
self.assertNotExists(self.chroot)
self.assertNotExists(self.general)
+ self.assertNotExists(self.distfiles)
def testGetState(self):
"""Test GetState."""
# No root dir.
results = cbuildbot_launch.GetState(self.root)
- self.assertEqual(results, (0, ''))
+ self.assertEqual(results, (0, '', None))
# Empty root dir.
osutils.SafeMakedirs(self.root)
results = cbuildbot_launch.GetState(self.root)
- self.assertEqual(results, (0, ''))
+ self.assertEqual(results, (0, '', None))
- # Empty Contents
+ # Empty contents
osutils.WriteFile(self.state, '')
results = cbuildbot_launch.GetState(self.root)
- self.assertEqual(results, (0, ''))
+ self.assertEqual(results, (0, '', None))
- # Old Format Contents
+ # Old format contents
osutils.WriteFile(self.state, 'happy-branch')
results = cbuildbot_launch.GetState(self.root)
- self.assertEqual(results, (0, ''))
+ self.assertEqual(results, (0, '', None))
- # Expected Contents
+ # Expected contents, without distfiles timestamp
osutils.WriteFile(self.state, '1 happy-branch')
results = cbuildbot_launch.GetState(self.root)
- self.assertEqual(results, (1, 'happy-branch'))
+ self.assertEqual(results, (1, 'happy-branch', None))
- # Future Contents
- osutils.WriteFile(self.state, '22 master')
- results = cbuildbot_launch.GetState(self.root)
- self.assertEqual(results, (22, 'master'))
+ # Expected contents
+ osutils.WriteFile(self.state, '1 happy-branch 1000.33')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 1)
+ self.assertEqual(branch, 'happy-branch')
+ self.assertEqual(distfiles_ts, 1000.33)
- # Read Write
+ # Future layout version contents
+ osutils.WriteFile(self.state, '22 happy-branch 222')
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 22)
+ self.assertEqual(branch, 'happy-branch')
+ self.assertEqual(distfiles_ts, 222)
+
+ # Read write
cbuildbot_launch.SetState('happy-branch', self.root)
- results = cbuildbot_launch.GetState(self.root)
- self.assertEqual(results, (2, 'happy-branch'))
+ version, branch, distfiles_ts = cbuildbot_launch.GetState(self.root)
+ self.assertEqual(version, 2)
+ self.assertEqual(branch, 'happy-branch')
+ self.assertIsNotNone(distfiles_ts)
def testSetState(self):
"""Test SetState."""
# Write out a state file.
osutils.SafeMakedirs(self.root)
cbuildbot_launch.SetState('happy-branch', self.root)
- self.assertEqual(osutils.ReadFile(self.state), '2 happy-branch')
+ state_file_parts = osutils.ReadFile(self.state).split()
+ self.assertEqual(state_file_parts[:2], ['2', 'happy-branch'])
+ # Will flake if this test takes > 1 hour to run.
+ self.assertGreater(float(state_file_parts[2]), time.time() - 3600)
+
+ # Explicitly provide a timestamp
+ cbuildbot_launch.SetState('happy-branch', self.root,
+ 333.33)
+ state_file_parts = osutils.ReadFile(self.state).split()
+ self.assertEqual(state_file_parts,
+ ['2', 'happy-branch', '333.330000'])
# Change to a future version.
self.PatchObject(cbuildbot_launch, 'BUILDROOT_BUILDROOT_LAYOUT', 22)
cbuildbot_launch.SetState('happy-branch', self.root)
- self.assertEqual(osutils.ReadFile(self.state), '22 happy-branch')
+ state_file_parts = osutils.ReadFile(self.state).split()
+ self.assertEqual(state_file_parts[:2], ['22', 'happy-branch'])
+ # Will flake if this test takes > 1 hour to run.
+ self.assertGreater(float(state_file_parts[2]), time.time() - 3600)