blob: 26ab3a7794caa2e57802710906e53939b17ac4b7 [file] [log] [blame]
Darin Petkovc3fd90c2011-05-11 14:23:00 -07001# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
rtc@google.comded22402009-10-26 22:36:21 +00002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Dale Curtisc9aaf3a2011-08-09 15:47:40 -07005import json
rtc@google.comded22402009-10-26 22:36:21 +00006import os
Chris Sosa05491b12010-11-08 17:14:16 -08007import subprocess
Darin Petkov2b2ff4b2010-07-27 15:02:09 -07008import time
Gilad Arnold0c9c8602012-10-02 23:58:58 -07009import urllib2
Don Garrett0ad09372010-12-06 16:20:30 -080010import urlparse
Chris Sosa7c931362010-10-11 19:49:01 -070011
Gilad Arnoldabb352e2012-09-23 01:24:27 -070012import cherrypy
13
14from build_util import BuildObject
Chris Sosa52148582012-11-15 15:35:58 -080015import autoupdate_lib
Gilad Arnold55a2a372012-10-02 09:46:32 -070016import common_util
Gilad Arnoldc65330c2012-09-20 15:17:48 -070017import log_util
Chris Sosa8dd80092012-12-10 13:39:11 -080018import subprocess
Chris Sosa05491b12010-11-08 17:14:16 -080019
Gilad Arnoldc65330c2012-09-20 15:17:48 -070020
21# Module-local log function.
Chris Sosa8dd80092012-12-10 13:39:11 -080022def _Log(message, *args):
23 return log_util.LogWithTag('UPDATE', message, *args)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070024
rtc@google.comded22402009-10-26 22:36:21 +000025
Chris Sosa417e55d2011-01-25 16:40:48 -080026UPDATE_FILE = 'update.gz'
Chris Sosa8dd80092012-12-10 13:39:11 -080027METADATA_FILE = 'update.meta'
Chris Sosa417e55d2011-01-25 16:40:48 -080028STATEFUL_FILE = 'stateful.tgz'
29CACHE_DIR = 'cache'
Chris Sosa0356d3b2010-09-16 15:46:22 -070030
Don Garrett0ad09372010-12-06 16:20:30 -080031
Gilad Arnold0c9c8602012-10-02 23:58:58 -070032class AutoupdateError(Exception):
33 """Exception classes used by this module."""
34 pass
35
36
Don Garrett0ad09372010-12-06 16:20:30 -080037def _ChangeUrlPort(url, new_port):
38 """Return the URL passed in with a different port"""
39 scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
40 host_port = netloc.split(':')
41
42 if len(host_port) == 1:
43 host_port.append(new_port)
44 else:
45 host_port[1] = new_port
46
47 print host_port
48 netloc = "%s:%s" % tuple(host_port)
49
50 return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
51
Chris Sosa8dd80092012-12-10 13:39:11 -080052def _NonePathJoin(*args):
53 """os.path.join that filters None's from the argument list."""
54 return os.path.join(*filter(None, args))
Don Garrett0ad09372010-12-06 16:20:30 -080055
Chris Sosa8dd80092012-12-10 13:39:11 -080056
57class HostInfo(object):
Gilad Arnold286a0062012-01-12 13:47:02 -080058 """Records information about an individual host.
59
60 Members:
61 attrs: Static attributes (legacy)
62 log: Complete log of recorded client entries
63 """
64
65 def __init__(self):
66 # A dictionary of current attributes pertaining to the host.
67 self.attrs = {}
68
69 # A list of pairs consisting of a timestamp and a dictionary of recorded
70 # attributes.
71 self.log = []
72
73 def __repr__(self):
74 return 'attrs=%s, log=%s' % (self.attrs, self.log)
75
76 def AddLogEntry(self, entry):
77 """Append a new log entry."""
78 # Append a timestamp.
79 assert not 'timestamp' in entry, 'Oops, timestamp field already in use'
80 entry['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S')
81 # Add entry to hosts' message log.
82 self.log.append(entry)
83
Gilad Arnold286a0062012-01-12 13:47:02 -080084
Chris Sosa8dd80092012-12-10 13:39:11 -080085class HostInfoTable(object):
Gilad Arnold286a0062012-01-12 13:47:02 -080086 """Records information about a set of hosts who engage in update activity.
87
88 Members:
89 table: Table of information on hosts.
90 """
91
92 def __init__(self):
93 # A dictionary of host information. Keys are normally IP addresses.
94 self.table = {}
95
96 def __repr__(self):
97 return '%s' % self.table
98
99 def GetInitHostInfo(self, host_id):
100 """Return a host's info object, or create a new one if none exists."""
101 return self.table.setdefault(host_id, HostInfo())
102
103 def GetHostInfo(self, host_id):
104 """Return an info object for given host, if such exists."""
Chris Sosa1885d032012-11-29 17:07:27 -0800105 return self.table.get(host_id)
Gilad Arnold286a0062012-01-12 13:47:02 -0800106
107
Chris Sosa8dd80092012-12-10 13:39:11 -0800108class UpdateMetadata(object):
109 """Object containing metadata about an update payload."""
110
111 def __init__(self, sha1, sha256, size, is_delta_format):
112 self.sha1 = sha1
113 self.sha256 = sha256
114 self.size = size
115 self.is_delta_format = is_delta_format
116
117
rtc@google.com64244662009-11-12 00:52:08 +0000118class Autoupdate(BuildObject):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700119 """Class that contains functionality that handles Chrome OS update pings.
120
121 Members:
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700122 serve_only: serve only pre-built updates. static_dir must contain
123 update.gz and stateful.tgz.
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700124 use_test_image: use chromiumos_test_image.bin rather than the standard.
125 urlbase: base URL, other than devserver, for update images.
126 forced_image: path to an image to use for all updates.
127 payload_path: path to pre-generated payload to serve.
128 src_image: if specified, creates a delta payload from this image.
129 proxy_port: port of local proxy to tell client to connect to you
130 through.
131 vm: set for VM images (doesn't patch kernel)
132 board: board for the image. Needed for pre-generating of updates.
133 copy_to_static_root: copies images generated from the cache to ~/static.
134 private_key: path to private key in PEM format.
Gilad Arnold8318eac2012-10-04 12:52:23 -0700135 critical_update: whether provisioned payload is critical.
136 remote_payload: whether provisioned payload is remotely staged.
137 max_updates: maximum number of updates we'll try to provision.
138 host_log: record full history of host update events.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700139 """
rtc@google.comded22402009-10-26 22:36:21 +0000140
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700141 _PAYLOAD_URL_PREFIX = '/static/'
142 _FILEINFO_URL_PREFIX = '/api/fileinfo/'
143
Chris Sosa8dd80092012-12-10 13:39:11 -0800144 SHA1_ATTR = 'sha1'
145 SHA256_ATTR = 'sha256'
146 SIZE_ATTR = 'size'
147 ISDELTA_ATTR = 'is_delta'
148
Sean O'Connor1f7fd362010-04-07 16:34:52 -0700149 def __init__(self, serve_only=None, test_image=False, urlbase=None,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700150 forced_image=None, payload_path=None,
151 proxy_port=None, src_image='', vm=False, board=None,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800152 copy_to_static_root=True, private_key=None,
Chris Sosa52148582012-11-15 15:35:58 -0800153 critical_update=False, remote_payload=False, max_updates= -1,
Chris Sosa8dd80092012-12-10 13:39:11 -0800154 host_log=False, *args, **kwargs):
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700155 super(Autoupdate, self).__init__(*args, **kwargs)
Sean O'Connor1f7fd362010-04-07 16:34:52 -0700156 self.serve_only = serve_only
Chris Sosa0356d3b2010-09-16 15:46:22 -0700157 self.use_test_image = test_image
Chris Sosa5d342a22010-09-28 16:54:41 -0700158 if urlbase:
Chris Sosa9841e1c2010-10-14 10:51:45 -0700159 self.urlbase = urlbase
Chris Sosa5d342a22010-09-28 16:54:41 -0700160 else:
Chris Sosa9841e1c2010-10-14 10:51:45 -0700161 self.urlbase = None
Chris Sosa5d342a22010-09-28 16:54:41 -0700162
Chris Sosa0356d3b2010-09-16 15:46:22 -0700163 self.forced_image = forced_image
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700164 self.payload_path = payload_path
Chris Sosa62f720b2010-10-26 21:39:48 -0700165 self.src_image = src_image
Don Garrett0ad09372010-12-06 16:20:30 -0800166 self.proxy_port = proxy_port
Chris Sosa4136e692010-10-28 23:42:37 -0700167 self.vm = vm
Chris Sosae67b78f2010-11-04 17:33:16 -0700168 self.board = board
Chris Sosa08d55a22011-01-19 16:08:02 -0800169 self.copy_to_static_root = copy_to_static_root
Chris Sosa0f1ec842011-02-14 16:33:22 -0800170 self.private_key = private_key
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800171 self.critical_update = critical_update
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700172 self.remote_payload = remote_payload
Jay Srinivasanac69d262012-10-30 19:05:53 -0700173 self.max_updates = max_updates
Gilad Arnold8318eac2012-10-04 12:52:23 -0700174 self.host_log = host_log
Don Garrettfff4c322010-11-19 13:37:12 -0800175
Chris Sosa417e55d2011-01-25 16:40:48 -0800176 # Path to pre-generated file.
177 self.pregenerated_path = None
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700178
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700179 # Initialize empty host info cache. Used to keep track of various bits of
Gilad Arnold286a0062012-01-12 13:47:02 -0800180 # information about a given host. A host is identified by its IP address.
181 # The info stored for each host includes a complete log of events for this
182 # host, as well as a dictionary of current attributes derived from events.
183 self.host_infos = HostInfoTable()
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700184
Chris Sosa8dd80092012-12-10 13:39:11 -0800185 @classmethod
186 def _ReadMetadataFromStream(cls, stream):
187 """Returns metadata obj from input json stream that implements .read()."""
188 file_attr_dict = {}
189 try:
190 file_attr_dict = json.loads(stream.read())
191 except IOError:
192 return None
193
194 sha1 = file_attr_dict.get(cls.SHA1_ATTR)
195 sha256 = file_attr_dict.get(cls.SHA256_ATTR)
196 size = file_attr_dict.get(cls.SIZE_ATTR)
197 is_delta = file_attr_dict.get(cls.ISDELTA_ATTR)
198 return UpdateMetadata(sha1, sha256, size, is_delta)
199
200 @staticmethod
201 def _ReadMetadataFromFile(payload_dir):
202 """Returns metadata object from the metadata_file in the payload_dir"""
203 metadata_file = os.path.join(payload_dir, METADATA_FILE)
204 if os.path.exists(metadata_file):
205 with open(metadata_file, 'r') as metadata_stream:
206 return Autoupdate._ReadMetadataFromStream(metadata_stream)
207
208 @classmethod
209 def _StoreMetadataToFile(cls, payload_dir, metadata_obj):
210 """Stores metadata object into the metadata_file of the payload_dir"""
211 file_dict = {cls.SHA1_ATTR: metadata_obj.sha1,
212 cls.SHA256_ATTR: metadata_obj.sha256,
213 cls.SIZE_ATTR: metadata_obj.size,
214 cls.ISDELTA_ATTR: metadata_obj.is_delta_format}
215 metadata_file = os.path.join(payload_dir, METADATA_FILE)
216 with open(metadata_file, 'w') as file_handle:
217 json.dump(file_dict, file_handle)
218
Chris Sosa0356d3b2010-09-16 15:46:22 -0700219 def _GetDefaultBoardID(self):
220 """Returns the default board id stored in .default_board."""
221 board_file = '%s/.default_board' % (self.scripts_dir)
222 try:
223 return open(board_file).read()
224 except IOError:
225 return 'x86-generic'
226
Chris Sosa8dd80092012-12-10 13:39:11 -0800227 def _GetLatestImageDir(self, board):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700228 """Returns the latest image dir based on shell script."""
Chris Sosa8dd80092012-12-10 13:39:11 -0800229 cmd = '%s/get_latest_image.sh --board %s' % (self.scripts_dir, board)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700230 return os.popen(cmd).read().strip()
231
Chris Sosa52148582012-11-15 15:35:58 -0800232 @staticmethod
233 def _GetVersionFromDir(image_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700234 """Returns the version of the image based on the name of the directory."""
235 latest_version = os.path.basename(image_dir)
Daniel Erat8a0bc4a2011-09-30 08:52:52 -0700236 parts = latest_version.split('-')
237 if len(parts) == 2:
238 # Old-style, e.g. "0.15.938.2011_08_23_0941-a1".
239 # TODO(derat): Remove the code for old-style versions after 20120101.
240 return parts[0]
241 else:
242 # New-style, e.g. "R16-1102.0.2011_09_30_0806-a1".
243 return parts[1]
Chris Sosa0356d3b2010-09-16 15:46:22 -0700244
Chris Sosa52148582012-11-15 15:35:58 -0800245 @staticmethod
246 def _CanUpdate(client_version, latest_version):
Don Garrettf90edf02010-11-16 17:36:14 -0800247 """Returns true if the latest_version is greater than the client_version.
248 """
Chris Sosa8dd80092012-12-10 13:39:11 -0800249 _Log('client version %s latest version %s', client_version, latest_version)
Daniel Erat8a0bc4a2011-09-30 08:52:52 -0700250
251 client_tokens = client_version.replace('_', '').split('.')
252 # If the client has an old four-token version like "0.16.892.0", drop the
253 # first two tokens -- we use versions like "892.0.0" now.
254 # TODO(derat): Remove the code for old-style versions after 20120101.
255 if len(client_tokens) == 4:
256 client_tokens = client_tokens[2:]
257
258 latest_tokens = latest_version.replace('_', '').split('.')
259 if len(latest_tokens) == 4:
260 latest_tokens = latest_tokens[2:]
261
262 for i in range(min(len(client_tokens), len(latest_tokens))):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700263 if int(latest_tokens[i]) == int(client_tokens[i]):
264 continue
265 return int(latest_tokens[i]) > int(client_tokens[i])
Daniel Erat8a0bc4a2011-09-30 08:52:52 -0700266
267 # Favor four-token new-style versions on the server over old-style versions
268 # on the client if everything else matches.
269 return len(latest_tokens) > len(client_tokens)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700270
Chris Sosa0356d3b2010-09-16 15:46:22 -0700271 def _GetImageName(self):
272 """Returns the name of the image that should be used."""
273 if self.use_test_image:
274 image_name = 'chromiumos_test_image.bin'
275 else:
276 image_name = 'chromiumos_image.bin'
Chris Sosa8dd80092012-12-10 13:39:11 -0800277
Chris Sosa0356d3b2010-09-16 15:46:22 -0700278 return image_name
279
Chris Sosa52148582012-11-15 15:35:58 -0800280 @staticmethod
281 def _IsDeltaFormatFile(filename):
Andrew de los Reyes5679b972010-10-25 17:34:49 -0700282 try:
283 file_handle = open(filename, 'r')
284 delta_magic = 'CrAU'
285 magic = file_handle.read(len(delta_magic))
286 return magic == delta_magic
Jay Srinivasanac69d262012-10-30 19:05:53 -0700287 except IOError:
288 # For unit tests, we may not have real files, so it's ok to
289 # ignore these IOErrors. In any case, this value is not being
290 # used in update_engine at all as of now.
Andrew de los Reyes5679b972010-10-25 17:34:49 -0700291 return False
292
Don Garrettf90edf02010-11-16 17:36:14 -0800293 def GenerateUpdateFile(self, src_image, image_path, output_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700294 """Generates an update gz given a full path to an image.
295
296 Args:
297 image_path: Full path to image.
Chris Sosa8dd80092012-12-10 13:39:11 -0800298 Raises:
299 subprocess.CalledProcessError if the update generator fails to generate a
300 stateful payload.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700301 """
Don Garrettfff4c322010-11-19 13:37:12 -0800302 update_path = os.path.join(output_dir, UPDATE_FILE)
Chris Sosa8dd80092012-12-10 13:39:11 -0800303 _Log('Generating update image %s', update_path)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700304
Chris Sosa0f1ec842011-02-14 16:33:22 -0800305 update_command = [
Chris Sosa5b8b5eb2012-03-27 11:15:27 -0700306 'cros_generate_update_payload',
Chris Sosa8dd80092012-12-10 13:39:11 -0800307 '--image', image_path,
308 '--output', update_path,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800309 ]
Chris Sosa4136e692010-10-28 23:42:37 -0700310
Chris Sosa52148582012-11-15 15:35:58 -0800311 if src_image:
Chris Sosa8dd80092012-12-10 13:39:11 -0800312 update_command.extend(['--src_image', src_image])
Chris Sosa52148582012-11-15 15:35:58 -0800313
314 if not self.vm:
315 update_command.append('--patch_kernel')
316
317 if self.private_key:
Chris Sosa8dd80092012-12-10 13:39:11 -0800318 update_command.extend(['--private_key', self.private_key])
Chris Sosa0f1ec842011-02-14 16:33:22 -0800319
Chris Sosa8dd80092012-12-10 13:39:11 -0800320 _Log('Running %s', ' '.join(update_command))
321 subprocess.check_call(update_command)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700322
Chris Sosa52148582012-11-15 15:35:58 -0800323 @staticmethod
324 def GenerateStatefulFile(image_path, output_dir):
Don Garrettf90edf02010-11-16 17:36:14 -0800325 """Generates a stateful update payload given a full path to an image.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700326
327 Args:
328 image_path: Full path to image.
Chris Sosa908fd6f2010-11-10 17:31:18 -0800329 Raises:
Chris Sosa8dd80092012-12-10 13:39:11 -0800330 subprocess.CalledProcessError if the update generator fails to generate a
Chris Sosa908fd6f2010-11-10 17:31:18 -0800331 stateful payload.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700332 """
Chris Sosa8dd80092012-12-10 13:39:11 -0800333 update_command = [
334 'cros_generate_stateful_update_payload',
335 '--image', image_path,
336 '--output_dir', output_dir,
337 ]
338 _Log('Running %s', ' '.join(update_command))
339 subprocess.check_call(update_command)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700340
Don Garrettf90edf02010-11-16 17:36:14 -0800341 def FindCachedUpdateImageSubDir(self, src_image, dest_image):
342 """Find directory to store a cached update.
343
Gilad Arnold55a2a372012-10-02 09:46:32 -0700344 Given one, or two images for an update, this finds which cache directory
345 should hold the update files, even if they don't exist yet.
Don Garrettf90edf02010-11-16 17:36:14 -0800346
Gilad Arnold55a2a372012-10-02 09:46:32 -0700347 Returns:
348 A directory path for storing a cached update, of the following form:
349 Non-delta updates:
350 CACHE_DIR/<dest_hash>
351 Delta updates:
352 CACHE_DIR/<src_hash>_<dest_hash>
353 Signed updates (self.private_key):
354 CACHE_DIR/<src_hash>_<dest_hash>+<private_key_hash>
Chris Sosa744e1472011-09-07 19:32:50 -0700355 """
Gilad Arnold55a2a372012-10-02 09:46:32 -0700356 update_dir = ''
Chris Sosa744e1472011-09-07 19:32:50 -0700357 if src_image:
Gilad Arnold55a2a372012-10-02 09:46:32 -0700358 update_dir += common_util.GetFileMd5(src_image) + '_'
Don Garrettf90edf02010-11-16 17:36:14 -0800359
Gilad Arnold55a2a372012-10-02 09:46:32 -0700360 update_dir += common_util.GetFileMd5(dest_image)
Chris Sosa744e1472011-09-07 19:32:50 -0700361 if self.private_key:
Gilad Arnold55a2a372012-10-02 09:46:32 -0700362 update_dir += '+' + common_util.GetFileMd5(self.private_key)
Chris Sosa744e1472011-09-07 19:32:50 -0700363
Chris Sosa9fba7562012-01-31 10:15:47 -0800364 if not self.vm:
Gilad Arnold55a2a372012-10-02 09:46:32 -0700365 update_dir += '+patched_kernel'
Chris Sosa9fba7562012-01-31 10:15:47 -0800366
Gilad Arnold55a2a372012-10-02 09:46:32 -0700367 return os.path.join(CACHE_DIR, update_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800368
Don Garrettfff4c322010-11-19 13:37:12 -0800369 def GenerateUpdateImage(self, image_path, output_dir):
Don Garrettf90edf02010-11-16 17:36:14 -0800370 """Force generates an update payload based on the given image_path.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700371
Chris Sosade91f672010-11-16 10:05:44 -0800372 Args:
Don Garrettf90edf02010-11-16 17:36:14 -0800373 src_image: image we are updating from (Null/empty for non-delta)
374 image_path: full path to the image.
Chris Sosa8dd80092012-12-10 13:39:11 -0800375 output_dir: the directory to write the update payloads to
376 Raises:
377 AutoupdateError if it failed to generate either update or stateful
378 payload.
Chris Sosade91f672010-11-16 10:05:44 -0800379 """
Chris Sosa8dd80092012-12-10 13:39:11 -0800380 _Log('Generating update for image %s', image_path)
Andrew de los Reyes9a528712010-06-30 10:29:43 -0700381
Chris Sosa8dd80092012-12-10 13:39:11 -0800382 # Delete any previous state in this directory.
383 os.system('rm -rf "%s"' % output_dir)
384 os.makedirs(output_dir)
rtc@google.comded22402009-10-26 22:36:21 +0000385
Chris Sosa8dd80092012-12-10 13:39:11 -0800386 try:
387 self.GenerateUpdateFile(self.src_image, image_path, output_dir)
388 self.GenerateStatefulFile(image_path, output_dir)
389 except subprocess.CalledProcessError:
390 os.system('rm -rf "%s"' % output_dir)
391 raise AutoupdateError('Failed to generate update in %s' % output_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800392
393 def GenerateUpdateImageWithCache(self, image_path, static_image_dir):
394 """Force generates an update payload based on the given image_path.
rtc@google.comded22402009-10-26 22:36:21 +0000395
Chris Sosa0356d3b2010-09-16 15:46:22 -0700396 Args:
397 image_path: full path to the image.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700398 static_image_dir: the directory to move images to after generating.
399 Returns:
Chris Sosa8dd80092012-12-10 13:39:11 -0800400 update directory relative to static_image_dir. None if it should
401 serve from the static_image_dir.
402 Raises:
403 AutoupdateError if it we need to generate a payload and fail to do so.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700404 """
Chris Sosa8dd80092012-12-10 13:39:11 -0800405 _Log('Generating update for src %s image %s', self.src_image, image_path)
Chris Sosae67b78f2010-11-04 17:33:16 -0700406
Chris Sosa417e55d2011-01-25 16:40:48 -0800407 # If it was pregenerated_path, don't regenerate
408 if self.pregenerated_path:
409 return self.pregenerated_path
Don Garrettfff4c322010-11-19 13:37:12 -0800410
Don Garrettf90edf02010-11-16 17:36:14 -0800411 # Which sub_dir of static_image_dir should hold our cached update image
412 cache_sub_dir = self.FindCachedUpdateImageSubDir(self.src_image, image_path)
Chris Sosa8dd80092012-12-10 13:39:11 -0800413 _Log('Caching in sub_dir "%s"', cache_sub_dir)
Chris Sosa417e55d2011-01-25 16:40:48 -0800414
Don Garrettf90edf02010-11-16 17:36:14 -0800415 # The cached payloads exist in a cache dir
416 cache_update_payload = os.path.join(static_image_dir,
Chris Sosa8dd80092012-12-10 13:39:11 -0800417 cache_sub_dir, UPDATE_FILE)
Don Garrettf90edf02010-11-16 17:36:14 -0800418 cache_stateful_payload = os.path.join(static_image_dir,
Chris Sosa8dd80092012-12-10 13:39:11 -0800419 cache_sub_dir, STATEFUL_FILE)
Don Garrettf90edf02010-11-16 17:36:14 -0800420
Chris Sosa417e55d2011-01-25 16:40:48 -0800421 # Check to see if this cache directory is valid.
422 if not os.path.exists(cache_update_payload) or not os.path.exists(
423 cache_stateful_payload):
Don Garrettf90edf02010-11-16 17:36:14 -0800424 full_cache_dir = os.path.join(static_image_dir, cache_sub_dir)
Chris Sosa8dd80092012-12-10 13:39:11 -0800425 self.GenerateUpdateImage(image_path, full_cache_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800426
Chris Sosa8dd80092012-12-10 13:39:11 -0800427 self.pregenerated_path = cache_sub_dir
Don Garrettf90edf02010-11-16 17:36:14 -0800428
Chris Sosa08d55a22011-01-19 16:08:02 -0800429 # Generation complete, copy if requested.
430 if self.copy_to_static_root:
Chris Sosa417e55d2011-01-25 16:40:48 -0800431 # The final results exist directly in static
432 update_payload = os.path.join(static_image_dir,
433 UPDATE_FILE)
434 stateful_payload = os.path.join(static_image_dir,
435 STATEFUL_FILE)
Gilad Arnold55a2a372012-10-02 09:46:32 -0700436 common_util.CopyFile(cache_update_payload, update_payload)
437 common_util.CopyFile(cache_stateful_payload, stateful_payload)
Chris Sosa8dd80092012-12-10 13:39:11 -0800438 return None
Chris Sosa417e55d2011-01-25 16:40:48 -0800439 else:
440 return self.pregenerated_path
Chris Sosa0356d3b2010-09-16 15:46:22 -0700441
Chris Sosa8dd80092012-12-10 13:39:11 -0800442 def GenerateLatestUpdateImage(self, board, client_version,
Don Garrettf90edf02010-11-16 17:36:14 -0800443 static_image_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700444 """Generates an update using the latest image that has been built.
445
446 This will only generate an update if the newest update is newer than that
447 on the client or client_version is 'ForcedUpdate'.
448
449 Args:
Chris Sosa8dd80092012-12-10 13:39:11 -0800450 board: Name of the board.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700451 client_version: Current version of the client or 'ForcedUpdate'
452 static_image_dir: the directory to move images to after generating.
453 Returns:
Chris Sosa8dd80092012-12-10 13:39:11 -0800454 Name of the update directory relative to the static dir. None if it should
455 serve from the static_image_dir.
456 Raises:
457 AutoupdateError if it failed to generate the payload or can't update
458 the given client_version.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700459 """
Chris Sosa8dd80092012-12-10 13:39:11 -0800460 latest_image_dir = self._GetLatestImageDir(board)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700461 latest_version = self._GetVersionFromDir(latest_image_dir)
462 latest_image_path = os.path.join(latest_image_dir, self._GetImageName())
463
Chris Sosa0356d3b2010-09-16 15:46:22 -0700464 # Check to see whether or not we should update.
465 if client_version != 'ForcedUpdate' and not self._CanUpdate(
466 client_version, latest_version):
Chris Sosa8dd80092012-12-10 13:39:11 -0800467 raise AutoupdateError('Update check received but no update available '
468 'for client')
Chris Sosa0356d3b2010-09-16 15:46:22 -0700469
Don Garrettf90edf02010-11-16 17:36:14 -0800470 return self.GenerateUpdateImageWithCache(latest_image_path,
471 static_image_dir=static_image_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700472
Chris Sosa8dd80092012-12-10 13:39:11 -0800473 def GenerateUpdatePayload(self, board, client_version, static_image_dir):
474 """Generates an update for an image and returns the relative payload dir.
Chris Sosaa73ec162010-05-03 20:18:02 -0700475
Chris Sosa8dd80092012-12-10 13:39:11 -0800476 Returns:
477 payload dir relative to static_image_dir. None if it should
478 serve from the static_image_dir.
479 Raises:
480 AutoupdateError if it failed to generate the payload.
Don Garrettf90edf02010-11-16 17:36:14 -0800481 """
Dale Curtis723ec472010-11-30 14:06:47 -0800482 dest_path = os.path.join(static_image_dir, UPDATE_FILE)
483 dest_stateful = os.path.join(static_image_dir, STATEFUL_FILE)
484
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700485 if self.payload_path:
Don Garrett0c880e22010-11-17 18:13:37 -0800486 # If the forced payload is not already in our static_image_dir,
487 # copy it there.
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700488 src_path = os.path.abspath(self.payload_path)
Chris Sosa8dd80092012-12-10 13:39:11 -0800489 src_stateful = os.path.join(os.path.dirname(src_path), STATEFUL_FILE)
Don Garrettee25e552010-11-23 12:09:35 -0800490 # Only copy the files if the source directory is different from dest.
491 if os.path.dirname(src_path) != os.path.abspath(static_image_dir):
Gilad Arnold55a2a372012-10-02 09:46:32 -0700492 common_util.CopyFile(src_path, dest_path)
Don Garrettee25e552010-11-23 12:09:35 -0800493
494 # The stateful payload is optional.
495 if os.path.exists(src_stateful):
Gilad Arnold55a2a372012-10-02 09:46:32 -0700496 common_util.CopyFile(src_stateful, dest_stateful)
Don Garrettee25e552010-11-23 12:09:35 -0800497 else:
Chris Sosa8dd80092012-12-10 13:39:11 -0800498 _Log('WARN: %s not found. Expected for dev and test builds',
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700499 STATEFUL_FILE)
Don Garrettee25e552010-11-23 12:09:35 -0800500 if os.path.exists(dest_stateful):
501 os.remove(dest_stateful)
Don Garrett0c880e22010-11-17 18:13:37 -0800502
Chris Sosa8dd80092012-12-10 13:39:11 -0800503 # Serve from the main directory so rel_path is None.
504 return None
Don Garrett0c880e22010-11-17 18:13:37 -0800505 elif self.forced_image:
Don Garrettf90edf02010-11-16 17:36:14 -0800506 return self.GenerateUpdateImageWithCache(
507 self.forced_image,
508 static_image_dir=static_image_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800509 else:
Chris Sosa8dd80092012-12-10 13:39:11 -0800510 if not board:
511 raise AutoupdateError(
512 'Failed to generate update. '
513 'You must set --board when pre-generating latest update.')
Don Garrettf90edf02010-11-16 17:36:14 -0800514
Chris Sosa8dd80092012-12-10 13:39:11 -0800515 return self.GenerateLatestUpdateImage(board, client_version,
516 static_image_dir)
Chris Sosa2c048f12010-10-27 16:05:27 -0700517
518 def PreGenerateUpdate(self):
Chris Sosa417e55d2011-01-25 16:40:48 -0800519 """Pre-generates an update and prints out the relative path it.
520
Chris Sosa8dd80092012-12-10 13:39:11 -0800521 Returns relative path of the update.
Chris Sosa417e55d2011-01-25 16:40:48 -0800522
Chris Sosa8dd80092012-12-10 13:39:11 -0800523 Raises:
524 AutoupdateError if it failed to generate the payload.
525 """
526 _Log('Pre-generating the update payload')
527 # Does not work with labels so just use static dir.
528 pregenerated_update = self.GenerateUpdatePayload(self.board, '0.0.0.0',
529 self.static_dir)
530 print 'PREGENERATED_UPDATE=%s' % _NonePathJoin(pregenerated_update,
531 UPDATE_FILE)
Chris Sosa417e55d2011-01-25 16:40:48 -0800532 return pregenerated_update
Chris Sosa2c048f12010-10-27 16:05:27 -0700533
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700534 def _GetRemotePayloadAttrs(self, url):
535 """Returns hashes, size and delta flag of a remote update payload.
536
537 Obtain attributes of a payload file available on a remote devserver. This
538 is based on the assumption that the payload URL uses the /static prefix. We
539 need to make sure that both clients (requests) and remote devserver
540 (provisioning) preserve this invariant.
541
542 Args:
543 url: URL of statically staged remote file (http://host:port/static/...)
544 Returns:
545 A tuple containing the SHA1, SHA256, file size and whether or not it's a
546 delta payload (Boolean).
547 """
548 if self._PAYLOAD_URL_PREFIX not in url:
549 raise AutoupdateError(
550 'Payload URL does not have the expected prefix (%s)' %
551 self._PAYLOAD_URL_PREFIX)
Chris Sosa8dd80092012-12-10 13:39:11 -0800552
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700553 fileinfo_url = url.replace(self._PAYLOAD_URL_PREFIX,
554 self._FILEINFO_URL_PREFIX)
Chris Sosa8dd80092012-12-10 13:39:11 -0800555 _Log('Retrieving file info for remote payload via %s', fileinfo_url)
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700556 try:
557 conn = urllib2.urlopen(fileinfo_url)
Chris Sosa8dd80092012-12-10 13:39:11 -0800558 metadata_obj = Autoupdate._ReadMetadataFromStream(conn)
559 # These fields are required for remote calls.
560 if not metadata_obj:
561 raise AutoupdateError('Failed to obtain remote payload info')
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700562
Chris Sosa8dd80092012-12-10 13:39:11 -0800563 if not metadata_obj.is_delta_format:
564 metadata_obj.is_delta_format = ('_mton' in url) or ('_nton' in url)
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700565
Chris Sosa8dd80092012-12-10 13:39:11 -0800566 return metadata_obj
567 except IOError as e:
568 raise AutoupdateError('Failed to obtain remote payload info: %s', e)
569
570 def GetLocalPayloadAttrs(self, payload_dir):
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700571 """Returns hashes, size and delta flag of a local update payload.
572
573 Args:
Chris Sosa8dd80092012-12-10 13:39:11 -0800574 payload_dir: Path to the directory the payload is in.
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700575 Returns:
576 A tuple containing the SHA1, SHA256, file size and whether or not it's a
577 delta payload (Boolean).
578 """
Chris Sosa8dd80092012-12-10 13:39:11 -0800579 filename = os.path.join(payload_dir, UPDATE_FILE)
580 if not os.path.exists(filename):
581 raise AutoupdateError('update.gz not present in payload dir %s' %
582 payload_dir)
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700583
Chris Sosa8dd80092012-12-10 13:39:11 -0800584 metadata_obj = Autoupdate._ReadMetadataFromFile(payload_dir)
585 if not metadata_obj or not (metadata_obj.sha1 and
586 metadata_obj.sha256 and
587 metadata_obj.size):
588 sha1 = common_util.GetFileSha1(filename)
589 sha256 = common_util.GetFileSha256(filename)
590 size = common_util.GetFileSize(filename)
591 is_delta_format = self._IsDeltaFormatFile(filename)
592 metadata_obj = UpdateMetadata(sha1, sha256, size, is_delta_format)
593 Autoupdate._StoreMetadataToFile(payload_dir, metadata_obj)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700594
Chris Sosa8dd80092012-12-10 13:39:11 -0800595 return metadata_obj
596
597 def _ProcessUpdateComponents(self, app, event):
598 """Processes the app and event components of an update request.
599
600 Returns tuple containing forced_update_label, client_version, and board.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700601 """
Chris Sosa8dd80092012-12-10 13:39:11 -0800602 # Initialize an empty dictionary for event attributes to log.
603 log_message = {}
Jay Srinivasanac69d262012-10-30 19:05:53 -0700604
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700605 # Determine request IP, strip any IPv6 data for simplicity.
606 client_ip = cherrypy.request.remote.ip.split(':')[-1]
Gilad Arnold286a0062012-01-12 13:47:02 -0800607 # Obtain (or init) info object for this client.
608 curr_host_info = self.host_infos.GetInitHostInfo(client_ip)
609
Chris Sosa8dd80092012-12-10 13:39:11 -0800610 client_version = 'ForcedUpdate'
611 board = None
612 if app:
613 client_version = app.getAttribute('version')
614 channel = app.getAttribute('track')
615 board = (app.hasAttribute('board') and app.getAttribute('board')
616 or self._GetDefaultBoardID())
617 # Add attributes to log message
618 log_message['version'] = client_version
619 log_message['track'] = channel
620 log_message['board'] = board
621 curr_host_info.attrs['last_known_version'] = client_version
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700622
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700623 if event:
Gilad Arnold286a0062012-01-12 13:47:02 -0800624 event_result = int(event[0].getAttribute('eventresult'))
625 event_type = int(event[0].getAttribute('eventtype'))
Gilad Arnoldb11a8942012-03-13 15:33:21 -0700626 client_previous_version = (event[0].getAttribute('previousversion')
627 if event[0].hasAttribute('previousversion')
628 else None)
Gilad Arnold286a0062012-01-12 13:47:02 -0800629 # Store attributes to legacy host info structure
630 curr_host_info.attrs['last_event_status'] = event_result
631 curr_host_info.attrs['last_event_type'] = event_type
632 # Add attributes to log message
633 log_message['event_result'] = event_result
634 log_message['event_type'] = event_type
Gilad Arnoldb11a8942012-03-13 15:33:21 -0700635 if client_previous_version is not None:
636 log_message['previous_version'] = client_previous_version
Gilad Arnold286a0062012-01-12 13:47:02 -0800637
Gilad Arnold8318eac2012-10-04 12:52:23 -0700638 # Log host event, if so instructed.
639 if self.host_log:
640 curr_host_info.AddLogEntry(log_message)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700641
Chris Sosa8dd80092012-12-10 13:39:11 -0800642 return (curr_host_info.attrs.pop('forced_update_label', None),
643 client_version, board)
644
645 def _GetStaticUrl(self):
646 """Returns the static url base that should prefix all payload responses."""
647 x_forwarded_host = cherrypy.request.headers.get('X-Forwarded-Host')
648 if x_forwarded_host:
649 hostname = 'http://' + x_forwarded_host
650 else:
651 hostname = cherrypy.request.base
652
653 if self.urlbase:
654 static_urlbase = self.urlbase
655 elif self.serve_only:
656 static_urlbase = '%s/static/archive' % hostname
657 else:
658 static_urlbase = '%s/static' % hostname
659
660 # If we have a proxy port, adjust the URL we instruct the client to
661 # use to go through the proxy.
662 if self.proxy_port:
663 static_urlbase = _ChangeUrlPort(static_urlbase, self.proxy_port)
664
665 _Log('Using static url base %s', static_urlbase)
666 _Log('Handling update ping as %s', hostname)
667 return static_urlbase
668
669 def HandleUpdatePing(self, data, label=None):
670 """Handles an update ping from an update client.
671
672 Args:
673 data: XML blob from client.
674 label: optional label for the update.
675 Returns:
676 Update payload message for client.
677 """
678 # Get the static url base that will form that base of our update url e.g.
679 # http://hostname:8080/static/update.gz.
680 static_urlbase = self._GetStaticUrl()
681
682 # Parse the XML we got into the components we care about.
683 protocol, app, event, update_check = autoupdate_lib.ParseUpdateRequest(data)
684
685 # #########################################################################
686 # Process attributes of the update check.
687 forced_update_label, client_version, board = self._ProcessUpdateComponents(
688 app, event)
689
690 # We only process update_checks in the update rpc.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700691 if not update_check:
Chris Sosa8dd80092012-12-10 13:39:11 -0800692 _Log('Non-update check received. Returning blank payload')
Chris Sosa0356d3b2010-09-16 15:46:22 -0700693 # TODO(sosa): Generate correct non-updatecheck payload to better test
694 # update clients.
Chris Sosa52148582012-11-15 15:35:58 -0800695 return autoupdate_lib.GetNoUpdateResponse(protocol)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700696
Chris Sosa8dd80092012-12-10 13:39:11 -0800697 # In case max_updates is used, return no response if max reached.
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700698 if self.max_updates > 0:
699 self.max_updates -= 1
700 elif self.max_updates == 0:
Chris Sosa8dd80092012-12-10 13:39:11 -0800701 _Log('Request received but max number of updates handled')
Chris Sosa52148582012-11-15 15:35:58 -0800702 return autoupdate_lib.GetNoUpdateResponse(protocol)
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700703
Chris Sosa8dd80092012-12-10 13:39:11 -0800704 _Log('Update Check Received. Client is using protocol version: %s',
705 protocol)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700706
Chris Sosa8dd80092012-12-10 13:39:11 -0800707 if forced_update_label:
708 if label:
709 _Log('Label: %s set but being overwritten to %s by request', label,
710 forced_update_label)
711
712 label = forced_update_label
713
714 # #########################################################################
715 # Finally its time to generate the omaha response to give to client that
716 # lets them know where to find the payload and its associated metadata.
717 metadata_obj = None
718
719 try:
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700720 # Are we provisioning a remote or local payload?
721 if self.remote_payload:
722 # If no explicit label was provided, use the value of --payload.
Chris Sosa8dd80092012-12-10 13:39:11 -0800723 if not label:
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700724 label = self.payload_path
Chris Sosa0356d3b2010-09-16 15:46:22 -0700725
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700726 # Form the URL of the update payload. This assumes that the payload
727 # file name is a devserver constant (which currently is the case).
728 url = '/'.join(filter(None, [static_urlbase, label, UPDATE_FILE]))
Chris Sosa5d342a22010-09-28 16:54:41 -0700729
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700730 # Get remote payload attributes.
Chris Sosa8dd80092012-12-10 13:39:11 -0800731 metadata_obj = self._GetRemotePayloadAttrs(url)
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700732 else:
Chris Sosa8dd80092012-12-10 13:39:11 -0800733 static_image_dir = _NonePathJoin(self.static_dir, label)
734 rel_path = None
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700735
Chris Sosa8dd80092012-12-10 13:39:11 -0800736 # Serving files only, don't generate an update.
737 if not self.serve_only:
738 # Generate payload if necessary.
739 rel_path = self.GenerateUpdatePayload(board, client_version,
740 static_image_dir)
741
742 url = '/'.join(filter(None, [static_urlbase, label, rel_path,
743 UPDATE_FILE]))
744 local_payload_dir = _NonePathJoin(static_image_dir, rel_path)
745 metadata_obj = self.GetLocalPayloadAttrs(local_payload_dir)
746
747 except AutoupdateError as e:
748 # Raised if we fail to generate an update payload.
749 _Log('Failed to process an update: %r', e)
750 return autoupdate_lib.GetNoUpdateResponse(protocol)
751
752 _Log('Responding to client to use url %s to get image', url)
753 return autoupdate_lib.GetUpdateResponse(
754 metadata_obj.sha1, metadata_obj.sha256, metadata_obj.size, url,
755 metadata_obj.is_delta_format, protocol, self.critical_update)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700756
757 def HandleHostInfoPing(self, ip):
758 """Returns host info dictionary for the given IP in JSON format."""
759 assert ip, 'No ip provided.'
Gilad Arnold286a0062012-01-12 13:47:02 -0800760 if ip in self.host_infos.table:
761 return json.dumps(self.host_infos.GetHostInfo(ip).attrs)
762
763 def HandleHostLogPing(self, ip):
764 """Returns a complete log of events for host in JSON format."""
Gilad Arnold4ba437d2012-10-05 15:28:27 -0700765 # If all events requested, return a dictionary of logs keyed by IP address.
Gilad Arnold286a0062012-01-12 13:47:02 -0800766 if ip == 'all':
767 return json.dumps(
768 dict([(key, self.host_infos.table[key].log)
769 for key in self.host_infos.table]))
Gilad Arnold4ba437d2012-10-05 15:28:27 -0700770
771 # Otherwise we're looking for a specific IP address, so find its log.
Gilad Arnold286a0062012-01-12 13:47:02 -0800772 if ip in self.host_infos.table:
773 return json.dumps(self.host_infos.GetHostInfo(ip).log)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700774
Gilad Arnold4ba437d2012-10-05 15:28:27 -0700775 # If no events were logged for this IP, return an empty log.
776 return json.dumps([])
777
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700778 def HandleSetUpdatePing(self, ip, label):
779 """Sets forced_update_label for a given host."""
780 assert ip, 'No ip provided.'
781 assert label, 'No label provided.'
Gilad Arnold286a0062012-01-12 13:47:02 -0800782 self.host_infos.GetInitHostInfo(ip).attrs['forced_update_label'] = label