blob: 1f71ef473fb7fdb633d6807d1deb56fad623331b [file] [log] [blame]
Darin Petkovc3fd90c2011-05-11 14:23:00 -07001# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
rtc@google.comded22402009-10-26 22:36:21 +00002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Dale Curtisc9aaf3a2011-08-09 15:47:40 -07005import json
rtc@google.comded22402009-10-26 22:36:21 +00006import os
Chris Sosa05491b12010-11-08 17:14:16 -08007import subprocess
Darin Petkov2b2ff4b2010-07-27 15:02:09 -07008import time
Gilad Arnold0c9c8602012-10-02 23:58:58 -07009import urllib2
Don Garrett0ad09372010-12-06 16:20:30 -080010import urlparse
Chris Sosa7c931362010-10-11 19:49:01 -070011
Gilad Arnoldabb352e2012-09-23 01:24:27 -070012import cherrypy
13
14from build_util import BuildObject
Chris Sosa52148582012-11-15 15:35:58 -080015import autoupdate_lib
Gilad Arnold55a2a372012-10-02 09:46:32 -070016import common_util
Gilad Arnoldc65330c2012-09-20 15:17:48 -070017import log_util
Chris Sosa05491b12010-11-08 17:14:16 -080018
Gilad Arnoldc65330c2012-09-20 15:17:48 -070019
20# Module-local log function.
Chris Sosa6a3697f2013-01-29 16:44:43 -080021def _Log(message, *args):
22 return log_util.LogWithTag('UPDATE', message, *args)
Gilad Arnoldc65330c2012-09-20 15:17:48 -070023
rtc@google.comded22402009-10-26 22:36:21 +000024
Chris Sosa417e55d2011-01-25 16:40:48 -080025UPDATE_FILE = 'update.gz'
Chris Sosa6a3697f2013-01-29 16:44:43 -080026METADATA_FILE = 'update.meta'
Chris Sosa417e55d2011-01-25 16:40:48 -080027STATEFUL_FILE = 'stateful.tgz'
28CACHE_DIR = 'cache'
Chris Sosa0356d3b2010-09-16 15:46:22 -070029
Don Garrett0ad09372010-12-06 16:20:30 -080030
Gilad Arnold0c9c8602012-10-02 23:58:58 -070031class AutoupdateError(Exception):
32 """Exception classes used by this module."""
33 pass
34
35
Don Garrett0ad09372010-12-06 16:20:30 -080036def _ChangeUrlPort(url, new_port):
37 """Return the URL passed in with a different port"""
38 scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
39 host_port = netloc.split(':')
40
41 if len(host_port) == 1:
42 host_port.append(new_port)
43 else:
44 host_port[1] = new_port
45
46 print host_port
47 netloc = "%s:%s" % tuple(host_port)
48
49 return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
50
Chris Sosa6a3697f2013-01-29 16:44:43 -080051def _NonePathJoin(*args):
52 """os.path.join that filters None's from the argument list."""
53 return os.path.join(*filter(None, args))
Don Garrett0ad09372010-12-06 16:20:30 -080054
Chris Sosa6a3697f2013-01-29 16:44:43 -080055
56class HostInfo(object):
Gilad Arnold286a0062012-01-12 13:47:02 -080057 """Records information about an individual host.
58
59 Members:
60 attrs: Static attributes (legacy)
61 log: Complete log of recorded client entries
62 """
63
64 def __init__(self):
65 # A dictionary of current attributes pertaining to the host.
66 self.attrs = {}
67
68 # A list of pairs consisting of a timestamp and a dictionary of recorded
69 # attributes.
70 self.log = []
71
72 def __repr__(self):
73 return 'attrs=%s, log=%s' % (self.attrs, self.log)
74
75 def AddLogEntry(self, entry):
76 """Append a new log entry."""
77 # Append a timestamp.
78 assert not 'timestamp' in entry, 'Oops, timestamp field already in use'
79 entry['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S')
80 # Add entry to hosts' message log.
81 self.log.append(entry)
82
Gilad Arnold286a0062012-01-12 13:47:02 -080083
Chris Sosa6a3697f2013-01-29 16:44:43 -080084class HostInfoTable(object):
Gilad Arnold286a0062012-01-12 13:47:02 -080085 """Records information about a set of hosts who engage in update activity.
86
87 Members:
88 table: Table of information on hosts.
89 """
90
91 def __init__(self):
92 # A dictionary of host information. Keys are normally IP addresses.
93 self.table = {}
94
95 def __repr__(self):
96 return '%s' % self.table
97
98 def GetInitHostInfo(self, host_id):
99 """Return a host's info object, or create a new one if none exists."""
100 return self.table.setdefault(host_id, HostInfo())
101
102 def GetHostInfo(self, host_id):
103 """Return an info object for given host, if such exists."""
Chris Sosa1885d032012-11-29 17:07:27 -0800104 return self.table.get(host_id)
Gilad Arnold286a0062012-01-12 13:47:02 -0800105
106
Chris Sosa6a3697f2013-01-29 16:44:43 -0800107class UpdateMetadata(object):
108 """Object containing metadata about an update payload."""
109
110 def __init__(self, sha1, sha256, size, is_delta_format):
111 self.sha1 = sha1
112 self.sha256 = sha256
113 self.size = size
114 self.is_delta_format = is_delta_format
115
116
rtc@google.com64244662009-11-12 00:52:08 +0000117class Autoupdate(BuildObject):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700118 """Class that contains functionality that handles Chrome OS update pings.
119
120 Members:
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700121 serve_only: serve only pre-built updates. static_dir must contain
122 update.gz and stateful.tgz.
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700123 use_test_image: use chromiumos_test_image.bin rather than the standard.
124 urlbase: base URL, other than devserver, for update images.
125 forced_image: path to an image to use for all updates.
126 payload_path: path to pre-generated payload to serve.
127 src_image: if specified, creates a delta payload from this image.
128 proxy_port: port of local proxy to tell client to connect to you
129 through.
130 vm: set for VM images (doesn't patch kernel)
131 board: board for the image. Needed for pre-generating of updates.
132 copy_to_static_root: copies images generated from the cache to ~/static.
133 private_key: path to private key in PEM format.
Gilad Arnold8318eac2012-10-04 12:52:23 -0700134 critical_update: whether provisioned payload is critical.
135 remote_payload: whether provisioned payload is remotely staged.
136 max_updates: maximum number of updates we'll try to provision.
137 host_log: record full history of host update events.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700138 """
rtc@google.comded22402009-10-26 22:36:21 +0000139
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700140 _PAYLOAD_URL_PREFIX = '/static/'
141 _FILEINFO_URL_PREFIX = '/api/fileinfo/'
142
Chris Sosa6a3697f2013-01-29 16:44:43 -0800143 SHA1_ATTR = 'sha1'
144 SHA256_ATTR = 'sha256'
145 SIZE_ATTR = 'size'
146 ISDELTA_ATTR = 'is_delta'
147
Sean O'Connor1f7fd362010-04-07 16:34:52 -0700148 def __init__(self, serve_only=None, test_image=False, urlbase=None,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700149 forced_image=None, payload_path=None,
150 proxy_port=None, src_image='', vm=False, board=None,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800151 copy_to_static_root=True, private_key=None,
Chris Sosa52148582012-11-15 15:35:58 -0800152 critical_update=False, remote_payload=False, max_updates= -1,
Chris Sosa6a3697f2013-01-29 16:44:43 -0800153 host_log=False, *args, **kwargs):
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700154 super(Autoupdate, self).__init__(*args, **kwargs)
Sean O'Connor1f7fd362010-04-07 16:34:52 -0700155 self.serve_only = serve_only
Chris Sosa0356d3b2010-09-16 15:46:22 -0700156 self.use_test_image = test_image
Chris Sosa5d342a22010-09-28 16:54:41 -0700157 if urlbase:
Chris Sosa9841e1c2010-10-14 10:51:45 -0700158 self.urlbase = urlbase
Chris Sosa5d342a22010-09-28 16:54:41 -0700159 else:
Chris Sosa9841e1c2010-10-14 10:51:45 -0700160 self.urlbase = None
Chris Sosa5d342a22010-09-28 16:54:41 -0700161
Chris Sosa0356d3b2010-09-16 15:46:22 -0700162 self.forced_image = forced_image
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700163 self.payload_path = payload_path
Chris Sosa62f720b2010-10-26 21:39:48 -0700164 self.src_image = src_image
Don Garrett0ad09372010-12-06 16:20:30 -0800165 self.proxy_port = proxy_port
Chris Sosa4136e692010-10-28 23:42:37 -0700166 self.vm = vm
Chris Sosae67b78f2010-11-04 17:33:16 -0700167 self.board = board
Chris Sosa08d55a22011-01-19 16:08:02 -0800168 self.copy_to_static_root = copy_to_static_root
Chris Sosa0f1ec842011-02-14 16:33:22 -0800169 self.private_key = private_key
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800170 self.critical_update = critical_update
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700171 self.remote_payload = remote_payload
Jay Srinivasanac69d262012-10-30 19:05:53 -0700172 self.max_updates = max_updates
Gilad Arnold8318eac2012-10-04 12:52:23 -0700173 self.host_log = host_log
Don Garrettfff4c322010-11-19 13:37:12 -0800174
Chris Sosa417e55d2011-01-25 16:40:48 -0800175 # Path to pre-generated file.
176 self.pregenerated_path = None
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700177
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700178 # Initialize empty host info cache. Used to keep track of various bits of
Gilad Arnold286a0062012-01-12 13:47:02 -0800179 # information about a given host. A host is identified by its IP address.
180 # The info stored for each host includes a complete log of events for this
181 # host, as well as a dictionary of current attributes derived from events.
182 self.host_infos = HostInfoTable()
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700183
Chris Sosa6a3697f2013-01-29 16:44:43 -0800184 @classmethod
185 def _ReadMetadataFromStream(cls, stream):
186 """Returns metadata obj from input json stream that implements .read()."""
187 file_attr_dict = {}
188 try:
189 file_attr_dict = json.loads(stream.read())
190 except IOError:
191 return None
192
193 sha1 = file_attr_dict.get(cls.SHA1_ATTR)
194 sha256 = file_attr_dict.get(cls.SHA256_ATTR)
195 size = file_attr_dict.get(cls.SIZE_ATTR)
196 is_delta = file_attr_dict.get(cls.ISDELTA_ATTR)
197 return UpdateMetadata(sha1, sha256, size, is_delta)
198
199 @staticmethod
200 def _ReadMetadataFromFile(payload_dir):
201 """Returns metadata object from the metadata_file in the payload_dir"""
202 metadata_file = os.path.join(payload_dir, METADATA_FILE)
203 if os.path.exists(metadata_file):
204 with open(metadata_file, 'r') as metadata_stream:
205 return Autoupdate._ReadMetadataFromStream(metadata_stream)
206
207 @classmethod
208 def _StoreMetadataToFile(cls, payload_dir, metadata_obj):
209 """Stores metadata object into the metadata_file of the payload_dir"""
210 file_dict = {cls.SHA1_ATTR: metadata_obj.sha1,
211 cls.SHA256_ATTR: metadata_obj.sha256,
212 cls.SIZE_ATTR: metadata_obj.size,
213 cls.ISDELTA_ATTR: metadata_obj.is_delta_format}
214 metadata_file = os.path.join(payload_dir, METADATA_FILE)
215 with open(metadata_file, 'w') as file_handle:
216 json.dump(file_dict, file_handle)
217
Chris Sosa0356d3b2010-09-16 15:46:22 -0700218 def _GetDefaultBoardID(self):
219 """Returns the default board id stored in .default_board."""
220 board_file = '%s/.default_board' % (self.scripts_dir)
221 try:
222 return open(board_file).read()
223 except IOError:
224 return 'x86-generic'
225
Chris Sosa6a3697f2013-01-29 16:44:43 -0800226 def _GetLatestImageDir(self, board):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700227 """Returns the latest image dir based on shell script."""
Chris Sosa6a3697f2013-01-29 16:44:43 -0800228 cmd = '%s/get_latest_image.sh --board %s' % (self.scripts_dir, board)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700229 return os.popen(cmd).read().strip()
230
Chris Sosa52148582012-11-15 15:35:58 -0800231 @staticmethod
232 def _GetVersionFromDir(image_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700233 """Returns the version of the image based on the name of the directory."""
234 latest_version = os.path.basename(image_dir)
Daniel Erat8a0bc4a2011-09-30 08:52:52 -0700235 parts = latest_version.split('-')
236 if len(parts) == 2:
237 # Old-style, e.g. "0.15.938.2011_08_23_0941-a1".
238 # TODO(derat): Remove the code for old-style versions after 20120101.
239 return parts[0]
240 else:
241 # New-style, e.g. "R16-1102.0.2011_09_30_0806-a1".
242 return parts[1]
Chris Sosa0356d3b2010-09-16 15:46:22 -0700243
Chris Sosa52148582012-11-15 15:35:58 -0800244 @staticmethod
245 def _CanUpdate(client_version, latest_version):
Don Garrettf90edf02010-11-16 17:36:14 -0800246 """Returns true if the latest_version is greater than the client_version.
247 """
Chris Sosa6a3697f2013-01-29 16:44:43 -0800248 _Log('client version %s latest version %s', client_version, latest_version)
Daniel Erat8a0bc4a2011-09-30 08:52:52 -0700249
250 client_tokens = client_version.replace('_', '').split('.')
251 # If the client has an old four-token version like "0.16.892.0", drop the
252 # first two tokens -- we use versions like "892.0.0" now.
253 # TODO(derat): Remove the code for old-style versions after 20120101.
254 if len(client_tokens) == 4:
255 client_tokens = client_tokens[2:]
256
257 latest_tokens = latest_version.replace('_', '').split('.')
258 if len(latest_tokens) == 4:
259 latest_tokens = latest_tokens[2:]
260
261 for i in range(min(len(client_tokens), len(latest_tokens))):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700262 if int(latest_tokens[i]) == int(client_tokens[i]):
263 continue
264 return int(latest_tokens[i]) > int(client_tokens[i])
Daniel Erat8a0bc4a2011-09-30 08:52:52 -0700265
266 # Favor four-token new-style versions on the server over old-style versions
267 # on the client if everything else matches.
268 return len(latest_tokens) > len(client_tokens)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700269
Chris Sosa0356d3b2010-09-16 15:46:22 -0700270 def _GetImageName(self):
271 """Returns the name of the image that should be used."""
272 if self.use_test_image:
273 image_name = 'chromiumos_test_image.bin'
274 else:
275 image_name = 'chromiumos_image.bin'
Chris Sosa6a3697f2013-01-29 16:44:43 -0800276
Chris Sosa0356d3b2010-09-16 15:46:22 -0700277 return image_name
278
Chris Sosa52148582012-11-15 15:35:58 -0800279 @staticmethod
Simran Basi40836b22013-03-28 15:08:17 -0700280 def _IsDeltaFormatFile(filename):
Andrew de los Reyes5679b972010-10-25 17:34:49 -0700281 try:
Simran Basi40836b22013-03-28 15:08:17 -0700282 file_handle = open(filename, 'r')
283 delta_magic = 'CrAU'
284 magic = file_handle.read(len(delta_magic))
285 return magic == delta_magic
286 except IOError:
287 # For unit tests, we may not have real files, so it's ok to
288 # ignore these IOErrors. In any case, this value is not being
289 # used in update_engine at all as of now.
Andrew de los Reyes5679b972010-10-25 17:34:49 -0700290 return False
291
Don Garrettf90edf02010-11-16 17:36:14 -0800292 def GenerateUpdateFile(self, src_image, image_path, output_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700293 """Generates an update gz given a full path to an image.
294
295 Args:
296 image_path: Full path to image.
Chris Sosa6a3697f2013-01-29 16:44:43 -0800297 Raises:
298 subprocess.CalledProcessError if the update generator fails to generate a
299 stateful payload.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700300 """
Don Garrettfff4c322010-11-19 13:37:12 -0800301 update_path = os.path.join(output_dir, UPDATE_FILE)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800302 _Log('Generating update image %s', update_path)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700303
Chris Sosa0f1ec842011-02-14 16:33:22 -0800304 update_command = [
Chris Sosa5b8b5eb2012-03-27 11:15:27 -0700305 'cros_generate_update_payload',
Chris Sosa6a3697f2013-01-29 16:44:43 -0800306 '--image', image_path,
307 '--output', update_path,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800308 ]
Chris Sosa4136e692010-10-28 23:42:37 -0700309
Chris Sosa52148582012-11-15 15:35:58 -0800310 if src_image:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800311 update_command.extend(['--src_image', src_image])
Chris Sosa52148582012-11-15 15:35:58 -0800312
313 if not self.vm:
314 update_command.append('--patch_kernel')
315
316 if self.private_key:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800317 update_command.extend(['--private_key', self.private_key])
Chris Sosa0f1ec842011-02-14 16:33:22 -0800318
Chris Sosa6a3697f2013-01-29 16:44:43 -0800319 _Log('Running %s', ' '.join(update_command))
320 subprocess.check_call(update_command)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700321
Chris Sosa52148582012-11-15 15:35:58 -0800322 @staticmethod
323 def GenerateStatefulFile(image_path, output_dir):
Don Garrettf90edf02010-11-16 17:36:14 -0800324 """Generates a stateful update payload given a full path to an image.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700325
326 Args:
327 image_path: Full path to image.
Chris Sosa908fd6f2010-11-10 17:31:18 -0800328 Raises:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800329 subprocess.CalledProcessError if the update generator fails to generate a
Chris Sosa908fd6f2010-11-10 17:31:18 -0800330 stateful payload.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700331 """
Chris Sosa6a3697f2013-01-29 16:44:43 -0800332 update_command = [
333 'cros_generate_stateful_update_payload',
334 '--image', image_path,
335 '--output_dir', output_dir,
336 ]
337 _Log('Running %s', ' '.join(update_command))
338 subprocess.check_call(update_command)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700339
Don Garrettf90edf02010-11-16 17:36:14 -0800340 def FindCachedUpdateImageSubDir(self, src_image, dest_image):
341 """Find directory to store a cached update.
342
Gilad Arnold55a2a372012-10-02 09:46:32 -0700343 Given one, or two images for an update, this finds which cache directory
344 should hold the update files, even if they don't exist yet.
Don Garrettf90edf02010-11-16 17:36:14 -0800345
Gilad Arnold55a2a372012-10-02 09:46:32 -0700346 Returns:
347 A directory path for storing a cached update, of the following form:
348 Non-delta updates:
349 CACHE_DIR/<dest_hash>
350 Delta updates:
351 CACHE_DIR/<src_hash>_<dest_hash>
352 Signed updates (self.private_key):
353 CACHE_DIR/<src_hash>_<dest_hash>+<private_key_hash>
Chris Sosa744e1472011-09-07 19:32:50 -0700354 """
Gilad Arnold55a2a372012-10-02 09:46:32 -0700355 update_dir = ''
Chris Sosa744e1472011-09-07 19:32:50 -0700356 if src_image:
Gilad Arnold55a2a372012-10-02 09:46:32 -0700357 update_dir += common_util.GetFileMd5(src_image) + '_'
Don Garrettf90edf02010-11-16 17:36:14 -0800358
Gilad Arnold55a2a372012-10-02 09:46:32 -0700359 update_dir += common_util.GetFileMd5(dest_image)
Chris Sosa744e1472011-09-07 19:32:50 -0700360 if self.private_key:
Gilad Arnold55a2a372012-10-02 09:46:32 -0700361 update_dir += '+' + common_util.GetFileMd5(self.private_key)
Chris Sosa744e1472011-09-07 19:32:50 -0700362
Chris Sosa9fba7562012-01-31 10:15:47 -0800363 if not self.vm:
Gilad Arnold55a2a372012-10-02 09:46:32 -0700364 update_dir += '+patched_kernel'
Chris Sosa9fba7562012-01-31 10:15:47 -0800365
Gilad Arnold55a2a372012-10-02 09:46:32 -0700366 return os.path.join(CACHE_DIR, update_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800367
Don Garrettfff4c322010-11-19 13:37:12 -0800368 def GenerateUpdateImage(self, image_path, output_dir):
Don Garrettf90edf02010-11-16 17:36:14 -0800369 """Force generates an update payload based on the given image_path.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700370
Chris Sosade91f672010-11-16 10:05:44 -0800371 Args:
Don Garrettf90edf02010-11-16 17:36:14 -0800372 src_image: image we are updating from (Null/empty for non-delta)
373 image_path: full path to the image.
Chris Sosa6a3697f2013-01-29 16:44:43 -0800374 output_dir: the directory to write the update payloads to
375 Raises:
376 AutoupdateError if it failed to generate either update or stateful
377 payload.
Chris Sosade91f672010-11-16 10:05:44 -0800378 """
Chris Sosa6a3697f2013-01-29 16:44:43 -0800379 _Log('Generating update for image %s', image_path)
Andrew de los Reyes9a528712010-06-30 10:29:43 -0700380
Chris Sosa6a3697f2013-01-29 16:44:43 -0800381 # Delete any previous state in this directory.
382 os.system('rm -rf "%s"' % output_dir)
383 os.makedirs(output_dir)
rtc@google.comded22402009-10-26 22:36:21 +0000384
Chris Sosa6a3697f2013-01-29 16:44:43 -0800385 try:
386 self.GenerateUpdateFile(self.src_image, image_path, output_dir)
387 self.GenerateStatefulFile(image_path, output_dir)
388 except subprocess.CalledProcessError:
389 os.system('rm -rf "%s"' % output_dir)
390 raise AutoupdateError('Failed to generate update in %s' % output_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800391
392 def GenerateUpdateImageWithCache(self, image_path, static_image_dir):
393 """Force generates an update payload based on the given image_path.
rtc@google.comded22402009-10-26 22:36:21 +0000394
Chris Sosa0356d3b2010-09-16 15:46:22 -0700395 Args:
396 image_path: full path to the image.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700397 static_image_dir: the directory to move images to after generating.
398 Returns:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800399 update directory relative to static_image_dir. None if it should
400 serve from the static_image_dir.
401 Raises:
402 AutoupdateError if it we need to generate a payload and fail to do so.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700403 """
Chris Sosa6a3697f2013-01-29 16:44:43 -0800404 _Log('Generating update for src %s image %s', self.src_image, image_path)
Chris Sosae67b78f2010-11-04 17:33:16 -0700405
Chris Sosa417e55d2011-01-25 16:40:48 -0800406 # If it was pregenerated_path, don't regenerate
407 if self.pregenerated_path:
408 return self.pregenerated_path
Don Garrettfff4c322010-11-19 13:37:12 -0800409
Don Garrettf90edf02010-11-16 17:36:14 -0800410 # Which sub_dir of static_image_dir should hold our cached update image
411 cache_sub_dir = self.FindCachedUpdateImageSubDir(self.src_image, image_path)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800412 _Log('Caching in sub_dir "%s"', cache_sub_dir)
Chris Sosa417e55d2011-01-25 16:40:48 -0800413
Don Garrettf90edf02010-11-16 17:36:14 -0800414 # The cached payloads exist in a cache dir
415 cache_update_payload = os.path.join(static_image_dir,
Chris Sosa6a3697f2013-01-29 16:44:43 -0800416 cache_sub_dir, UPDATE_FILE)
Don Garrettf90edf02010-11-16 17:36:14 -0800417 cache_stateful_payload = os.path.join(static_image_dir,
Chris Sosa6a3697f2013-01-29 16:44:43 -0800418 cache_sub_dir, STATEFUL_FILE)
Don Garrettf90edf02010-11-16 17:36:14 -0800419
Chris Sosa6a3697f2013-01-29 16:44:43 -0800420 full_cache_dir = os.path.join(static_image_dir, cache_sub_dir)
Chris Sosa417e55d2011-01-25 16:40:48 -0800421 # Check to see if this cache directory is valid.
422 if not os.path.exists(cache_update_payload) or not os.path.exists(
423 cache_stateful_payload):
Chris Sosa6a3697f2013-01-29 16:44:43 -0800424 self.GenerateUpdateImage(image_path, full_cache_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800425
Chris Sosa6a3697f2013-01-29 16:44:43 -0800426 self.pregenerated_path = cache_sub_dir
Chris Sosa65d339b2013-01-21 18:59:21 -0800427
Chris Sosa6a3697f2013-01-29 16:44:43 -0800428 # Generate the cache file.
429 self.GetLocalPayloadAttrs(full_cache_dir)
430 cache_metadata_file = os.path.join(full_cache_dir, METADATA_FILE)
Don Garrettf90edf02010-11-16 17:36:14 -0800431
Chris Sosa08d55a22011-01-19 16:08:02 -0800432 # Generation complete, copy if requested.
433 if self.copy_to_static_root:
Chris Sosa417e55d2011-01-25 16:40:48 -0800434 # The final results exist directly in static
Simran Basi40836b22013-03-28 15:08:17 -0700435 update_payload = os.path.join(static_image_dir,
436 UPDATE_FILE)
437 stateful_payload = os.path.join(static_image_dir,
438 STATEFUL_FILE)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800439 metadata_file = os.path.join(static_image_dir, METADATA_FILE)
Simran Basi40836b22013-03-28 15:08:17 -0700440 common_util.CopyFile(cache_update_payload, update_payload)
Gilad Arnold55a2a372012-10-02 09:46:32 -0700441 common_util.CopyFile(cache_stateful_payload, stateful_payload)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800442 common_util.CopyFile(cache_metadata_file, metadata_file)
443 return None
Chris Sosa417e55d2011-01-25 16:40:48 -0800444 else:
445 return self.pregenerated_path
Chris Sosa0356d3b2010-09-16 15:46:22 -0700446
Chris Sosa6a3697f2013-01-29 16:44:43 -0800447 def GenerateLatestUpdateImage(self, board, client_version,
Don Garrettf90edf02010-11-16 17:36:14 -0800448 static_image_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700449 """Generates an update using the latest image that has been built.
450
451 This will only generate an update if the newest update is newer than that
452 on the client or client_version is 'ForcedUpdate'.
453
454 Args:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800455 board: Name of the board.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700456 client_version: Current version of the client or 'ForcedUpdate'
457 static_image_dir: the directory to move images to after generating.
458 Returns:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800459 Name of the update directory relative to the static dir. None if it should
460 serve from the static_image_dir.
461 Raises:
462 AutoupdateError if it failed to generate the payload or can't update
463 the given client_version.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700464 """
Chris Sosa6a3697f2013-01-29 16:44:43 -0800465 latest_image_dir = self._GetLatestImageDir(board)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700466 latest_version = self._GetVersionFromDir(latest_image_dir)
467 latest_image_path = os.path.join(latest_image_dir, self._GetImageName())
468
Chris Sosa0356d3b2010-09-16 15:46:22 -0700469 # Check to see whether or not we should update.
470 if client_version != 'ForcedUpdate' and not self._CanUpdate(
471 client_version, latest_version):
Chris Sosa6a3697f2013-01-29 16:44:43 -0800472 raise AutoupdateError('Update check received but no update available '
473 'for client')
Chris Sosa0356d3b2010-09-16 15:46:22 -0700474
Don Garrettf90edf02010-11-16 17:36:14 -0800475 return self.GenerateUpdateImageWithCache(latest_image_path,
476 static_image_dir=static_image_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700477
Chris Sosa6a3697f2013-01-29 16:44:43 -0800478 def GenerateUpdatePayload(self, board, client_version, static_image_dir):
479 """Generates an update for an image and returns the relative payload dir.
Chris Sosaa73ec162010-05-03 20:18:02 -0700480
Chris Sosa6a3697f2013-01-29 16:44:43 -0800481 Returns:
482 payload dir relative to static_image_dir. None if it should
483 serve from the static_image_dir.
484 Raises:
485 AutoupdateError if it failed to generate the payload.
Don Garrettf90edf02010-11-16 17:36:14 -0800486 """
Dale Curtis723ec472010-11-30 14:06:47 -0800487 dest_path = os.path.join(static_image_dir, UPDATE_FILE)
488 dest_stateful = os.path.join(static_image_dir, STATEFUL_FILE)
489
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700490 if self.payload_path:
Don Garrett0c880e22010-11-17 18:13:37 -0800491 # If the forced payload is not already in our static_image_dir,
492 # copy it there.
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700493 src_path = os.path.abspath(self.payload_path)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800494 src_stateful = os.path.join(os.path.dirname(src_path), STATEFUL_FILE)
Don Garrettee25e552010-11-23 12:09:35 -0800495 # Only copy the files if the source directory is different from dest.
496 if os.path.dirname(src_path) != os.path.abspath(static_image_dir):
Gilad Arnold55a2a372012-10-02 09:46:32 -0700497 common_util.CopyFile(src_path, dest_path)
Don Garrettee25e552010-11-23 12:09:35 -0800498
499 # The stateful payload is optional.
500 if os.path.exists(src_stateful):
Gilad Arnold55a2a372012-10-02 09:46:32 -0700501 common_util.CopyFile(src_stateful, dest_stateful)
Don Garrettee25e552010-11-23 12:09:35 -0800502 else:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800503 _Log('WARN: %s not found. Expected for dev and test builds',
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700504 STATEFUL_FILE)
Don Garrettee25e552010-11-23 12:09:35 -0800505 if os.path.exists(dest_stateful):
506 os.remove(dest_stateful)
Don Garrett0c880e22010-11-17 18:13:37 -0800507
Chris Sosa6a3697f2013-01-29 16:44:43 -0800508 # Serve from the main directory so rel_path is None.
509 return None
Don Garrett0c880e22010-11-17 18:13:37 -0800510 elif self.forced_image:
Don Garrettf90edf02010-11-16 17:36:14 -0800511 return self.GenerateUpdateImageWithCache(
512 self.forced_image,
513 static_image_dir=static_image_dir)
Chris Sosa65d339b2013-01-21 18:59:21 -0800514 else:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800515 if not board:
516 raise AutoupdateError(
517 'Failed to generate update. '
518 'You must set --board when pre-generating latest update.')
Chris Sosa65d339b2013-01-21 18:59:21 -0800519
Chris Sosa6a3697f2013-01-29 16:44:43 -0800520 return self.GenerateLatestUpdateImage(board, client_version,
521 static_image_dir)
Chris Sosa2c048f12010-10-27 16:05:27 -0700522
523 def PreGenerateUpdate(self):
Chris Sosa417e55d2011-01-25 16:40:48 -0800524 """Pre-generates an update and prints out the relative path it.
525
Chris Sosa6a3697f2013-01-29 16:44:43 -0800526 Returns relative path of the update.
Chris Sosa65d339b2013-01-21 18:59:21 -0800527
Chris Sosa6a3697f2013-01-29 16:44:43 -0800528 Raises:
529 AutoupdateError if it failed to generate the payload.
530 """
531 _Log('Pre-generating the update payload')
532 # Does not work with labels so just use static dir.
533 pregenerated_update = self.GenerateUpdatePayload(self.board, '0.0.0.0',
534 self.static_dir)
535 print 'PREGENERATED_UPDATE=%s' % _NonePathJoin(pregenerated_update,
536 UPDATE_FILE)
Chris Sosa417e55d2011-01-25 16:40:48 -0800537 return pregenerated_update
Chris Sosa2c048f12010-10-27 16:05:27 -0700538
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700539 def _GetRemotePayloadAttrs(self, url):
540 """Returns hashes, size and delta flag of a remote update payload.
541
542 Obtain attributes of a payload file available on a remote devserver. This
543 is based on the assumption that the payload URL uses the /static prefix. We
544 need to make sure that both clients (requests) and remote devserver
545 (provisioning) preserve this invariant.
546
547 Args:
548 url: URL of statically staged remote file (http://host:port/static/...)
549 Returns:
550 A tuple containing the SHA1, SHA256, file size and whether or not it's a
551 delta payload (Boolean).
552 """
553 if self._PAYLOAD_URL_PREFIX not in url:
554 raise AutoupdateError(
555 'Payload URL does not have the expected prefix (%s)' %
556 self._PAYLOAD_URL_PREFIX)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800557
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700558 fileinfo_url = url.replace(self._PAYLOAD_URL_PREFIX,
559 self._FILEINFO_URL_PREFIX)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800560 _Log('Retrieving file info for remote payload via %s', fileinfo_url)
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700561 try:
562 conn = urllib2.urlopen(fileinfo_url)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800563 metadata_obj = Autoupdate._ReadMetadataFromStream(conn)
564 # These fields are required for remote calls.
565 if not metadata_obj:
566 raise AutoupdateError('Failed to obtain remote payload info')
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700567
Simran Basi40836b22013-03-28 15:08:17 -0700568 if not metadata_obj.is_delta_format:
569 metadata_obj.is_delta_format = ('_mton' in url) or ('_nton' in url)
570
Chris Sosa6a3697f2013-01-29 16:44:43 -0800571 return metadata_obj
572 except IOError as e:
573 raise AutoupdateError('Failed to obtain remote payload info: %s', e)
574
575 def GetLocalPayloadAttrs(self, payload_dir):
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700576 """Returns hashes, size and delta flag of a local update payload.
577
578 Args:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800579 payload_dir: Path to the directory the payload is in.
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700580 Returns:
581 A tuple containing the SHA1, SHA256, file size and whether or not it's a
582 delta payload (Boolean).
583 """
Chris Sosa6a3697f2013-01-29 16:44:43 -0800584 filename = os.path.join(payload_dir, UPDATE_FILE)
585 if not os.path.exists(filename):
586 raise AutoupdateError('update.gz not present in payload dir %s' %
587 payload_dir)
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700588
Chris Sosa6a3697f2013-01-29 16:44:43 -0800589 metadata_obj = Autoupdate._ReadMetadataFromFile(payload_dir)
590 if not metadata_obj or not (metadata_obj.sha1 and
591 metadata_obj.sha256 and
592 metadata_obj.size):
593 sha1 = common_util.GetFileSha1(filename)
594 sha256 = common_util.GetFileSha256(filename)
595 size = common_util.GetFileSize(filename)
Simran Basi40836b22013-03-28 15:08:17 -0700596 is_delta_format = self._IsDeltaFormatFile(filename)
Chris Sosa6a3697f2013-01-29 16:44:43 -0800597 metadata_obj = UpdateMetadata(sha1, sha256, size, is_delta_format)
598 Autoupdate._StoreMetadataToFile(payload_dir, metadata_obj)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700599
Chris Sosa6a3697f2013-01-29 16:44:43 -0800600 return metadata_obj
601
602 def _ProcessUpdateComponents(self, app, event):
603 """Processes the app and event components of an update request.
604
605 Returns tuple containing forced_update_label, client_version, and board.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700606 """
Chris Sosa6a3697f2013-01-29 16:44:43 -0800607 # Initialize an empty dictionary for event attributes to log.
608 log_message = {}
Jay Srinivasanac69d262012-10-30 19:05:53 -0700609
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700610 # Determine request IP, strip any IPv6 data for simplicity.
611 client_ip = cherrypy.request.remote.ip.split(':')[-1]
Gilad Arnold286a0062012-01-12 13:47:02 -0800612 # Obtain (or init) info object for this client.
613 curr_host_info = self.host_infos.GetInitHostInfo(client_ip)
614
Chris Sosa6a3697f2013-01-29 16:44:43 -0800615 client_version = 'ForcedUpdate'
616 board = None
617 if app:
618 client_version = app.getAttribute('version')
619 channel = app.getAttribute('track')
620 board = (app.hasAttribute('board') and app.getAttribute('board')
621 or self._GetDefaultBoardID())
622 # Add attributes to log message
623 log_message['version'] = client_version
624 log_message['track'] = channel
625 log_message['board'] = board
626 curr_host_info.attrs['last_known_version'] = client_version
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700627
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700628 if event:
Gilad Arnold286a0062012-01-12 13:47:02 -0800629 event_result = int(event[0].getAttribute('eventresult'))
630 event_type = int(event[0].getAttribute('eventtype'))
Gilad Arnoldb11a8942012-03-13 15:33:21 -0700631 client_previous_version = (event[0].getAttribute('previousversion')
632 if event[0].hasAttribute('previousversion')
633 else None)
Gilad Arnold286a0062012-01-12 13:47:02 -0800634 # Store attributes to legacy host info structure
635 curr_host_info.attrs['last_event_status'] = event_result
636 curr_host_info.attrs['last_event_type'] = event_type
637 # Add attributes to log message
638 log_message['event_result'] = event_result
639 log_message['event_type'] = event_type
Gilad Arnoldb11a8942012-03-13 15:33:21 -0700640 if client_previous_version is not None:
641 log_message['previous_version'] = client_previous_version
Gilad Arnold286a0062012-01-12 13:47:02 -0800642
Gilad Arnold8318eac2012-10-04 12:52:23 -0700643 # Log host event, if so instructed.
644 if self.host_log:
645 curr_host_info.AddLogEntry(log_message)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700646
Chris Sosa6a3697f2013-01-29 16:44:43 -0800647 return (curr_host_info.attrs.pop('forced_update_label', None),
648 client_version, board)
649
650 def _GetStaticUrl(self):
651 """Returns the static url base that should prefix all payload responses."""
652 x_forwarded_host = cherrypy.request.headers.get('X-Forwarded-Host')
653 if x_forwarded_host:
654 hostname = 'http://' + x_forwarded_host
655 else:
656 hostname = cherrypy.request.base
657
658 if self.urlbase:
659 static_urlbase = self.urlbase
660 elif self.serve_only:
661 static_urlbase = '%s/static/archive' % hostname
662 else:
663 static_urlbase = '%s/static' % hostname
664
665 # If we have a proxy port, adjust the URL we instruct the client to
666 # use to go through the proxy.
667 if self.proxy_port:
668 static_urlbase = _ChangeUrlPort(static_urlbase, self.proxy_port)
669
670 _Log('Using static url base %s', static_urlbase)
671 _Log('Handling update ping as %s', hostname)
672 return static_urlbase
673
674 def HandleUpdatePing(self, data, label=None):
675 """Handles an update ping from an update client.
676
677 Args:
678 data: XML blob from client.
679 label: optional label for the update.
680 Returns:
681 Update payload message for client.
682 """
683 # Get the static url base that will form that base of our update url e.g.
684 # http://hostname:8080/static/update.gz.
685 static_urlbase = self._GetStaticUrl()
686
687 # Parse the XML we got into the components we care about.
688 protocol, app, event, update_check = autoupdate_lib.ParseUpdateRequest(data)
689
690 # #########################################################################
691 # Process attributes of the update check.
692 forced_update_label, client_version, board = self._ProcessUpdateComponents(
693 app, event)
694
695 # We only process update_checks in the update rpc.
Chris Sosa0356d3b2010-09-16 15:46:22 -0700696 if not update_check:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800697 _Log('Non-update check received. Returning blank payload')
Chris Sosa0356d3b2010-09-16 15:46:22 -0700698 # TODO(sosa): Generate correct non-updatecheck payload to better test
699 # update clients.
Chris Sosa52148582012-11-15 15:35:58 -0800700 return autoupdate_lib.GetNoUpdateResponse(protocol)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700701
Chris Sosa6a3697f2013-01-29 16:44:43 -0800702 # In case max_updates is used, return no response if max reached.
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700703 if self.max_updates > 0:
704 self.max_updates -= 1
705 elif self.max_updates == 0:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800706 _Log('Request received but max number of updates handled')
Chris Sosa52148582012-11-15 15:35:58 -0800707 return autoupdate_lib.GetNoUpdateResponse(protocol)
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700708
Chris Sosa6a3697f2013-01-29 16:44:43 -0800709 _Log('Update Check Received. Client is using protocol version: %s',
710 protocol)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700711
Chris Sosa6a3697f2013-01-29 16:44:43 -0800712 if forced_update_label:
713 if label:
714 _Log('Label: %s set but being overwritten to %s by request', label,
715 forced_update_label)
716
717 label = forced_update_label
718
719 # #########################################################################
720 # Finally its time to generate the omaha response to give to client that
721 # lets them know where to find the payload and its associated metadata.
722 metadata_obj = None
723
724 try:
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700725 # Are we provisioning a remote or local payload?
726 if self.remote_payload:
727 # If no explicit label was provided, use the value of --payload.
Chris Sosa6a3697f2013-01-29 16:44:43 -0800728 if not label:
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700729 label = self.payload_path
Chris Sosa0356d3b2010-09-16 15:46:22 -0700730
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700731 # Form the URL of the update payload. This assumes that the payload
732 # file name is a devserver constant (which currently is the case).
733 url = '/'.join(filter(None, [static_urlbase, label, UPDATE_FILE]))
Chris Sosa5d342a22010-09-28 16:54:41 -0700734
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700735 # Get remote payload attributes.
Chris Sosa6a3697f2013-01-29 16:44:43 -0800736 metadata_obj = self._GetRemotePayloadAttrs(url)
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700737 else:
Chris Sosa6a3697f2013-01-29 16:44:43 -0800738 static_image_dir = _NonePathJoin(self.static_dir, label)
739 rel_path = None
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700740
Chris Sosa6a3697f2013-01-29 16:44:43 -0800741 # Serving files only, don't generate an update.
742 if not self.serve_only:
743 # Generate payload if necessary.
744 rel_path = self.GenerateUpdatePayload(board, client_version,
745 static_image_dir)
746
747 url = '/'.join(filter(None, [static_urlbase, label, rel_path,
748 UPDATE_FILE]))
749 local_payload_dir = _NonePathJoin(static_image_dir, rel_path)
750 metadata_obj = self.GetLocalPayloadAttrs(local_payload_dir)
751
752 except AutoupdateError as e:
753 # Raised if we fail to generate an update payload.
754 _Log('Failed to process an update: %r', e)
755 return autoupdate_lib.GetNoUpdateResponse(protocol)
756
757 _Log('Responding to client to use url %s to get image', url)
758 return autoupdate_lib.GetUpdateResponse(
759 metadata_obj.sha1, metadata_obj.sha256, metadata_obj.size, url,
760 metadata_obj.is_delta_format, protocol, self.critical_update)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700761
762 def HandleHostInfoPing(self, ip):
763 """Returns host info dictionary for the given IP in JSON format."""
764 assert ip, 'No ip provided.'
Gilad Arnold286a0062012-01-12 13:47:02 -0800765 if ip in self.host_infos.table:
766 return json.dumps(self.host_infos.GetHostInfo(ip).attrs)
767
768 def HandleHostLogPing(self, ip):
769 """Returns a complete log of events for host in JSON format."""
Gilad Arnold4ba437d2012-10-05 15:28:27 -0700770 # If all events requested, return a dictionary of logs keyed by IP address.
Gilad Arnold286a0062012-01-12 13:47:02 -0800771 if ip == 'all':
772 return json.dumps(
773 dict([(key, self.host_infos.table[key].log)
774 for key in self.host_infos.table]))
Gilad Arnold4ba437d2012-10-05 15:28:27 -0700775
776 # Otherwise we're looking for a specific IP address, so find its log.
Gilad Arnold286a0062012-01-12 13:47:02 -0800777 if ip in self.host_infos.table:
778 return json.dumps(self.host_infos.GetHostInfo(ip).log)
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700779
Gilad Arnold4ba437d2012-10-05 15:28:27 -0700780 # If no events were logged for this IP, return an empty log.
781 return json.dumps([])
782
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700783 def HandleSetUpdatePing(self, ip, label):
784 """Sets forced_update_label for a given host."""
785 assert ip, 'No ip provided.'
786 assert label, 'No label provided.'
Gilad Arnold286a0062012-01-12 13:47:02 -0800787 self.host_infos.GetInitHostInfo(ip).attrs['forced_update_label'] = label