blob: 40622512d5d778f6521d415eec130c2ee50ec955 [file] [log] [blame]
Ahmad Sharif4467f002012-12-20 12:09:49 -08001#!/usr/bin/python
Ahmad Shariffd356fb2012-05-07 12:02:16 -07002#
3# Copyright 2010 Google Inc. All Rights Reserved.
4
5"""Utilities for toolchain build."""
6
7__author__ = "asharif@google.com (Ahmad Sharif)"
8
Ahmad Shariff395c262012-10-09 17:48:09 -07009from contextlib import contextmanager
Ahmad Shariffd356fb2012-05-07 12:02:16 -070010import os
11import re
Ahmad Sharif4467f002012-12-20 12:09:49 -080012import shutil
13import sys
14import time
15
16import lock_machine
Ahmad Shariff395c262012-10-09 17:48:09 -070017
Ahmad Shariffd356fb2012-05-07 12:02:16 -070018import command_executer
19import logger
Ahmad Shariff395c262012-10-09 17:48:09 -070020
21
22def GetChromeOSVersionFromLSBVersion(lsb_version):
Ahmad Sharif4467f002012-12-20 12:09:49 -080023 """Get Chromeos version from Lsb version."""
Ahmad Shariff395c262012-10-09 17:48:09 -070024 ce = command_executer.GetCommandExecuter()
25 command = "git ls-remote http://git.chromium.org/chromiumos/manifest.git"
26 ret, out, _ = ce.RunCommand(command, return_output=True,
27 print_to_console=False)
28 assert ret == 0, "Command %s failed" % command
29 lower = []
30 for line in out.splitlines():
Ahmad Sharif4467f002012-12-20 12:09:49 -080031 mo = re.search(r"refs/heads/release-R(\d+)-(\d+)\.B", line)
Ahmad Shariff395c262012-10-09 17:48:09 -070032 if mo:
33 revision = int(mo.group(1))
34 build = int(mo.group(2))
35 lsb_build = int(lsb_version.split(".")[0])
36 if lsb_build > build:
37 lower.append(revision)
38 lower = sorted(lower)
39 if lower:
40 return "R%d-%s" % (lower[-1] + 1, lsb_version)
41 else:
42 return "Unknown"
Ahmad Shariffd356fb2012-05-07 12:02:16 -070043
44
45def ApplySubs(string, *substitutions):
46 for pattern, replacement in substitutions:
47 string = re.sub(pattern, replacement, string)
48 return string
49
50
Ahmad Sharif4467f002012-12-20 12:09:49 -080051def UnitToNumber(unit_num, base=1000):
52 """Convert a number with unit to float."""
Ahmad Sharif5ae8a5c2012-05-18 10:59:51 -070053 unit_dict = {"kilo": base,
54 "mega": base**2,
55 "giga": base**3}
Ahmad Sharif4467f002012-12-20 12:09:49 -080056 unit_num = unit_num.lower()
57 mo = re.search(r"(\d*)(.+)?", unit_num)
Ahmad Sharif5ae8a5c2012-05-18 10:59:51 -070058 number = mo.group(1)
59 unit = mo.group(2)
Ahmad Shariff395c262012-10-09 17:48:09 -070060 if not unit:
61 return float(number)
Ahmad Sharif5ae8a5c2012-05-18 10:59:51 -070062 for k, v in unit_dict.items():
63 if k.startswith(unit):
64 return float(number) * v
65 raise Exception("Unit: %s not found in byte: %s!" %
66 (unit,
Ahmad Sharif4467f002012-12-20 12:09:49 -080067 unit_num))
Ahmad Sharif5ae8a5c2012-05-18 10:59:51 -070068
69
Ahmad Shariffd356fb2012-05-07 12:02:16 -070070def GetFilenameFromString(string):
71 return ApplySubs(string,
Ahmad Sharif4467f002012-12-20 12:09:49 -080072 (r"/", "__"),
73 (r"\s", "_"),
74 (r"[\^\$=\"\\\?]", ""),
75 )
Ahmad Shariffd356fb2012-05-07 12:02:16 -070076
77
78def GetRoot(scr_name):
79 """Break up pathname into (dir+name)."""
80 abs_path = os.path.abspath(scr_name)
81 return (os.path.dirname(abs_path), os.path.basename(abs_path))
82
83
Ahmad Sharif4467f002012-12-20 12:09:49 -080084def GetChromeOSKeyFile(chromeos_root):
85 return os.path.join(chromeos_root,
86 "src",
87 "scripts",
88 "mod_for_test_scripts",
89 "ssh_keys",
90 "testing_rsa")
91
92
Ahmad Sharif5ae8a5c2012-05-18 10:59:51 -070093def GetChrootPath(chromeos_root):
94 return os.path.join(chromeos_root,
95 "chroot")
96
97
98def GetInsideChrootPath(chromeos_root, file_path):
99 if not file_path.startswith(GetChrootPath(chromeos_root)):
100 raise Exception("File: %s doesn't seem to be in the chroot: %s" %
101 (file_path,
102 chromeos_root))
103 return file_path[len(GetChrootPath(chromeos_root)):]
104
105
106def GetOutsideChrootPath(chromeos_root, file_path):
107 return os.path.join(GetChrootPath(chromeos_root),
108 file_path.lstrip("/"))
109
110
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700111def FormatQuotedCommand(command):
112 return ApplySubs(command,
113 ("\"", "\\\""))
114
115
116def FormatCommands(commands):
117 return ApplySubs(str(commands),
118 ("&&", "&&\n"),
119 (";", ";\n"),
Ahmad Sharif4467f002012-12-20 12:09:49 -0800120 (r"\n+\s*", "\n"))
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700121
122
123def GetImageDir(chromeos_root, board):
124 return os.path.join(chromeos_root,
125 "src",
126 "build",
127 "images",
128 board)
129
130
131def LabelLatestImage(chromeos_root, board, label):
132 image_dir = GetImageDir(chromeos_root, board)
133 latest_image_dir = os.path.join(image_dir, "latest")
134 latest_image_dir = os.path.realpath(latest_image_dir)
135 latest_image_dir = os.path.basename(latest_image_dir)
136 with WorkingDirectory(image_dir):
137 command = "ln -sf -T %s %s" % (latest_image_dir, label)
138 ce = command_executer.GetCommandExecuter()
139 return ce.RunCommand(command)
140
141
142def DoesLabelExist(chromeos_root, board, label):
143 image_label = os.path.join(GetImageDir(chromeos_root, board),
144 label)
145 return os.path.exists(image_label)
146
147
148def GetBuildPackagesCommand(board, usepkg=False):
149 if usepkg:
150 usepkg_flag = "--usepkg"
151 else:
152 usepkg_flag = "--nousepkg"
153 return ("./build_packages %s --withdev --withtest --withautotest "
154 "--skip_toolchain_update --nowithdebug --board=%s" %
155 (usepkg_flag, board))
156
157
158def GetBuildImageCommand(board):
159 return "./build_image --board=%s test" % board
160
161
162def GetSetupBoardCommand(board, gcc_version=None, binutils_version=None,
163 usepkg=None, force=None):
Ahmad Sharif4467f002012-12-20 12:09:49 -0800164 """Get setup_board command."""
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700165 options = []
166
167 if gcc_version:
168 options.append("--gcc_version=%s" % gcc_version)
169
170 if binutils_version:
171 options.append("--binutils_version=%s" % binutils_version)
172
173 if usepkg:
174 options.append("--usepkg")
175 else:
176 options.append("--nousepkg")
177
178 if force:
179 options.append("--force")
180
181 return "./setup_board --board=%s %s" % (board, " ".join(options))
182
183
184def CanonicalizePath(path):
185 path = os.path.expanduser(path)
186 path = os.path.realpath(path)
187 return path
188
189
190def GetCtargetFromBoard(board, chromeos_root):
Ahmad Sharif4467f002012-12-20 12:09:49 -0800191 """Get Ctarget from board."""
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700192 base_board = board.split("_")[0]
193 command = ("source "
194 "../platform/dev/toolchain_utils.sh; get_ctarget_from_board %s" %
195 base_board)
196 ce = command_executer.GetCommandExecuter()
Ahmad Shariff395c262012-10-09 17:48:09 -0700197 ret, out, _ = ce.ChrootRunCommand(chromeos_root,
198 command,
199 return_output=True)
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700200 if ret != 0:
201 raise ValueError("Board %s is invalid!" % board)
Ahmad Shariff395c262012-10-09 17:48:09 -0700202 # Remove ANSI escape sequences.
203 out = StripANSIEscapeSequences(out)
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700204 return out.strip()
205
206
Ahmad Shariff395c262012-10-09 17:48:09 -0700207def StripANSIEscapeSequences(string):
Ahmad Sharif4467f002012-12-20 12:09:49 -0800208 string = re.sub(r"\x1b\[[0-9]*[a-zA-Z]", "", string)
Ahmad Shariff395c262012-10-09 17:48:09 -0700209 return string
210
211
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700212def GetChromeSrcDir():
213 return "var/cache/distfiles/target/chrome-src/src"
214
215
216def GetEnvStringFromDict(env_dict):
217 return " ".join(["%s=\"%s\"" % var for var in env_dict.items()])
218
219
Ahmad Shariff395c262012-10-09 17:48:09 -0700220def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
Ahmad Sharif4467f002012-12-20 12:09:49 -0800221 """Merge env string with dict."""
Ahmad Shariff395c262012-10-09 17:48:09 -0700222 if not env_string.strip():
223 return GetEnvStringFromDict(env_dict)
224 override_env_list = []
225 ce = command_executer.GetCommandExecuter()
226 for k, v in env_dict.items():
227 v = v.strip("\"'")
228 if prepend:
229 new_env = "%s=\"%s $%s\"" % (k, v, k)
230 else:
231 new_env = "%s=\"$%s %s\"" % (k, k, v)
232 command = "; ".join([env_string, new_env, "echo $%s" % k])
233 ret, out, _ = ce.RunCommand(command, return_output=True)
234 override_env_list.append("%s=%r" % (k, out.strip()))
235 ret = env_string + " " + " ".join(override_env_list)
236 return ret.strip()
237
238
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700239def GetAllImages(chromeos_root, board):
240 ce = command_executer.GetCommandExecuter()
241 command = ("find %s/src/build/images/%s -name chromiumos_test_image.bin" %
242 (chromeos_root, board))
Ahmad Shariff395c262012-10-09 17:48:09 -0700243 ret, out, _ = ce.RunCommand(command, return_output=True)
244 assert ret == 0, "Could not run command: %s" % command
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700245 return out.splitlines()
246
247
Ahmad Sharif4467f002012-12-20 12:09:49 -0800248def AcquireLock(lock_file, timeout=1200):
249 """Acquire a lock with timeout."""
250 start_time = time.time()
251 locked = False
252 abs_path = os.path.abspath(lock_file)
253 dir_path = os.path.dirname(abs_path)
254 sleep_time = min(10, timeout/10.0)
255 if not os.path.exists(dir_path):
256 try:
257 os.makedirs(dir_path)
258 except OSError:
259 print "Cannot create dir {0}, exiting...".format(dir_path)
260 exit(0)
261 while True:
262 locked = (lock_machine.Lock(lock_file).NonBlockingLock(True, sys.argv[0]))
263 if locked:
264 break
265 time.sleep(sleep_time)
266 if time.time() - start_time > timeout:
267 logger.GetLogger().LogWarning(
268 "Could not acquire lock on this file: {0} within {1} seconds."
269 "Manually remove the file if you think the lock is stale"
270 .format(abs_path, timeout))
271 break
272 return locked
273
274
275def ReleaseLock(lock_file):
276 lock_file = os.path.abspath(lock_file)
277 ret = lock_machine.Lock(lock_file).Unlock(True)
278 assert ret, ("Could not unlock {0},"
279 "Please remove it manually".format(lock_file))
280
281
282def IsFloat(text):
283 if text is None:
284 return False
285 try:
286 float(text)
287 return True
288 except ValueError:
289 return False
290
291def RemoveChromeBrowserObjectFiles(chromeos_root, board):
292 """ Remove any object files from all the posible locations """
293 out_dir = os.path.join(
294 GetChrootPath(chromeos_root),
295 "var/cache/chromeos-chrome/chrome-src/src/out_%s" % board)
296 if os.path.exists(out_dir):
297 shutil.rmtree(out_dir)
298 logger.GetLogger().LogCmd("rm -rf %s" % out_dir)
299 out_dir = os.path.join(
300 GetChrootPath(chromeos_root),
301 "var/cache/chromeos-chrome/chrome-src-internal/src/out_%s" % board)
302 if os.path.exists(out_dir):
303 shutil.rmtree(out_dir)
304 logger.GetLogger().LogCmd("rm -rf %s" % out_dir)
305
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700306@contextmanager
307def WorkingDirectory(new_dir):
Ahmad Sharif4467f002012-12-20 12:09:49 -0800308 """Get the working directory."""
Ahmad Shariffd356fb2012-05-07 12:02:16 -0700309 old_dir = os.getcwd()
310 if old_dir != new_dir:
311 msg = "cd %s" % new_dir
312 logger.GetLogger().LogCmd(msg)
313 os.chdir(new_dir)
314 yield new_dir
315 if old_dir != new_dir:
316 msg = "cd %s" % old_dir
317 logger.GetLogger().LogCmd(msg)
318 os.chdir(old_dir)