blob: a052d42cab7caf8d70259793b8bf8fb260efa4c1 [file] [log] [blame]
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00001# Copyright 2009 Google Inc. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000015"""Generic utils."""
16
maruel@chromium.org167b9e62009-09-17 17:41:02 +000017import errno
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +000018import logging
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000019import os
msb@chromium.orgac915bb2009-11-13 17:03:01 +000020import re
bradnelson@google.com8f9c69f2009-09-17 00:48:28 +000021import stat
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000022import subprocess
23import sys
maruel@chromium.org167b9e62009-09-17 17:41:02 +000024import time
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000025import xml.dom.minidom
maruel@chromium.org167b9e62009-09-17 17:41:02 +000026import xml.parsers.expat
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000027
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000028
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000029class CheckCallError(OSError):
30 """CheckCall() returned non-0."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000031 def __init__(self, command, cwd, retcode, stdout, stderr=None):
32 OSError.__init__(self, command, cwd, retcode, stdout, stderr)
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000033 self.command = command
34 self.cwd = cwd
35 self.retcode = retcode
36 self.stdout = stdout
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000037 self.stderr = stderr
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000038
39
maruel@chromium.orgea8c1a92009-12-20 17:21:59 +000040def CheckCall(command, cwd=None, print_error=True):
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000041 """Like subprocess.check_call() but returns stdout.
42
43 Works on python 2.4
44 """
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +000045 logging.debug("%s, cwd=%s" % (str(command), str(cwd)))
maruel@chromium.org18111352009-12-20 17:21:28 +000046 try:
maruel@chromium.orgea8c1a92009-12-20 17:21:59 +000047 stderr = None
48 if not print_error:
49 stderr = subprocess.PIPE
maruel@chromium.org18111352009-12-20 17:21:28 +000050 process = subprocess.Popen(command, cwd=cwd,
51 shell=sys.platform.startswith('win'),
maruel@chromium.orgea8c1a92009-12-20 17:21:59 +000052 stdout=subprocess.PIPE,
53 stderr=stderr)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000054 std_out, std_err = process.communicate()
maruel@chromium.org18111352009-12-20 17:21:28 +000055 except OSError, e:
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +000056 raise CheckCallError(command, cwd, e.errno, None)
maruel@chromium.org18111352009-12-20 17:21:28 +000057 if process.returncode:
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000058 raise CheckCallError(command, cwd, process.returncode, std_out, std_err)
59 return std_out, std_err
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000060
61
msb@chromium.orgac915bb2009-11-13 17:03:01 +000062def SplitUrlRevision(url):
63 """Splits url and returns a two-tuple: url, rev"""
64 if url.startswith('ssh:'):
65 # Make sure ssh://test@example.com/test.git@stable works
msb@chromium.orgb9f2f622009-11-19 23:45:35 +000066 regex = r"(ssh://(?:[\w]+@)?[-\w:\.]+/[-\w\./]+)(?:@(.+))?"
msb@chromium.orgac915bb2009-11-13 17:03:01 +000067 components = re.search(regex, url).groups()
68 else:
69 components = url.split("@")
70 if len(components) == 1:
71 components += [None]
72 return tuple(components)
73
74
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000075def ParseXML(output):
76 try:
77 return xml.dom.minidom.parseString(output)
78 except xml.parsers.expat.ExpatError:
79 return None
80
81
82def GetNamedNodeText(node, node_name):
83 child_nodes = node.getElementsByTagName(node_name)
84 if not child_nodes:
85 return None
86 assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1
87 return child_nodes[0].firstChild.nodeValue
88
89
90def GetNodeNamedAttributeText(node, node_name, attribute_name):
91 child_nodes = node.getElementsByTagName(node_name)
92 if not child_nodes:
93 return None
94 assert len(child_nodes) == 1
95 return child_nodes[0].getAttribute(attribute_name)
96
97
98class Error(Exception):
99 """gclient exception class."""
100 pass
101
102
103class PrintableObject(object):
104 def __str__(self):
105 output = ''
106 for i in dir(self):
107 if i.startswith('__'):
108 continue
109 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
110 return output
111
112
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000113def FileRead(filename, mode='rU'):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000114 content = None
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000115 f = open(filename, mode)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000116 try:
117 content = f.read()
118 finally:
119 f.close()
120 return content
121
122
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000123def FileWrite(filename, content, mode='w'):
124 f = open(filename, mode)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000125 try:
126 f.write(content)
127 finally:
128 f.close()
129
130
131def RemoveDirectory(*path):
132 """Recursively removes a directory, even if it's marked read-only.
133
134 Remove the directory located at *path, if it exists.
135
136 shutil.rmtree() doesn't work on Windows if any of the files or directories
137 are read-only, which svn repositories and some .svn files are. We need to
138 be able to force the files to be writable (i.e., deletable) as we traverse
139 the tree.
140
141 Even with all this, Windows still sometimes fails to delete a file, citing
142 a permission error (maybe something to do with antivirus scans or disk
143 indexing). The best suggestion any of the user forums had was to wait a
144 bit and try again, so we do that too. It's hand-waving, but sometimes it
145 works. :/
146
147 On POSIX systems, things are a little bit simpler. The modes of the files
148 to be deleted doesn't matter, only the modes of the directories containing
149 them are significant. As the directory tree is traversed, each directory
150 has its mode set appropriately before descending into it. This should
151 result in the entire tree being removed, with the possible exception of
152 *path itself, because nothing attempts to change the mode of its parent.
153 Doing so would be hazardous, as it's not a directory slated for removal.
154 In the ordinary case, this is not a problem: for our purposes, the user
155 will never lack write permission on *path's parent.
156 """
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +0000157 logging.debug(path)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000158 file_path = os.path.join(*path)
159 if not os.path.exists(file_path):
160 return
161
162 if os.path.islink(file_path) or not os.path.isdir(file_path):
163 raise Error("RemoveDirectory asked to remove non-directory %s" % file_path)
164
165 has_win32api = False
166 if sys.platform == 'win32':
167 has_win32api = True
168 # Some people don't have the APIs installed. In that case we'll do without.
169 try:
170 win32api = __import__('win32api')
171 win32con = __import__('win32con')
172 except ImportError:
173 has_win32api = False
174 else:
175 # On POSIX systems, we need the x-bit set on the directory to access it,
176 # the r-bit to see its contents, and the w-bit to remove files from it.
177 # The actual modes of the files within the directory is irrelevant.
178 os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
179 for fn in os.listdir(file_path):
180 fullpath = os.path.join(file_path, fn)
181
182 # If fullpath is a symbolic link that points to a directory, isdir will
183 # be True, but we don't want to descend into that as a directory, we just
184 # want to remove the link. Check islink and treat links as ordinary files
185 # would be treated regardless of what they reference.
186 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
187 if sys.platform == 'win32':
188 os.chmod(fullpath, stat.S_IWRITE)
189 if has_win32api:
190 win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL)
191 try:
192 os.remove(fullpath)
193 except OSError, e:
194 if e.errno != errno.EACCES or sys.platform != 'win32':
195 raise
196 print 'Failed to delete %s: trying again' % fullpath
197 time.sleep(0.1)
198 os.remove(fullpath)
199 else:
200 RemoveDirectory(fullpath)
201
202 if sys.platform == 'win32':
203 os.chmod(file_path, stat.S_IWRITE)
204 if has_win32api:
205 win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL)
206 try:
207 os.rmdir(file_path)
208 except OSError, e:
209 if e.errno != errno.EACCES or sys.platform != 'win32':
210 raise
211 print 'Failed to remove %s: trying again' % file_path
212 time.sleep(0.1)
213 os.rmdir(file_path)
214
215
216def SubprocessCall(command, in_directory, fail_status=None):
217 """Runs command, a list, in directory in_directory.
218
219 This function wraps SubprocessCallAndFilter, but does not perform the
220 filtering functions. See that function for a more complete usage
221 description.
222 """
223 # Call subprocess and capture nothing:
224 SubprocessCallAndFilter(command, in_directory, True, True, fail_status)
225
226
227def SubprocessCallAndFilter(command,
228 in_directory,
229 print_messages,
230 print_stdout,
231 fail_status=None, filter=None):
232 """Runs command, a list, in directory in_directory.
233
234 If print_messages is true, a message indicating what is being done
dpranke@google.com22e29d42009-10-28 00:48:26 +0000235 is printed to stdout. If print_messages is false, the message is printed
236 only if we actually need to print something else as well, so you can
237 get the context of the output. If print_messages is false and print_stdout
238 is false, no output at all is generated.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000239
240 Also, if print_stdout is true, the command's stdout is also forwarded
241 to stdout.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000242
243 If a filter function is specified, it is expected to take a single
244 string argument, and it will be called with each line of the
245 subprocess's output. Each line has had the trailing newline character
246 trimmed.
247
248 If the command fails, as indicated by a nonzero exit status, gclient will
249 exit with an exit status of fail_status. If fail_status is None (the
250 default), gclient will raise an Error exception.
251 """
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +0000252 logging.debug(command)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000253 if print_messages:
254 print("\n________ running \'%s\' in \'%s\'"
255 % (' '.join(command), in_directory))
256
257 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
258 # executable, but shell=True makes subprocess on Linux fail when it's called
259 # with a list because it only tries to execute the first item in the list.
260 kid = subprocess.Popen(command, bufsize=0, cwd=in_directory,
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000261 shell=(sys.platform == 'win32'), stdout=subprocess.PIPE,
dpranke@google.com5cc6c572009-11-06 20:04:56 +0000262 stderr=subprocess.STDOUT)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000263
264 # Also, we need to forward stdout to prevent weird re-ordering of output.
265 # This has to be done on a per byte basis to make sure it is not buffered:
266 # normally buffering is done for each line, but if svn requests input, no
267 # end-of-line character is output after the prompt and it would not show up.
268 in_byte = kid.stdout.read(1)
269 in_line = ""
270 while in_byte:
271 if in_byte != "\r":
272 if print_stdout:
dpranke@google.com22e29d42009-10-28 00:48:26 +0000273 if not print_messages:
274 print("\n________ running \'%s\' in \'%s\'"
275 % (' '.join(command), in_directory))
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000276 print_messages = True
dpranke@google.com9e890f92009-10-28 01:32:29 +0000277 sys.stdout.write(in_byte)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000278 if in_byte != "\n":
279 in_line += in_byte
280 if in_byte == "\n" and filter:
281 filter(in_line)
282 in_line = ""
283 in_byte = kid.stdout.read(1)
284 rv = kid.wait()
285
286 if rv:
287 msg = "failed to run command: %s" % " ".join(command)
288
289 if fail_status != None:
290 print >>sys.stderr, msg
291 sys.exit(fail_status)
292
293 raise Error(msg)
294
295
296def IsUsingGit(root, paths):
297 """Returns True if we're using git to manage any of our checkouts.
298 |entries| is a list of paths to check."""
299 for path in paths:
300 if os.path.exists(os.path.join(root, path, '.git')):
301 return True
302 return False
maruel@chromium.orga9371762009-12-22 18:27:38 +0000303
304def FindGclientRoot(from_dir):
305 """Tries to find the gclient root."""
306 path = os.path.realpath(from_dir)
307 while not os.path.exists(os.path.join(path, '.gclient')):
308 next = os.path.split(path)
309 if not next[1]:
310 return None
311 path = next[0]
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +0000312 logging.info('Found gclient root at ' + path)
maruel@chromium.orga9371762009-12-22 18:27:38 +0000313 return path
maruel@chromium.org3ccbf7e2009-12-22 20:46:42 +0000314
315def PathDifference(root, subpath):
316 """Returns the difference subpath minus root."""
317 root = os.path.realpath(root)
318 subpath = os.path.realpath(subpath)
319 if not subpath.startswith(root):
320 return None
321 # If the root does not have a trailing \ or /, we add it so the returned
322 # path starts immediately after the seperator regardless of whether it is
323 # provided.
324 root = os.path.join(root, '')
325 return subpath[len(root):]
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000326
327
328def FindFileUpwards(filename, path=None):
329 """Search upwards from the a directory (default: current) to find a file."""
330 if not path:
331 path = os.getcwd()
332 path = os.path.realpath(path)
333 while True:
334 file_path = os.path.join(path, filename)
335 if os.path.isfile(file_path):
336 return file_path
337 (new_path, _) = os.path.split(path)
338 if new_path == path:
339 return None
340 path = new_path
341
342
343def GetGClientRootAndEntries(path=None):
344 """Returns the gclient root and the dict of entries."""
345 config_file = '.gclient_entries'
346 config_path = FindFileUpwards(config_file, path)
347
348 if not config_path:
349 print "Can't find", config_file
350 return None
351
352 env = {}
353 execfile(config_path, env)
354 config_dir = os.path.dirname(config_path)
355 return config_dir, env['entries']