blob: aea544ce80fab91625e5f6cfb06cd85a74a31592 [file] [log] [blame]
mblighf1c52842007-10-16 15:21:38 +00001"""
2The main job wrapper for the server side.
3
4This is the core infrastructure. Derived from the client side job.py
5
6Copyright Martin J. Bligh, Andy Whitcroft 2007
7"""
8
9__author__ = """
10Martin J. Bligh <mbligh@google.com>
11Andy Whitcroft <apw@shadowen.org>
12"""
13
mbligh6e294382007-11-05 18:11:29 +000014import os, sys, re, time
mbligh03f4fc72007-11-29 20:56:14 +000015import test
mblighf1c52842007-10-16 15:21:38 +000016from utils import *
mblighf31b0c02007-11-29 18:19:22 +000017from common.error import *
mblighf1c52842007-10-16 15:21:38 +000018
mbligh3f4bced2007-11-05 17:55:53 +000019# this magic incantation should give us access to a client library
20server_dir = os.path.dirname(__file__)
21client_dir = os.path.join(server_dir, "..", "client", "bin")
22sys.path.append(client_dir)
23import fd_stack
24sys.path.pop()
25
mblighed5a4102007-11-20 00:46:41 +000026# load up a control segment
27# these are all stored in <server_dir>/control_segments
28def load_control_segment(name):
29 server_dir = os.path.dirname(os.path.abspath(__file__))
mbligh7f86e0b2007-11-24 19:45:07 +000030 script_file = os.path.join(server_dir, "control_segments", name)
mblighed5a4102007-11-20 00:46:41 +000031 if os.path.exists(script_file):
32 return file(script_file).read()
33 else:
34 return ""
35
36
mblighf1c52842007-10-16 15:21:38 +000037preamble = """\
38import os, sys
39
mbligh03f4fc72007-11-29 20:56:14 +000040import hosts, autotest, kvm
mblighf1c52842007-10-16 15:21:38 +000041import source_kernel, rpm_kernel, deb_kernel
mbligh03f4fc72007-11-29 20:56:14 +000042from common.error import *
mblighf1c52842007-10-16 15:21:38 +000043from subcommand import *
44from utils import run, get_tmp_dir, sh_escape
45
mbligh119c12a2007-11-12 22:13:44 +000046autotest.Autotest.job = job
mbligh31a49de2007-11-05 18:41:19 +000047hosts.SSHHost.job = job
mblighf1c52842007-10-16 15:21:38 +000048"""
49
50client_wrapper = """
51at = autotest.Autotest()
52
53def run_client(machine):
54 host = hosts.SSHHost(machine)
55 at.run(control, host=host)
56
57if len(machines) > 1:
mbligh7b32ba32007-11-05 18:14:20 +000058 open('.machines', 'w').write('\\n'.join(machines) + '\\n')
mblighf1c52842007-10-16 15:21:38 +000059 parallel_simple(run_client, machines)
60else:
61 run_client(machines[0])
62"""
63
mbligh303ccac2007-11-05 18:07:28 +000064crashdumps = """
65def crashdumps(machine):
66 host = hosts.SSHHost(machine, initialize=False)
67 host.get_crashdumps(test_start_time)
68
69parallel_simple(crashdumps, machines, log=False)
70"""
71
mblighf1c52842007-10-16 15:21:38 +000072cleanup="""\
73def cleanup(machine):
mbligh17f0c662007-11-05 18:28:19 +000074 host = hosts.SSHHost(machine, initialize=False)
75 host.reboot()
mblighf1c52842007-10-16 15:21:38 +000076
mbligh84c0ab12007-10-24 21:28:58 +000077parallel_simple(cleanup, machines, log=False)
mblighf1c52842007-10-16 15:21:38 +000078"""
79
mblighf36243d2007-10-30 15:36:16 +000080install="""\
81def install(machine):
mbligh17f0c662007-11-05 18:28:19 +000082 host = hosts.SSHHost(machine, initialize=False)
83 host.machine_install()
mblighf36243d2007-10-30 15:36:16 +000084
mbligh009b25a2007-11-05 18:38:51 +000085parallel_simple(install, machines, log=False)
mblighf36243d2007-10-30 15:36:16 +000086"""
87
mbligh7f86e0b2007-11-24 19:45:07 +000088# load up the verifier control segment, with an optional site-specific hook
mblighed5a4102007-11-20 00:46:41 +000089verify = load_control_segment("site_verify")
90verify += load_control_segment("verify")
mbligh1d42d4e2007-11-05 22:42:00 +000091
mbligh7f86e0b2007-11-24 19:45:07 +000092# load up the repair control segment, with an optional site-specific hook
93repair = load_control_segment("site_repair")
94repair += load_control_segment("repair")
95
96cleanup="""\
mbligh1d42d4e2007-11-05 22:42:00 +000097def cleanup(machine):
98 host = hosts.SSHHost(machine, initialize=False)
99 host.ssh_ping(150*60) # wait for 2.5 hours
100
101parallel_simple(cleanup, machines, log=False)
102"""
103
104def verify_machines(machines):
mbligh7f86e0b2007-11-24 19:45:07 +0000105 if not machines:
mbligh03f4fc72007-11-29 20:56:14 +0000106 raise AutoservError('No machines specified to verify')
mbligh1d42d4e2007-11-05 22:42:00 +0000107 namespace = {'machines' : machines, 'job' : None}
108 exec(preamble + verify, namespace, namespace)
109
110
111def repair_machines(machines):
mbligh7f86e0b2007-11-24 19:45:07 +0000112 if not machines:
mbligh03f4fc72007-11-29 20:56:14 +0000113 raise AutoservError('No machines specified to repair')
mbligh1d42d4e2007-11-05 22:42:00 +0000114 namespace = {'machines' : machines, 'job' : None}
115 exec(preamble + repair, namespace, namespace)
116
117
mblighf1c52842007-10-16 15:21:38 +0000118class server_job:
119 """The actual job against which we do everything.
120
121 Properties:
122 autodir
123 The top level autotest directory (/usr/local/autotest).
124 serverdir
125 <autodir>/server/
126 clientdir
127 <autodir>/client/
128 conmuxdir
129 <autodir>/conmux/
130 testdir
131 <autodir>/server/tests/
132 control
133 the control file for this job
134 """
135
mbligh18420c22007-10-16 22:27:14 +0000136 def __init__(self, control, args, resultdir, label, user, client=False):
mblighf1c52842007-10-16 15:21:38 +0000137 """
138 control
139 The control file (pathname of)
140 args
141 args to pass to the control file
142 resultdir
143 where to throw the results
mbligh18420c22007-10-16 22:27:14 +0000144 label
145 label for the job
mblighf1c52842007-10-16 15:21:38 +0000146 user
147 Username for the job (email address)
148 client
149 True if a client-side control file
150 """
mbligh05269362007-10-16 16:58:11 +0000151 path = os.path.dirname(sys.modules['server_job'].__file__)
mblighf1c52842007-10-16 15:21:38 +0000152 self.autodir = os.path.abspath(os.path.join(path, '..'))
153 self.serverdir = os.path.join(self.autodir, 'server')
mbligh05269362007-10-16 16:58:11 +0000154 self.testdir = os.path.join(self.serverdir, 'tests')
155 self.tmpdir = os.path.join(self.serverdir, 'tmp')
mblighf1c52842007-10-16 15:21:38 +0000156 self.conmuxdir = os.path.join(self.autodir, 'conmux')
157 self.clientdir = os.path.join(self.autodir, 'client')
158 self.control = re.sub('\r\n', '\n', open(control, 'r').read())
159 self.resultdir = resultdir
160 if not os.path.exists(resultdir):
161 os.mkdir(resultdir)
mbligh3ccb8592007-11-05 18:13:40 +0000162 self.debugdir = os.path.join(resultdir, 'debug')
163 if not os.path.exists(self.debugdir):
164 os.mkdir(self.debugdir)
mbligh3dcf2c92007-10-16 22:24:00 +0000165 self.status = os.path.join(resultdir, 'status')
mbligh18420c22007-10-16 22:27:14 +0000166 self.label = label
mblighf1c52842007-10-16 15:21:38 +0000167 self.user = user
168 self.args = args
169 self.client = client
170 self.record_prefix = ''
171
mbligh3f4bced2007-11-05 17:55:53 +0000172 self.stdout = fd_stack.fd_stack(1, sys.stdout)
173 self.stderr = fd_stack.fd_stack(2, sys.stderr)
174
mbligh3dcf2c92007-10-16 22:24:00 +0000175 if os.path.exists(self.status):
176 os.unlink(self.status)
mbligh18420c22007-10-16 22:27:14 +0000177 job_data = { 'label' : label, 'user' : user}
mblighf1c52842007-10-16 15:21:38 +0000178 write_keyval(self.resultdir, job_data)
179
180
mblighf36243d2007-10-30 15:36:16 +0000181 def run(self, machines, reboot = False, install_before = False,
182 install_after = False, namespace = {}):
mbligh60dbd502007-10-26 14:59:31 +0000183 # use a copy so changes don't affect the original dictionary
184 namespace = namespace.copy()
185
mblighfaf0cd42007-11-19 16:00:24 +0000186 self.aborted = False
mblighf1c52842007-10-16 15:21:38 +0000187 namespace['machines'] = machines
188 namespace['args'] = self.args
189 namespace['job'] = self
mbligh6e294382007-11-05 18:11:29 +0000190 test_start_time = int(time.time())
mblighf1c52842007-10-16 15:21:38 +0000191
mbligh87c5d882007-10-29 17:07:24 +0000192 os.chdir(self.resultdir)
193
194 status_log = os.path.join(self.resultdir, 'status.log')
mblighf1c52842007-10-16 15:21:38 +0000195 try:
mblighf36243d2007-10-30 15:36:16 +0000196 if install_before and machines:
197 exec(preamble + install, namespace, namespace)
mblighf1c52842007-10-16 15:21:38 +0000198 if self.client:
199 namespace['control'] = self.control
200 open('control', 'w').write(self.control)
201 open('control.srv', 'w').write(client_wrapper)
202 server_control = client_wrapper
203 else:
204 open('control.srv', 'w').write(self.control)
205 server_control = self.control
mblighf1c52842007-10-16 15:21:38 +0000206 exec(preamble + server_control, namespace, namespace)
207
208 finally:
mbligh6e294382007-11-05 18:11:29 +0000209 if machines:
210 namespace['test_start_time'] = test_start_time
211 exec(preamble + crashdumps, namespace,
212 namespace)
mblighf1c52842007-10-16 15:21:38 +0000213 if reboot and machines:
214 exec(preamble + cleanup, namespace, namespace)
mblighf36243d2007-10-30 15:36:16 +0000215 if install_after and machines:
216 exec(preamble + install, namespace, namespace)
mblighf1c52842007-10-16 15:21:38 +0000217
218
219 def run_test(self, url, *args, **dargs):
220 """Summon a test object and run it.
221
222 tag
223 tag to add to testname
224 url
225 url of the test to run
226 """
227
mblighf1c52842007-10-16 15:21:38 +0000228 (group, testname) = test.testname(url)
229 tag = None
230 subdir = testname
mbligh43ac5222007-10-16 15:55:01 +0000231
mblighf1c52842007-10-16 15:21:38 +0000232 if dargs.has_key('tag'):
233 tag = dargs['tag']
234 del dargs['tag']
235 if tag:
236 subdir += '.' + tag
mblighf1c52842007-10-16 15:21:38 +0000237
mbligh43ac5222007-10-16 15:55:01 +0000238 try:
239 test.runtest(self, url, tag, args, dargs)
240 self.record('GOOD', subdir, testname, 'completed successfully')
241 except Exception, detail:
mbligh05269362007-10-16 16:58:11 +0000242 self.record('FAIL', subdir, testname, format_error())
mblighf1c52842007-10-16 15:21:38 +0000243
244
245 def run_group(self, function, *args, **dargs):
246 """\
247 function:
248 subroutine to run
249 *args:
250 arguments for the function
251 """
252
253 result = None
254 name = function.__name__
255
256 # Allow the tag for the group to be specified.
257 if dargs.has_key('tag'):
258 tag = dargs['tag']
259 del dargs['tag']
260 if tag:
261 name = tag
262
263 # if tag:
264 # name += '.' + tag
265 old_record_prefix = self.record_prefix
266 try:
267 try:
268 self.record('START', None, name)
269 self.record_prefix += '\t'
270 result = function(*args, **dargs)
271 self.record_prefix = old_record_prefix
272 self.record('END GOOD', None, name)
273 except:
274 self.record_prefix = old_record_prefix
275 self.record('END FAIL', None, name, format_error())
276 # We don't want to raise up an error higher if it's just
277 # a TestError - we want to carry on to other tests. Hence
278 # this outer try/except block.
279 except TestError:
280 pass
281 except:
282 raise TestError(name + ' failed\n' + format_error())
283
284 return result
285
286
287 def record(self, status_code, subdir, operation, status = ''):
288 """
289 Record job-level status
290
291 The intent is to make this file both machine parseable and
292 human readable. That involves a little more complexity, but
293 really isn't all that bad ;-)
294
295 Format is <status code>\t<subdir>\t<operation>\t<status>
296
297 status code: (GOOD|WARN|FAIL|ABORT)
298 or START
299 or END (GOOD|WARN|FAIL|ABORT)
300
301 subdir: MUST be a relevant subdirectory in the results,
302 or None, which will be represented as '----'
303
304 operation: description of what you ran (e.g. "dbench", or
305 "mkfs -t foobar /dev/sda9")
306
307 status: error message or "completed sucessfully"
308
309 ------------------------------------------------------------
310
311 Initial tabs indicate indent levels for grouping, and is
312 governed by self.record_prefix
313
314 multiline messages have secondary lines prefaced by a double
315 space (' ')
316 """
317
318 if subdir:
319 if re.match(r'[\n\t]', subdir):
320 raise "Invalid character in subdir string"
321 substr = subdir
322 else:
323 substr = '----'
324
325 if not re.match(r'(START|(END )?(GOOD|WARN|FAIL|ABORT))$', \
326 status_code):
327 raise "Invalid status code supplied: %s" % status_code
328 if re.match(r'[\n\t]', operation):
329 raise "Invalid character in operation string"
330 operation = operation.rstrip()
331 status = status.rstrip()
332 status = re.sub(r"\t", " ", status)
333 # Ensure any continuation lines are marked so we can
334 # detect them in the status file to ensure it is parsable.
335 status = re.sub(r"\n", "\n" + self.record_prefix + " ", status)
336
mbligh30270302007-11-05 20:33:52 +0000337 # Generate timestamps for inclusion in the logs
338 epoch_time = int(time.time()) # seconds since epoch, in UTC
339 local_time = time.localtime(epoch_time)
340 epoch_time_str = "timestamp=%d" % (epoch_time,)
341 local_time_str = time.strftime("localtime=%b %d %H:%M:%S",
342 local_time)
343
344 msg = '\t'.join(str(x) for x in (status_code, substr, operation,
345 epoch_time_str, local_time_str,
346 status))
mblighf1c52842007-10-16 15:21:38 +0000347
mbligh31a49de2007-11-05 18:41:19 +0000348 status_file = os.path.join(self.resultdir, 'status.log')
mblighf1c52842007-10-16 15:21:38 +0000349 print msg
350 open(status_file, "a").write(self.record_prefix + msg + "\n")
351 if subdir:
352 status_file = os.path.join(self.resultdir, subdir, 'status')
353 open(status_file, "a").write(msg + "\n")