blob: 1bca2f3147562b836492a047d42fa270b670a210 [file] [log] [blame]
mbligh67647152008-11-19 00:18:14 +00001# Copyright Martin J. Bligh, Google Inc 2008
2# Released under the GPL v2
3
4"""
5This class allows you to communicate with the frontend to submit jobs etc
6It is designed for writing more sophisiticated server-side control files that
7can recursively add and manage other jobs.
8
9We turn the JSON dictionaries into real objects that are more idiomatic
10
mblighc31e4022008-12-11 19:32:30 +000011For docs, see:
12 http://autotest/afe/server/noauth/rpc/
13 http://autotest/new_tko/server/noauth/rpc/
14 http://docs.djangoproject.com/en/dev/ref/models/querysets/#queryset-api
mbligh67647152008-11-19 00:18:14 +000015"""
16
mblighdb59e3c2009-11-21 01:45:18 +000017import getpass, os, time, traceback, re
mbligh67647152008-11-19 00:18:14 +000018import common
19from autotest_lib.frontend.afe import rpc_client_lib
mbligh37eceaa2008-12-15 22:56:37 +000020from autotest_lib.client.common_lib import global_config
mbligh67647152008-11-19 00:18:14 +000021from autotest_lib.client.common_lib import utils
mbligh4e576612008-12-22 14:56:36 +000022try:
23 from autotest_lib.server.site_common import site_utils as server_utils
24except:
25 from autotest_lib.server import utils as server_utils
26form_ntuples_from_machines = server_utils.form_ntuples_from_machines
mbligh67647152008-11-19 00:18:14 +000027
mbligh37eceaa2008-12-15 22:56:37 +000028GLOBAL_CONFIG = global_config.global_config
29DEFAULT_SERVER = 'autotest'
30
mbligh67647152008-11-19 00:18:14 +000031def dump_object(header, obj):
32 """
33 Standard way to print out the frontend objects (eg job, host, acl, label)
34 in a human-readable fashion for debugging
35 """
36 result = header + '\n'
37 for key in obj.hash:
38 if key == 'afe' or key == 'hash':
39 continue
40 result += '%20s: %s\n' % (key, obj.hash[key])
41 return result
42
43
mbligh5280e3b2008-12-22 14:39:28 +000044class RpcClient(object):
mbligh67647152008-11-19 00:18:14 +000045 """
mbligh451ede12009-02-12 21:54:03 +000046 Abstract RPC class for communicating with the autotest frontend
47 Inherited for both TKO and AFE uses.
mbligh67647152008-11-19 00:18:14 +000048
mbligh1ef218d2009-08-03 16:57:56 +000049 All the constructors go in the afe / tko class.
mbligh451ede12009-02-12 21:54:03 +000050 Manipulating methods go in the object classes themselves
mbligh67647152008-11-19 00:18:14 +000051 """
mbligh99b24f42009-06-08 16:45:55 +000052 def __init__(self, path, user, server, print_log, debug, reply_debug):
mbligh67647152008-11-19 00:18:14 +000053 """
mbligh451ede12009-02-12 21:54:03 +000054 Create a cached instance of a connection to the frontend
mbligh67647152008-11-19 00:18:14 +000055
56 user: username to connect as
mbligh451ede12009-02-12 21:54:03 +000057 server: frontend server to connect to
mbligh67647152008-11-19 00:18:14 +000058 print_log: pring a logging message to stdout on every operation
59 debug: print out all RPC traffic
60 """
mblighc31e4022008-12-11 19:32:30 +000061 if not user:
mblighdb59e3c2009-11-21 01:45:18 +000062 user = getpass.getuser()
mbligh451ede12009-02-12 21:54:03 +000063 if not server:
mbligh475f7762009-01-30 00:34:04 +000064 if 'AUTOTEST_WEB' in os.environ:
mbligh451ede12009-02-12 21:54:03 +000065 server = os.environ['AUTOTEST_WEB']
mbligh475f7762009-01-30 00:34:04 +000066 else:
mbligh451ede12009-02-12 21:54:03 +000067 server = GLOBAL_CONFIG.get_config_value('SERVER', 'hostname',
68 default=DEFAULT_SERVER)
69 self.server = server
mbligh67647152008-11-19 00:18:14 +000070 self.user = user
71 self.print_log = print_log
72 self.debug = debug
mbligh99b24f42009-06-08 16:45:55 +000073 self.reply_debug = reply_debug
mbligh67647152008-11-19 00:18:14 +000074 headers = {'AUTHORIZATION' : self.user}
mbligh451ede12009-02-12 21:54:03 +000075 rpc_server = 'http://' + server + path
mbligh1354c9d2008-12-22 14:56:13 +000076 if debug:
77 print 'SERVER: %s' % rpc_server
78 print 'HEADERS: %s' % headers
mbligh67647152008-11-19 00:18:14 +000079 self.proxy = rpc_client_lib.get_proxy(rpc_server, headers=headers)
80
81
82 def run(self, call, **dargs):
83 """
84 Make a RPC call to the AFE server
85 """
86 rpc_call = getattr(self.proxy, call)
87 if self.debug:
88 print 'DEBUG: %s %s' % (call, dargs)
mbligh451ede12009-02-12 21:54:03 +000089 try:
mbligh99b24f42009-06-08 16:45:55 +000090 result = utils.strip_unicode(rpc_call(**dargs))
91 if self.reply_debug:
92 print result
93 return result
mbligh451ede12009-02-12 21:54:03 +000094 except Exception:
95 print 'FAILED RPC CALL: %s %s' % (call, dargs)
96 raise
mbligh67647152008-11-19 00:18:14 +000097
98
99 def log(self, message):
100 if self.print_log:
101 print message
102
103
mbligh5280e3b2008-12-22 14:39:28 +0000104class TKO(RpcClient):
mbligh99b24f42009-06-08 16:45:55 +0000105 def __init__(self, user=None, server=None, print_log=True, debug=False,
106 reply_debug=False):
107 super(TKO, self).__init__(path='/new_tko/server/noauth/rpc/',
108 user=user,
109 server=server,
110 print_log=print_log,
111 debug=debug,
112 reply_debug=reply_debug)
mblighc31e4022008-12-11 19:32:30 +0000113
114
115 def get_status_counts(self, job, **data):
116 entries = self.run('get_status_counts',
mbligh1ef218d2009-08-03 16:57:56 +0000117 group_by=['hostname', 'test_name', 'reason'],
mblighc31e4022008-12-11 19:32:30 +0000118 job_tag__startswith='%s-' % job, **data)
mbligh5280e3b2008-12-22 14:39:28 +0000119 return [TestStatus(self, e) for e in entries['groups']]
mblighc31e4022008-12-11 19:32:30 +0000120
121
mbligh5280e3b2008-12-22 14:39:28 +0000122class AFE(RpcClient):
mbligh17c75e62009-06-08 16:18:21 +0000123 def __init__(self, user=None, server=None, print_log=True, debug=False,
mbligh99b24f42009-06-08 16:45:55 +0000124 reply_debug=False, job=None):
mbligh17c75e62009-06-08 16:18:21 +0000125 self.job = job
mbligh99b24f42009-06-08 16:45:55 +0000126 super(AFE, self).__init__(path='/afe/server/noauth/rpc/',
127 user=user,
128 server=server,
129 print_log=print_log,
130 debug=debug,
131 reply_debug=reply_debug)
mblighc31e4022008-12-11 19:32:30 +0000132
mbligh1ef218d2009-08-03 16:57:56 +0000133
mbligh67647152008-11-19 00:18:14 +0000134 def host_statuses(self, live=None):
mblighc2847b72009-03-25 19:32:20 +0000135 dead_statuses = ['Dead', 'Repair Failed', 'Repairing']
mbligh67647152008-11-19 00:18:14 +0000136 statuses = self.run('get_static_data')['host_statuses']
137 if live == True:
mblighc2847b72009-03-25 19:32:20 +0000138 return list(set(statuses) - set(dead_statuses))
mbligh67647152008-11-19 00:18:14 +0000139 if live == False:
140 return dead_statuses
141 else:
142 return statuses
143
144
mbligh71094012009-12-19 05:35:21 +0000145 @staticmethod
146 def _dict_for_host_query(hostnames=(), status=None, label=None):
147 query_args = {}
mbligh4e545a52009-12-19 05:30:39 +0000148 if hostnames:
149 query_args['hostname__in'] = hostnames
150 if status:
151 query_args['status'] = status
152 if label:
153 query_args['labels__name'] = label
mbligh71094012009-12-19 05:35:21 +0000154 return query_args
155
156
157 def get_hosts(self, hostnames=(), status=None, label=None, **dargs):
158 query_args = dict(dargs)
159 query_args.update(self._dict_for_host_query(hostnames=hostnames,
160 status=status,
161 label=label))
162 hosts = self.run('get_hosts', **query_args)
163 return [Host(self, h) for h in hosts]
164
165
166 def get_hostnames(self, status=None, label=None, **dargs):
167 """Like get_hosts() but returns hostnames instead of Host objects."""
168 # This implementation can be replaced with a more efficient one
169 # that does not query for entire host objects in the future.
170 return [host_obj.hostname for host_obj in
171 self.get_hosts(status=status, label=label, **dargs)]
172
173
174 def reverify_hosts(self, hostnames=(), status=None, label=None):
175 query_args = dict(locked=False,
176 aclgroup__users__login=self.user)
177 query_args.update(self._dict_for_host_query(hostnames=hostnames,
178 status=status,
179 label=label))
mbligh4e545a52009-12-19 05:30:39 +0000180 return self.run('reverify_hosts', **query_args)
181
182
mbligh67647152008-11-19 00:18:14 +0000183 def create_host(self, hostname, **dargs):
mbligh54459c72009-01-21 19:26:44 +0000184 id = self.run('add_host', hostname=hostname, **dargs)
mbligh67647152008-11-19 00:18:14 +0000185 return self.get_hosts(id=id)[0]
186
187
188 def get_labels(self, **dargs):
189 labels = self.run('get_labels', **dargs)
mbligh5280e3b2008-12-22 14:39:28 +0000190 return [Label(self, l) for l in labels]
mbligh67647152008-11-19 00:18:14 +0000191
192
193 def create_label(self, name, **dargs):
mbligh54459c72009-01-21 19:26:44 +0000194 id = self.run('add_label', name=name, **dargs)
mbligh67647152008-11-19 00:18:14 +0000195 return self.get_labels(id=id)[0]
196
197
198 def get_acls(self, **dargs):
199 acls = self.run('get_acl_groups', **dargs)
mbligh5280e3b2008-12-22 14:39:28 +0000200 return [Acl(self, a) for a in acls]
mbligh67647152008-11-19 00:18:14 +0000201
202
203 def create_acl(self, name, **dargs):
mbligh54459c72009-01-21 19:26:44 +0000204 id = self.run('add_acl_group', name=name, **dargs)
mbligh67647152008-11-19 00:18:14 +0000205 return self.get_acls(id=id)[0]
206
207
mbligh54459c72009-01-21 19:26:44 +0000208 def get_users(self, **dargs):
209 users = self.run('get_users', **dargs)
210 return [User(self, u) for u in users]
211
212
mbligh1354c9d2008-12-22 14:56:13 +0000213 def generate_control_file(self, tests, **dargs):
214 ret = self.run('generate_control_file', tests=tests, **dargs)
215 return ControlFile(self, ret)
216
217
mbligh67647152008-11-19 00:18:14 +0000218 def get_jobs(self, summary=False, **dargs):
219 if summary:
220 jobs_data = self.run('get_jobs_summary', **dargs)
221 else:
222 jobs_data = self.run('get_jobs', **dargs)
mblighafbba0c2009-06-08 16:44:45 +0000223 jobs = []
224 for j in jobs_data:
225 job = Job(self, j)
226 # Set up some extra information defaults
227 job.testname = re.sub('\s.*', '', job.name) # arbitrary default
228 job.platform_results = {}
229 job.platform_reasons = {}
230 jobs.append(job)
231 return jobs
mbligh67647152008-11-19 00:18:14 +0000232
233
234 def get_host_queue_entries(self, **data):
235 entries = self.run('get_host_queue_entries', **data)
mblighf9e35862009-02-26 01:03:11 +0000236 job_statuses = [JobStatus(self, e) for e in entries]
mbligh99b24f42009-06-08 16:45:55 +0000237
238 # Sadly, get_host_queue_entries doesn't return platforms, we have
239 # to get those back from an explicit get_hosts queury, then patch
240 # the new host objects back into the host list.
241 hostnames = [s.host.hostname for s in job_statuses if s.host]
242 host_hash = {}
243 for host in self.get_hosts(hostname__in=hostnames):
244 host_hash[host.hostname] = host
245 for status in job_statuses:
246 if status.host:
247 status.host = host_hash[status.host.hostname]
mblighf9e35862009-02-26 01:03:11 +0000248 # filter job statuses that have either host or meta_host
249 return [status for status in job_statuses if (status.host or
250 status.meta_host)]
mbligh67647152008-11-19 00:18:14 +0000251
252
mblighb9db5162009-04-17 22:21:41 +0000253 def create_job_by_test(self, tests, kernel=None, use_container=False,
mbligh1354c9d2008-12-22 14:56:13 +0000254 **dargs):
mbligh67647152008-11-19 00:18:14 +0000255 """
256 Given a test name, fetch the appropriate control file from the server
mbligh4e576612008-12-22 14:56:36 +0000257 and submit it.
258
259 Returns a list of job objects
mbligh67647152008-11-19 00:18:14 +0000260 """
mblighb9db5162009-04-17 22:21:41 +0000261 assert ('hosts' in dargs or
262 'atomic_group_name' in dargs and 'synch_count' in dargs)
showarda2cd72b2009-10-01 18:43:53 +0000263 if kernel:
264 kernel_list = re.split('[\s,]+', kernel.strip())
265 kernel_info = [{'version': version} for version in kernel_list]
266 else:
267 kernel_info = None
268 control_file = self.generate_control_file(
269 tests=tests, kernel=kernel_info, use_container=use_container,
270 do_push_packages=True)
mbligh1354c9d2008-12-22 14:56:13 +0000271 if control_file.is_server:
mbligh67647152008-11-19 00:18:14 +0000272 dargs['control_type'] = 'Server'
273 else:
274 dargs['control_type'] = 'Client'
275 dargs['dependencies'] = dargs.get('dependencies', []) + \
mbligh1354c9d2008-12-22 14:56:13 +0000276 control_file.dependencies
277 dargs['control_file'] = control_file.control_file
mbligh672666c2009-07-28 23:22:13 +0000278 if not dargs.get('synch_count', None):
mblighc99fccf2009-07-11 00:59:33 +0000279 dargs['synch_count'] = control_file.synch_count
mblighb9db5162009-04-17 22:21:41 +0000280 if 'hosts' in dargs and len(dargs['hosts']) < dargs['synch_count']:
281 # will not be able to satisfy this request
mbligh38b09152009-04-28 18:34:25 +0000282 return None
283 return self.create_job(**dargs)
mbligh67647152008-11-19 00:18:14 +0000284
285
286 def create_job(self, control_file, name=' ', priority='Medium',
287 control_type='Client', **dargs):
288 id = self.run('create_job', name=name, priority=priority,
289 control_file=control_file, control_type=control_type, **dargs)
290 return self.get_jobs(id=id)[0]
291
292
mbligh1f23f362008-12-22 14:46:12 +0000293 def run_test_suites(self, pairings, kernel, kernel_label, priority='Medium',
mblighd50b1252009-06-08 16:43:37 +0000294 wait=True, poll_interval=10, email_from=None,
mbligh7b312282009-01-07 16:45:43 +0000295 email_to=None, timeout=168):
mbligh5b618382008-12-03 15:24:01 +0000296 """
297 Run a list of test suites on a particular kernel.
mbligh1ef218d2009-08-03 16:57:56 +0000298
mbligh5b618382008-12-03 15:24:01 +0000299 Poll for them to complete, and return whether they worked or not.
mbligh1ef218d2009-08-03 16:57:56 +0000300
mbligh5b618382008-12-03 15:24:01 +0000301 pairings: list of MachineTestPairing objects to invoke
302 kernel: name of the kernel to run
303 kernel_label: label of the kernel to run
304 (<kernel-version> : <config> : <date>)
305 wait: boolean - wait for the results to come back?
306 poll_interval: interval between polling for job results (in minutes)
mbligh45ffc432008-12-09 23:35:17 +0000307 email_from: send notification email upon completion from here
308 email_from: send notification email upon completion to here
mbligh5b618382008-12-03 15:24:01 +0000309 """
310 jobs = []
311 for pairing in pairings:
mbligh0c4f8d72009-05-12 20:52:18 +0000312 try:
313 new_job = self.invoke_test(pairing, kernel, kernel_label,
314 priority, timeout=timeout)
315 if not new_job:
316 continue
mbligh0c4f8d72009-05-12 20:52:18 +0000317 jobs.append(new_job)
318 except Exception, e:
319 traceback.print_exc()
mblighb9db5162009-04-17 22:21:41 +0000320 if not wait or not jobs:
mbligh5b618382008-12-03 15:24:01 +0000321 return
mbligh5280e3b2008-12-22 14:39:28 +0000322 tko = TKO()
mbligh5b618382008-12-03 15:24:01 +0000323 while True:
324 time.sleep(60 * poll_interval)
mbligh5280e3b2008-12-22 14:39:28 +0000325 result = self.poll_all_jobs(tko, jobs, email_from, email_to)
mbligh5b618382008-12-03 15:24:01 +0000326 if result is not None:
327 return result
328
329
mbligh45ffc432008-12-09 23:35:17 +0000330 def result_notify(self, job, email_from, email_to):
mbligh5b618382008-12-03 15:24:01 +0000331 """
mbligh45ffc432008-12-09 23:35:17 +0000332 Notify about the result of a job. Will always print, if email data
333 is provided, will send email for it as well.
334
335 job: job object to notify about
336 email_from: send notification email upon completion from here
337 email_from: send notification email upon completion to here
338 """
339 if job.result == True:
340 subject = 'Testing PASSED: '
341 else:
342 subject = 'Testing FAILED: '
343 subject += '%s : %s\n' % (job.name, job.id)
344 text = []
345 for platform in job.results_platform_map:
346 for status in job.results_platform_map[platform]:
347 if status == 'Total':
348 continue
mbligh451ede12009-02-12 21:54:03 +0000349 for host in job.results_platform_map[platform][status]:
350 text.append('%20s %10s %10s' % (platform, status, host))
351 if status == 'Failed':
352 for test_status in job.test_status[host].fail:
353 text.append('(%s, %s) : %s' % \
354 (host, test_status.test_name,
355 test_status.reason))
356 text.append('')
mbligh37eceaa2008-12-15 22:56:37 +0000357
mbligh451ede12009-02-12 21:54:03 +0000358 base_url = 'http://' + self.server
mbligh37eceaa2008-12-15 22:56:37 +0000359
360 params = ('columns=test',
361 'rows=machine_group',
362 "condition=tag~'%s-%%25'" % job.id,
363 'title=Report')
364 query_string = '&'.join(params)
mbligh451ede12009-02-12 21:54:03 +0000365 url = '%s/tko/compose_query.cgi?%s' % (base_url, query_string)
366 text.append(url + '\n')
367 url = '%s/afe/#tab_id=view_job&object_id=%s' % (base_url, job.id)
368 text.append(url + '\n')
mbligh37eceaa2008-12-15 22:56:37 +0000369
370 body = '\n'.join(text)
371 print '---------------------------------------------------'
372 print 'Subject: ', subject
mbligh45ffc432008-12-09 23:35:17 +0000373 print body
mbligh37eceaa2008-12-15 22:56:37 +0000374 print '---------------------------------------------------'
mbligh45ffc432008-12-09 23:35:17 +0000375 if email_from and email_to:
mbligh37eceaa2008-12-15 22:56:37 +0000376 print 'Sending email ...'
mbligh45ffc432008-12-09 23:35:17 +0000377 utils.send_email(email_from, email_to, subject, body)
378 print
mbligh37eceaa2008-12-15 22:56:37 +0000379
mbligh45ffc432008-12-09 23:35:17 +0000380
mbligh1354c9d2008-12-22 14:56:13 +0000381 def print_job_result(self, job):
382 """
383 Print the result of a single job.
384 job: a job object
385 """
386 if job.result is None:
387 print 'PENDING',
388 elif job.result == True:
389 print 'PASSED',
390 elif job.result == False:
391 print 'FAILED',
mbligh912c3f32009-03-25 19:31:30 +0000392 elif job.result == "Abort":
393 print 'ABORT',
mbligh1354c9d2008-12-22 14:56:13 +0000394 print ' %s : %s' % (job.id, job.name)
395
396
mbligh451ede12009-02-12 21:54:03 +0000397 def poll_all_jobs(self, tko, jobs, email_from=None, email_to=None):
mbligh45ffc432008-12-09 23:35:17 +0000398 """
399 Poll all jobs in a list.
400 jobs: list of job objects to poll
401 email_from: send notification email upon completion from here
402 email_from: send notification email upon completion to here
403
404 Returns:
mbligh5b618382008-12-03 15:24:01 +0000405 a) All complete successfully (return True)
406 b) One or more has failed (return False)
407 c) Cannot tell yet (return None)
408 """
mbligh45ffc432008-12-09 23:35:17 +0000409 results = []
mbligh5b618382008-12-03 15:24:01 +0000410 for job in jobs:
mbligh676dcbe2009-06-15 21:57:27 +0000411 if getattr(job, 'result', None) is None:
412 job.result = self.poll_job_results(tko, job)
413 if job.result is not None:
414 self.result_notify(job, email_from, email_to)
mbligh45ffc432008-12-09 23:35:17 +0000415
mbligh676dcbe2009-06-15 21:57:27 +0000416 results.append(job.result)
mbligh1354c9d2008-12-22 14:56:13 +0000417 self.print_job_result(job)
mbligh45ffc432008-12-09 23:35:17 +0000418
419 if None in results:
420 return None
mbligh912c3f32009-03-25 19:31:30 +0000421 elif False in results or "Abort" in results:
mbligh45ffc432008-12-09 23:35:17 +0000422 return False
423 else:
424 return True
mbligh5b618382008-12-03 15:24:01 +0000425
426
mbligh1f23f362008-12-22 14:46:12 +0000427 def _included_platform(self, host, platforms):
428 """
429 See if host's platforms matches any of the patterns in the included
430 platforms list.
431 """
432 if not platforms:
433 return True # No filtering of platforms
434 for platform in platforms:
435 if re.search(platform, host.platform):
436 return True
437 return False
438
439
mbligh7b312282009-01-07 16:45:43 +0000440 def invoke_test(self, pairing, kernel, kernel_label, priority='Medium',
441 **dargs):
mbligh5b618382008-12-03 15:24:01 +0000442 """
443 Given a pairing of a control file to a machine label, find all machines
444 with that label, and submit that control file to them.
mbligh1ef218d2009-08-03 16:57:56 +0000445
mbligh4e576612008-12-22 14:56:36 +0000446 Returns a list of job objects
mbligh5b618382008-12-03 15:24:01 +0000447 """
448 job_name = '%s : %s' % (pairing.machine_label, kernel_label)
449 hosts = self.get_hosts(multiple_labels=[pairing.machine_label])
mbligh1f23f362008-12-22 14:46:12 +0000450 platforms = pairing.platforms
451 hosts = [h for h in hosts if self._included_platform(h, platforms)]
mblighc2847b72009-03-25 19:32:20 +0000452 dead_statuses = self.host_statuses(live=False)
453 host_list = [h.hostname for h in hosts if h.status not in dead_statuses]
mbligh1f23f362008-12-22 14:46:12 +0000454 print 'HOSTS: %s' % host_list
mblighb9db5162009-04-17 22:21:41 +0000455 if pairing.atomic_group_sched:
mblighc99fccf2009-07-11 00:59:33 +0000456 dargs['synch_count'] = pairing.synch_count
mblighb9db5162009-04-17 22:21:41 +0000457 dargs['atomic_group_name'] = pairing.machine_label
458 else:
459 dargs['hosts'] = host_list
mbligh38b09152009-04-28 18:34:25 +0000460 new_job = self.create_job_by_test(name=job_name,
mbligh17c75e62009-06-08 16:18:21 +0000461 dependencies=[pairing.machine_label],
462 tests=[pairing.control_file],
463 priority=priority,
464 kernel=kernel,
465 use_container=pairing.container,
466 **dargs)
mbligh38b09152009-04-28 18:34:25 +0000467 if new_job:
mbligh17c75e62009-06-08 16:18:21 +0000468 if pairing.testname:
469 new_job.testname = pairing.testname
mbligh4e576612008-12-22 14:56:36 +0000470 print 'Invoked test %s : %s' % (new_job.id, job_name)
mbligh38b09152009-04-28 18:34:25 +0000471 return new_job
mbligh5b618382008-12-03 15:24:01 +0000472
473
mblighb9db5162009-04-17 22:21:41 +0000474 def _job_test_results(self, tko, job, debug, tests=[]):
mbligh5b618382008-12-03 15:24:01 +0000475 """
mbligh5280e3b2008-12-22 14:39:28 +0000476 Retrieve test results for a job
mbligh5b618382008-12-03 15:24:01 +0000477 """
mbligh5280e3b2008-12-22 14:39:28 +0000478 job.test_status = {}
479 try:
480 test_statuses = tko.get_status_counts(job=job.id)
481 except Exception:
482 print "Ignoring exception on poll job; RPC interface is flaky"
483 traceback.print_exc()
484 return
485
486 for test_status in test_statuses:
mbligh7479a182009-01-07 16:46:24 +0000487 # SERVER_JOB is buggy, and often gives false failures. Ignore it.
488 if test_status.test_name == 'SERVER_JOB':
489 continue
mblighb9db5162009-04-17 22:21:41 +0000490 # if tests is not empty, restrict list of test_statuses to tests
491 if tests and test_status.test_name not in tests:
492 continue
mbligh451ede12009-02-12 21:54:03 +0000493 if debug:
494 print test_status
mbligh5280e3b2008-12-22 14:39:28 +0000495 hostname = test_status.hostname
496 if hostname not in job.test_status:
497 job.test_status[hostname] = TestResults()
498 job.test_status[hostname].add(test_status)
499
500
mbligh451ede12009-02-12 21:54:03 +0000501 def _job_results_platform_map(self, job, debug):
mblighc9e427e2009-04-28 18:35:06 +0000502 # Figure out which hosts passed / failed / aborted in a job
503 # Creates a 2-dimensional hash, stored as job.results_platform_map
504 # 1st index - platform type (string)
505 # 2nd index - Status (string)
506 # 'Completed' / 'Failed' / 'Aborted'
507 # Data indexed by this hash is a list of hostnames (text strings)
mbligh5280e3b2008-12-22 14:39:28 +0000508 job.results_platform_map = {}
mbligh5b618382008-12-03 15:24:01 +0000509 try:
mbligh45ffc432008-12-09 23:35:17 +0000510 job_statuses = self.get_host_queue_entries(job=job.id)
mbligh5b618382008-12-03 15:24:01 +0000511 except Exception:
512 print "Ignoring exception on poll job; RPC interface is flaky"
513 traceback.print_exc()
514 return None
mbligh5280e3b2008-12-22 14:39:28 +0000515
mbligh5b618382008-12-03 15:24:01 +0000516 platform_map = {}
mbligh5280e3b2008-12-22 14:39:28 +0000517 job.job_status = {}
mbligh451ede12009-02-12 21:54:03 +0000518 job.metahost_index = {}
mbligh5b618382008-12-03 15:24:01 +0000519 for job_status in job_statuses:
mblighc9e427e2009-04-28 18:35:06 +0000520 # This is basically "for each host / metahost in the job"
mbligh451ede12009-02-12 21:54:03 +0000521 if job_status.host:
522 hostname = job_status.host.hostname
523 else: # This is a metahost
524 metahost = job_status.meta_host
525 index = job.metahost_index.get(metahost, 1)
526 job.metahost_index[metahost] = index + 1
527 hostname = '%s.%s' % (metahost, index)
mbligh5280e3b2008-12-22 14:39:28 +0000528 job.job_status[hostname] = job_status.status
mbligh5b618382008-12-03 15:24:01 +0000529 status = job_status.status
mbligh0ecbe632009-05-13 21:34:56 +0000530 # Skip hosts that failed verify or repair:
531 # that's a machine failure, not a job failure
mbligh451ede12009-02-12 21:54:03 +0000532 if hostname in job.test_status:
533 verify_failed = False
534 for failure in job.test_status[hostname].fail:
mbligh0ecbe632009-05-13 21:34:56 +0000535 if (failure.test_name == 'verify' or
536 failure.test_name == 'repair'):
mbligh451ede12009-02-12 21:54:03 +0000537 verify_failed = True
538 break
539 if verify_failed:
540 continue
mblighc9e427e2009-04-28 18:35:06 +0000541 if hostname in job.test_status and job.test_status[hostname].fail:
542 # If the any tests failed in the job, we want to mark the
543 # job result as failed, overriding the default job status.
544 if status != "Aborted": # except if it's an aborted job
545 status = 'Failed'
mbligh451ede12009-02-12 21:54:03 +0000546 if job_status.host:
547 platform = job_status.host.platform
548 else: # This is a metahost
549 platform = job_status.meta_host
mbligh5b618382008-12-03 15:24:01 +0000550 if platform not in platform_map:
551 platform_map[platform] = {'Total' : [hostname]}
552 else:
553 platform_map[platform]['Total'].append(hostname)
554 new_host_list = platform_map[platform].get(status, []) + [hostname]
555 platform_map[platform][status] = new_host_list
mbligh45ffc432008-12-09 23:35:17 +0000556 job.results_platform_map = platform_map
mbligh5280e3b2008-12-22 14:39:28 +0000557
558
mbligh17c75e62009-06-08 16:18:21 +0000559 def set_platform_results(self, test_job, platform, result):
560 """
561 Result must be None, 'FAIL', 'WARN' or 'GOOD'
562 """
563 if test_job.platform_results[platform] is not None:
564 # We're already done, and results recorded. This can't change later.
565 return
566 test_job.platform_results[platform] = result
567 # Note that self.job refers to the metajob we're IN, not the job
568 # that we're excuting from here.
569 testname = '%s.%s' % (test_job.testname, platform)
570 if self.job:
571 self.job.record(result, None, testname, status='')
572
573
mbligh5280e3b2008-12-22 14:39:28 +0000574 def poll_job_results(self, tko, job, debug=False):
575 """
576 Analyse all job results by platform, return:
mbligh1ef218d2009-08-03 16:57:56 +0000577
mbligh5280e3b2008-12-22 14:39:28 +0000578 False: if any platform has more than one failure
579 None: if any platform has more than one machine not yet Good.
580 True: if all platforms have at least all-but-one machines Good.
581 """
mbligh451ede12009-02-12 21:54:03 +0000582 self._job_test_results(tko, job, debug)
mblighe7fcf562009-05-21 01:43:17 +0000583 if job.test_status == {}:
584 return None
mbligh451ede12009-02-12 21:54:03 +0000585 self._job_results_platform_map(job, debug)
mbligh5280e3b2008-12-22 14:39:28 +0000586
mbligh5b618382008-12-03 15:24:01 +0000587 good_platforms = []
mbligh912c3f32009-03-25 19:31:30 +0000588 failed_platforms = []
589 aborted_platforms = []
mbligh5b618382008-12-03 15:24:01 +0000590 unknown_platforms = []
mbligh5280e3b2008-12-22 14:39:28 +0000591 platform_map = job.results_platform_map
mbligh5b618382008-12-03 15:24:01 +0000592 for platform in platform_map:
mbligh17c75e62009-06-08 16:18:21 +0000593 if not job.platform_results.has_key(platform):
594 # record test start, but there's no way to do this right now
595 job.platform_results[platform] = None
mbligh5b618382008-12-03 15:24:01 +0000596 total = len(platform_map[platform]['Total'])
597 completed = len(platform_map[platform].get('Completed', []))
mbligh912c3f32009-03-25 19:31:30 +0000598 failed = len(platform_map[platform].get('Failed', []))
599 aborted = len(platform_map[platform].get('Aborted', []))
mbligh17c75e62009-06-08 16:18:21 +0000600
mbligh1ef218d2009-08-03 16:57:56 +0000601 # We set up what we want to record here, but don't actually do
mbligh17c75e62009-06-08 16:18:21 +0000602 # it yet, until we have a decisive answer for this platform
603 if aborted or failed:
604 bad = aborted + failed
605 if (bad > 1) or (bad * 2 >= total):
606 platform_test_result = 'FAIL'
607 else:
608 platform_test_result = 'WARN'
609
mbligh912c3f32009-03-25 19:31:30 +0000610 if aborted > 1:
611 aborted_platforms.append(platform)
mbligh17c75e62009-06-08 16:18:21 +0000612 self.set_platform_results(job, platform, platform_test_result)
mbligh912c3f32009-03-25 19:31:30 +0000613 elif (failed * 2 >= total) or (failed > 1):
614 failed_platforms.append(platform)
mbligh17c75e62009-06-08 16:18:21 +0000615 self.set_platform_results(job, platform, platform_test_result)
mbligh451ede12009-02-12 21:54:03 +0000616 elif (completed >= 1) and (completed + 1 >= total):
mbligh5b618382008-12-03 15:24:01 +0000617 # if all or all but one are good, call the job good.
618 good_platforms.append(platform)
mbligh17c75e62009-06-08 16:18:21 +0000619 self.set_platform_results(job, platform, 'GOOD')
mbligh5b618382008-12-03 15:24:01 +0000620 else:
621 unknown_platforms.append(platform)
622 detail = []
623 for status in platform_map[platform]:
624 if status == 'Total':
625 continue
626 detail.append('%s=%s' % (status,platform_map[platform][status]))
627 if debug:
mbligh1ef218d2009-08-03 16:57:56 +0000628 print '%20s %d/%d %s' % (platform, completed, total,
mbligh5b618382008-12-03 15:24:01 +0000629 ' '.join(detail))
630 print
mbligh1ef218d2009-08-03 16:57:56 +0000631
mbligh912c3f32009-03-25 19:31:30 +0000632 if len(aborted_platforms) > 0:
mbligh5b618382008-12-03 15:24:01 +0000633 if debug:
mbligh17c75e62009-06-08 16:18:21 +0000634 print 'Result aborted - platforms: ',
635 print ' '.join(aborted_platforms)
mbligh912c3f32009-03-25 19:31:30 +0000636 return "Abort"
637 if len(failed_platforms) > 0:
638 if debug:
639 print 'Result bad - platforms: ' + ' '.join(failed_platforms)
mbligh5b618382008-12-03 15:24:01 +0000640 return False
641 if len(unknown_platforms) > 0:
642 if debug:
643 platform_list = ' '.join(unknown_platforms)
644 print 'Result unknown - platforms: ', platform_list
645 return None
646 if debug:
647 platform_list = ' '.join(good_platforms)
648 print 'Result good - all platforms passed: ', platform_list
649 return True
650
651
mbligh5280e3b2008-12-22 14:39:28 +0000652class TestResults(object):
653 """
654 Container class used to hold the results of the tests for a job
655 """
656 def __init__(self):
657 self.good = []
658 self.fail = []
mbligh451ede12009-02-12 21:54:03 +0000659 self.pending = []
mbligh5280e3b2008-12-22 14:39:28 +0000660
661
662 def add(self, result):
mbligh451ede12009-02-12 21:54:03 +0000663 if result.complete_count > result.pass_count:
664 self.fail.append(result)
665 elif result.incomplete_count > 0:
666 self.pending.append(result)
mbligh5280e3b2008-12-22 14:39:28 +0000667 else:
mbligh451ede12009-02-12 21:54:03 +0000668 self.good.append(result)
mbligh5280e3b2008-12-22 14:39:28 +0000669
670
671class RpcObject(object):
mbligh67647152008-11-19 00:18:14 +0000672 """
673 Generic object used to construct python objects from rpc calls
674 """
675 def __init__(self, afe, hash):
676 self.afe = afe
677 self.hash = hash
678 self.__dict__.update(hash)
679
680
681 def __str__(self):
682 return dump_object(self.__repr__(), self)
683
684
mbligh1354c9d2008-12-22 14:56:13 +0000685class ControlFile(RpcObject):
686 """
687 AFE control file object
688
689 Fields: synch_count, dependencies, control_file, is_server
690 """
691 def __repr__(self):
692 return 'CONTROL FILE: %s' % self.control_file
693
694
mbligh5280e3b2008-12-22 14:39:28 +0000695class Label(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000696 """
697 AFE label object
698
699 Fields:
700 name, invalid, platform, kernel_config, id, only_if_needed
701 """
702 def __repr__(self):
703 return 'LABEL: %s' % self.name
704
705
706 def add_hosts(self, hosts):
707 return self.afe.run('label_add_hosts', self.id, hosts)
708
709
710 def remove_hosts(self, hosts):
711 return self.afe.run('label_remove_hosts', self.id, hosts)
712
713
mbligh5280e3b2008-12-22 14:39:28 +0000714class Acl(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000715 """
716 AFE acl object
717
718 Fields:
719 users, hosts, description, name, id
720 """
721 def __repr__(self):
722 return 'ACL: %s' % self.name
723
724
725 def add_hosts(self, hosts):
726 self.afe.log('Adding hosts %s to ACL %s' % (hosts, self.name))
727 return self.afe.run('acl_group_add_hosts', self.id, hosts)
728
729
730 def remove_hosts(self, hosts):
731 self.afe.log('Removing hosts %s from ACL %s' % (hosts, self.name))
732 return self.afe.run('acl_group_remove_hosts', self.id, hosts)
733
734
mbligh54459c72009-01-21 19:26:44 +0000735 def add_users(self, users):
736 self.afe.log('Adding users %s to ACL %s' % (users, self.name))
737 return self.afe.run('acl_group_add_users', id=self.name, users=users)
738
739
mbligh5280e3b2008-12-22 14:39:28 +0000740class Job(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000741 """
742 AFE job object
743
744 Fields:
745 name, control_file, control_type, synch_count, reboot_before,
746 run_verify, priority, email_list, created_on, dependencies,
747 timeout, owner, reboot_after, id
748 """
749 def __repr__(self):
750 return 'JOB: %s' % self.id
751
752
mbligh5280e3b2008-12-22 14:39:28 +0000753class JobStatus(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000754 """
755 AFE job_status object
756
757 Fields:
758 status, complete, deleted, meta_host, host, active, execution_subdir, id
759 """
760 def __init__(self, afe, hash):
761 # This should call super
762 self.afe = afe
763 self.hash = hash
764 self.__dict__.update(hash)
mbligh5280e3b2008-12-22 14:39:28 +0000765 self.job = Job(afe, self.job)
mbligh67647152008-11-19 00:18:14 +0000766 if self.host:
mbligh99b24f42009-06-08 16:45:55 +0000767 self.host = Host(afe, self.host)
mbligh67647152008-11-19 00:18:14 +0000768
769
770 def __repr__(self):
mbligh451ede12009-02-12 21:54:03 +0000771 if self.host and self.host.hostname:
772 hostname = self.host.hostname
773 else:
774 hostname = 'None'
775 return 'JOB STATUS: %s-%s' % (self.job.id, hostname)
mbligh67647152008-11-19 00:18:14 +0000776
777
mbligh5280e3b2008-12-22 14:39:28 +0000778class Host(RpcObject):
mbligh67647152008-11-19 00:18:14 +0000779 """
780 AFE host object
781
782 Fields:
783 status, lock_time, locked_by, locked, hostname, invalid,
784 synch_id, labels, platform, protection, dirty, id
785 """
786 def __repr__(self):
787 return 'HOST OBJECT: %s' % self.hostname
788
789
790 def show(self):
791 labels = list(set(self.labels) - set([self.platform]))
792 print '%-6s %-7s %-7s %-16s %s' % (self.hostname, self.status,
793 self.locked, self.platform,
794 ', '.join(labels))
795
796
mbligh54459c72009-01-21 19:26:44 +0000797 def delete(self):
798 return self.afe.run('delete_host', id=self.id)
799
800
mbligh6463c4b2009-01-30 00:33:37 +0000801 def modify(self, **dargs):
802 return self.afe.run('modify_host', id=self.id, **dargs)
803
804
mbligh67647152008-11-19 00:18:14 +0000805 def get_acls(self):
806 return self.afe.get_acls(hosts__hostname=self.hostname)
807
808
809 def add_acl(self, acl_name):
810 self.afe.log('Adding ACL %s to host %s' % (acl_name, self.hostname))
811 return self.afe.run('acl_group_add_hosts', id=acl_name,
812 hosts=[self.hostname])
813
814
815 def remove_acl(self, acl_name):
816 self.afe.log('Removing ACL %s from host %s' % (acl_name, self.hostname))
817 return self.afe.run('acl_group_remove_hosts', id=acl_name,
818 hosts=[self.hostname])
819
820
821 def get_labels(self):
822 return self.afe.get_labels(host__hostname__in=[self.hostname])
823
824
825 def add_labels(self, labels):
826 self.afe.log('Adding labels %s to host %s' % (labels, self.hostname))
827 return self.afe.run('host_add_labels', id=self.id, labels=labels)
828
829
830 def remove_labels(self, labels):
831 self.afe.log('Removing labels %s from host %s' % (labels,self.hostname))
832 return self.afe.run('host_remove_labels', id=self.id, labels=labels)
mbligh5b618382008-12-03 15:24:01 +0000833
834
mbligh54459c72009-01-21 19:26:44 +0000835class User(RpcObject):
836 def __repr__(self):
837 return 'USER: %s' % self.login
838
839
mbligh5280e3b2008-12-22 14:39:28 +0000840class TestStatus(RpcObject):
mblighc31e4022008-12-11 19:32:30 +0000841 """
842 TKO test status object
843
844 Fields:
845 test_idx, hostname, testname, id
846 complete_count, incomplete_count, group_count, pass_count
847 """
848 def __repr__(self):
849 return 'TEST STATUS: %s' % self.id
850
851
mbligh5b618382008-12-03 15:24:01 +0000852class MachineTestPairing(object):
853 """
854 Object representing the pairing of a machine label with a control file
mbligh1f23f362008-12-22 14:46:12 +0000855
856 machine_label: use machines from this label
857 control_file: use this control file (by name in the frontend)
858 platforms: list of rexeps to filter platforms by. [] => no filtering
mbligh5b618382008-12-03 15:24:01 +0000859 """
mbligh1354c9d2008-12-22 14:56:13 +0000860 def __init__(self, machine_label, control_file, platforms=[],
mbligh17c75e62009-06-08 16:18:21 +0000861 container=False, atomic_group_sched=False, synch_count=0,
862 testname=None):
mbligh5b618382008-12-03 15:24:01 +0000863 self.machine_label = machine_label
864 self.control_file = control_file
mbligh1f23f362008-12-22 14:46:12 +0000865 self.platforms = platforms
mbligh1354c9d2008-12-22 14:56:13 +0000866 self.container = container
mblighb9db5162009-04-17 22:21:41 +0000867 self.atomic_group_sched = atomic_group_sched
868 self.synch_count = synch_count
mbligh17c75e62009-06-08 16:18:21 +0000869 self.testname = testname
mbligh1354c9d2008-12-22 14:56:13 +0000870
871
872 def __repr__(self):
873 return '%s %s %s %s' % (self.machine_label, self.control_file,
874 self.platforms, self.container)