blob: e13b5f593c3f6e96ad3cbf5bf648e927a4037ec2 [file] [log] [blame]
Matthew Hartfb6fd362020-03-04 21:03:59 +00001#!/usr/bin/env python3
2
3from __future__ import print_function
4
5__copyright__ = """
6/*
Xinyu Zhang78c146a2022-09-05 19:06:40 +08007 * Copyright (c) 2020-2022, Arm Limited. All rights reserved.
Matthew Hartfb6fd362020-03-04 21:03:59 +00008 *
9 * SPDX-License-Identifier: BSD-3-Clause
10 *
11 */
12 """
13
14"""
15Script for waiting for LAVA jobs and parsing the results
16"""
17
18import os
Matthew Hartfb6fd362020-03-04 21:03:59 +000019import time
20import yaml
21import argparse
Xinyu Zhangc8a670c2021-05-18 20:20:53 +080022import shutil
Paul Sokolovskybf0f5492022-12-18 01:10:18 +030023import traceback
Paul Sokolovskya95abd92022-12-27 13:48:11 +030024import logging
Matthew Hartfb6fd362020-03-04 21:03:59 +000025from jinja2 import Environment, FileSystemLoader
Matthew Hartfb6fd362020-03-04 21:03:59 +000026from lava_helper import test_lava_dispatch_credentials
Xinyu Zhangc918b6e2022-10-08 17:13:17 +080027from lava_submit_jobs import submit_lava_jobs
Paul Sokolovsky2512ec52022-03-04 00:15:39 +030028import codecov_helper
29
Matthew Hartfb6fd362020-03-04 21:03:59 +000030
Paul Sokolovskya95abd92022-12-27 13:48:11 +030031_log = logging.getLogger("lavaci")
32
33
Matthew Hartfb6fd362020-03-04 21:03:59 +000034def wait_for_jobs(user_args):
35 job_list = user_args.job_ids.split(",")
36 job_list = [int(x) for x in job_list if x != '']
37 lava = test_lava_dispatch_credentials(user_args)
Xinyu Zhangf2b7cbf2021-05-18 20:17:34 +080038 finished_jobs = get_finished_jobs(job_list, user_args, lava)
Xinyu Zhangc8a670c2021-05-18 20:20:53 +080039 resubmit_jobs = resubmit_failed_jobs(finished_jobs, user_args)
Paul Sokolovskyc87beee2022-04-30 08:50:47 +030040 if resubmit_jobs:
41 info_print("Waiting for resubmitted jobs: {}".format(resubmit_jobs))
42 finished_resubmit_jobs = get_finished_jobs(resubmit_jobs, user_args, lava)
43 finished_jobs.update(finished_resubmit_jobs)
Paul Sokolovsky451f67b2022-03-08 19:44:41 +030044 return finished_jobs
45
Paul Sokolovsky451f67b2022-03-08 19:44:41 +030046def process_finished_jobs(finished_jobs, user_args):
Xinyu Zhangf2b7cbf2021-05-18 20:17:34 +080047 print_lava_urls(finished_jobs, user_args)
Paul Sokolovsky451f67b2022-03-08 19:44:41 +030048 test_report(finished_jobs, user_args)
Xinyu Zhang82dab282022-10-09 16:33:19 +080049 job_links(finished_jobs, user_args)
Paul Sokolovsky2512ec52022-03-04 00:15:39 +030050 codecov_helper.coverage_reports(finished_jobs, user_args)
Xinyu Zhangf2b7cbf2021-05-18 20:17:34 +080051
52def get_finished_jobs(job_list, user_args, lava):
Paul Sokolovskya95abd92022-12-27 13:48:11 +030053 _log.info("Waiting for %d LAVA jobs", len(job_list))
Paul Sokolovsky697f9552022-05-05 10:44:27 +030054 finished_jobs = lava.block_wait_for_jobs(job_list, user_args.dispatch_timeout, 5)
Matthew Hartfb6fd362020-03-04 21:03:59 +000055 unfinished_jobs = [item for item in job_list if item not in finished_jobs]
56 for job in unfinished_jobs:
57 info_print("Cancelling unfinished job: {}".format(job))
58 lava.cancel_job(job)
59 if user_args.artifacts_path:
60 for job, info in finished_jobs.items():
61 info['job_dir'] = os.path.join(user_args.artifacts_path, "{}_{}".format(str(job), info['description']))
62 finished_jobs[job] = info
63 finished_jobs = fetch_artifacts(finished_jobs, user_args, lava)
Xinyu Zhangf2b7cbf2021-05-18 20:17:34 +080064 return finished_jobs
Matthew Hartfb6fd362020-03-04 21:03:59 +000065
Xinyu Zhangc8a670c2021-05-18 20:20:53 +080066def resubmit_failed_jobs(jobs, user_args):
67 if not jobs:
68 return []
Xinyu Zhang4aca6d02021-05-31 11:43:32 +080069 time.sleep(2) # be friendly to LAVA
Xinyu Zhangc8a670c2021-05-18 20:20:53 +080070 failed_job = []
71 os.makedirs('failed_jobs', exist_ok=True)
72 for job_id, info in jobs.items():
73 if not (info['health'] == "Complete" and info['state'] == "Finished"):
74 job_dir = info['job_dir']
75 def_path = os.path.join(job_dir, 'definition.yaml')
76 os.rename(def_path, 'failed_jobs/{}_definition.yaml'.format(job_id))
77 shutil.rmtree(job_dir)
78 failed_job.append(job_id)
79 for failed_job_id in failed_job:
80 jobs.pop(failed_job_id)
Xinyu Zhangc918b6e2022-10-08 17:13:17 +080081 resubmitted_jobs = submit_lava_jobs(user_args, job_dir='failed_jobs')
Xinyu Zhangc8a670c2021-05-18 20:20:53 +080082 resubmitted_jobs = [int(x) for x in resubmitted_jobs if x != '']
83 return resubmitted_jobs
84
Matthew Hartfb6fd362020-03-04 21:03:59 +000085def fetch_artifacts(jobs, user_args, lava):
86 if not user_args.artifacts_path:
87 return
88 for job_id, info in jobs.items():
89 job_dir = info['job_dir']
Paul Sokolovskydc8281a2022-12-27 21:54:42 +030090 t = time.time()
91 _log.info("Fetching artifacts for job %d to %s", job_id, job_dir)
Matthew Hartfb6fd362020-03-04 21:03:59 +000092 os.makedirs(job_dir, exist_ok=True)
93 def_path = os.path.join(job_dir, 'definition.yaml')
94 target_log = os.path.join(job_dir, 'target_log.txt')
Matthew Hart4a4f1202020-06-12 15:52:46 +010095 config = os.path.join(job_dir, 'config.tar.bz2')
96 results_file = os.path.join(job_dir, 'results.yaml')
Xinyu Zhang82dab282022-10-09 16:33:19 +080097 definition = lava.get_job_definition(job_id, def_path)
98 jobs[job_id]['metadata'] = definition.get('metadata', [])
Matthew Hartfb6fd362020-03-04 21:03:59 +000099 time.sleep(0.2) # be friendly to LAVA
Matthew Hart4a4f1202020-06-12 15:52:46 +0100100 lava.get_job_log(job_id, target_log)
Matthew Hartfb6fd362020-03-04 21:03:59 +0000101 time.sleep(0.2)
102 lava.get_job_config(job_id, config)
103 time.sleep(0.2)
Matthew Hart4a4f1202020-06-12 15:52:46 +0100104 lava.get_job_results(job_id, results_file)
Paul Sokolovskydc8281a2022-12-27 21:54:42 +0300105 _log.info("Fetched artifacts in %ds", time.time() - t)
Paul Sokolovskyc2d6d882022-02-25 19:11:18 +0300106 codecov_helper.extract_trace_data(target_log, job_dir)
Matthew Hartfb6fd362020-03-04 21:03:59 +0000107 return(jobs)
108
109
110def lava_id_to_url(id, user_args):
111 return "{}/scheduler/job/{}".format(user_args.lava_url, id)
112
Xinyu Zhang97ee3fd2020-12-14 14:45:06 +0800113def job_links(jobs, user_args):
114 job_links = ""
115 for job, info in jobs.items():
Xinyu Zhang82dab282022-10-09 16:33:19 +0800116 job_links += "\nLAVA Test Config:\n"
117 job_links += "Config Name: {}\n".format(info['metadata']['build_name'])
118 job_links += "Test Result: {}\n".format(info['result'])
119 job_links += "Device Type: {}\n".format(info['metadata']['device_type'])
Xinyu Zhang97ee3fd2020-12-14 14:45:06 +0800120 job_links += "Build link: {}\n".format(info['metadata']['build_job_url'])
Xinyu Zhang78c146a2022-09-05 19:06:40 +0800121 job_links += "LAVA link: {}\n".format(lava_id_to_url(job, user_args))
Xinyu Zhang82dab282022-10-09 16:33:19 +0800122 job_links += "TFM LOG: {}artifact/{}/target_log.txt\n".format(os.getenv("BUILD_URL"), info['job_dir'])
Xinyu Zhang97ee3fd2020-12-14 14:45:06 +0800123 print(job_links)
124
Matthew Hartfb6fd362020-03-04 21:03:59 +0000125def remove_lava_dupes(results):
126 for result in results:
127 if result['result'] != 'pass':
128 if result['suite'] == "lava":
129 for other in [x for x in results if x != result]:
130 if other['name'] == result['name']:
131 if other['result'] == 'pass':
132 results.remove(result)
133 return(results)
134
Paul Sokolovsky451f67b2022-03-08 19:44:41 +0300135def test_report(jobs, user_args):
Matthew Hartfb6fd362020-03-04 21:03:59 +0000136 # parsing of test results is WIP
137 fail_j = []
138 jinja_data = []
139 for job, info in jobs.items():
Xinyu Zhang0f78e7a2022-10-17 13:55:52 +0800140 info['result'] = 'SUCCESS'
Xinyu Zhang82dab282022-10-09 16:33:19 +0800141 if info['health'] != 'Complete':
Xinyu Zhang0f78e7a2022-10-17 13:55:52 +0800142 info['result'] = 'FAILURE'
Xinyu Zhang82dab282022-10-09 16:33:19 +0800143 fail_j.append(job)
144 continue
Matthew Hart4a4f1202020-06-12 15:52:46 +0100145 results_file = os.path.join(info['job_dir'], 'results.yaml')
146 if not os.path.exists(results_file) or (os.path.getsize(results_file) == 0):
Xinyu Zhang0f78e7a2022-10-17 13:55:52 +0800147 info['result'] = 'FAILURE'
Matthew Hart4a4f1202020-06-12 15:52:46 +0100148 fail_j.append(job)
149 continue
150 with open(results_file, "r") as F:
151 res_data = F.read()
Paul Sokolovskyf2f385d2022-01-11 00:36:31 +0300152 results = yaml.safe_load(res_data)
Paul Sokolovsky07f6dfb2022-07-15 12:26:24 +0300153 non_lava_results = [x for x in results if x['suite'] != 'lava' or x['name'] == 'lava-test-monitor']
Matthew Hartfb6fd362020-03-04 21:03:59 +0000154 info['lava_url'] = lava_id_to_url(job, user_args)
Arthur She38d5f5a2022-09-02 17:32:14 -0700155 info['artifacts_dir'] = info['job_dir']
Matthew Hartfb6fd362020-03-04 21:03:59 +0000156 jinja_data.append({job: [info, non_lava_results]})
157 for result in non_lava_results:
Paul Sokolovsky58f00de2022-02-01 00:26:32 +0300158 if result['result'] == 'fail':
Xinyu Zhang0f78e7a2022-10-17 13:55:52 +0800159 info['result'] = 'FAILURE'
Matthew Hartfb6fd362020-03-04 21:03:59 +0000160 fail_j.append(job) if job not in fail_j else fail_j
161 time.sleep(0.5) # be friendly to LAVA
Matthew Hartfb6fd362020-03-04 21:03:59 +0000162 data = {}
163 data['jobs'] = jinja_data
164 render_jinja(data)
165
166def render_jinja(data):
167 work_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "jinja2_templates")
168 template_loader = FileSystemLoader(searchpath=work_dir)
169 template_env = Environment(loader=template_loader)
170 html = template_env.get_template("test_summary.jinja2").render(data)
171 csv = template_env.get_template("test_summary_csv.jinja2").render(data)
172 with open('test_summary.html', "w") as F:
173 F.write(html)
174 with open('test_summary.csv', "w") as F:
175 F.write(csv)
176
177def print_lava_urls(jobs, user_args):
178 output = [lava_id_to_url(x, user_args) for x in jobs]
Xinyu Zhang78c146a2022-09-05 19:06:40 +0800179 info_print("LAVA jobs triggered for this build: {}".format(output))
Matthew Hartfb6fd362020-03-04 21:03:59 +0000180
181
Xinyu Zhang78c146a2022-09-05 19:06:40 +0800182def info_print(line, silent=True):
183 if not silent:
184 print("INFO: {}".format(line))
Matthew Hartfb6fd362020-03-04 21:03:59 +0000185
186def main(user_args):
187 """ Main logic """
Xinyu Zhang3e8f6602021-04-28 10:57:32 +0800188 for try_time in range(3):
189 try:
Paul Sokolovsky451f67b2022-03-08 19:44:41 +0300190 finished_jobs = wait_for_jobs(user_args)
Xinyu Zhang3e8f6602021-04-28 10:57:32 +0800191 break
192 except Exception as e:
Xinyu Zhang3e8f6602021-04-28 10:57:32 +0800193 if try_time < 2:
Paul Sokolovskycc51ea92022-02-02 19:34:02 +0300194 print("Exception in wait_for_jobs: {!r}".format(e))
Paul Sokolovskybf0f5492022-12-18 01:10:18 +0300195 traceback.print_exception(type(e), e, e.__traceback__)
Paul Sokolovskycc51ea92022-02-02 19:34:02 +0300196 print("Trying to get LAVA jobs again...")
Xinyu Zhang3e8f6602021-04-28 10:57:32 +0800197 else:
198 raise e
Paul Sokolovsky451f67b2022-03-08 19:44:41 +0300199 process_finished_jobs(finished_jobs, user_args)
Matthew Hartfb6fd362020-03-04 21:03:59 +0000200
201def get_cmd_args():
202 """ Parse command line arguments """
203
204 # Parse command line arguments to override config
205 parser = argparse.ArgumentParser(description="Lava Wait Jobs")
206 cmdargs = parser.add_argument_group("Lava Wait Jobs")
207
208 # Configuration control
209 cmdargs.add_argument(
210 "--lava-url", dest="lava_url", action="store", help="LAVA lab URL (without RPC2)"
211 )
212 cmdargs.add_argument(
213 "--job-ids", dest="job_ids", action="store", required=True, help="Comma separated list of job IDS"
214 )
215 cmdargs.add_argument(
Xinyu Zhangf2b7cbf2021-05-18 20:17:34 +0800216 "--lava-token", dest="lava_token", action="store", help="LAVA auth token"
Matthew Hartfb6fd362020-03-04 21:03:59 +0000217 )
218 cmdargs.add_argument(
Xinyu Zhangf2b7cbf2021-05-18 20:17:34 +0800219 "--lava-user", dest="lava_user", action="store", help="LAVA username"
Matthew Hartfb6fd362020-03-04 21:03:59 +0000220 )
221 cmdargs.add_argument(
222 "--use-env", dest="token_from_env", action="store_true", default=False, help="Use LAVA auth info from environment"
223 )
224 cmdargs.add_argument(
225 "--lava-timeout", dest="dispatch_timeout", action="store", type=int, default=3600, help="Time in seconds to wait for all jobs"
226 )
227 cmdargs.add_argument(
228 "--artifacts-path", dest="artifacts_path", action="store", help="Download LAVA artifacts to this directory"
229 )
230 return parser.parse_args()
231
232
233if __name__ == "__main__":
Paul Sokolovskya95abd92022-12-27 13:48:11 +0300234 logging.basicConfig(level=logging.INFO)
Matthew Hartfb6fd362020-03-04 21:03:59 +0000235 main(get_cmd_args())