Email Notification: Show Failure Details for Nightly Job
Email Notification would contain failure details, including:
1. URL of current nightly job;
2. Failed configs and URLs for Build or Test jobs;
3. Results of jobs in CSV format.
Change-Id: I78f4ac30621e45875928a4b8a02c9d5fd79903f3
Signed-off-by: Xinyu Zhang <xinyu.zhang@arm.com>
diff --git a/jenkins/ci.jpl b/jenkins/ci.jpl
index fc93d40..167c1d4 100644
--- a/jenkins/ci.jpl
+++ b/jenkins/ci.jpl
@@ -338,7 +338,20 @@
}
}
-def emailNotification(results, stage) {
+def generateEmailBody(stage, failed_jobs) {
+ body = "Check console output at ${env.BUILD_URL} \n\n"
+
+ body += "Failed Jobs:\n"
+ failed_jobs.each { job ->
+ body += "${job.key} ${job.value}\n"
+ }
+
+ body += "\nFor detailed ${stage} results please refer to \
+ ${env.BUILD_URL}artifact/${stage}_results.csv \n"
+ return body
+}
+
+def emailNotification(results, stage, failed_jobs) {
script {
if (env.JOB_NAME.equals("tf-m-nightly") && !env.EMAIL_NOTIFICATION.equals('')) {
def result = "Fail."
@@ -349,7 +362,7 @@
else {
emailext (
subject: ("Job ${env.JOB_NAME} ${stage} ${env.BUILD_NUMBER} ${result}"),
- body: "Check console output at ${env.BUILD_URL}",
+ body: generateEmailBody(stage, failed_jobs),
to: "${EMAIL_NOTIFICATION}"
)
}
@@ -357,6 +370,30 @@
} /* script */
}
+def filterFailedBuild(results) {
+ def failed_builds = [:]
+ results.each { result ->
+ if (result.value[0].getResult() == "FAILURE") {
+ failed_builds[result.value[1]] = result.value[0].getAbsoluteUrl()
+ }
+ }
+ return failed_builds
+}
+
+def filterFailedTest(string) {
+ def failed_tests = [:]
+ line = lineInString(string, "FAILURE_TESTS:")
+ a = line.split(' ')
+ if (a.size() > 1) {
+ a = line.split(' ')[1..-1]
+ a.each { fail_test ->
+ config_link = fail_test.split(':')
+ failed_tests[config_link[0]] = config_link[1..-1].join(':')
+ }
+ }
+ return failed_tests
+}
+
@NonCPS
def generateCsvContent(results) {
def resultsParam = []
@@ -535,7 +572,8 @@
success = false
} finally {
print("Verifying status")
- emailNotification(success, 'Build')
+ def failed_builds = filterFailedBuild(results['builds'])
+ emailNotification(success, 'build', failed_builds)
g = new Gerrit()
g.verifyStatus(verify, 'tf-m-build', 'build')
print("Building CSV")
@@ -574,7 +612,6 @@
""", returnStdout: true).trim()
archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true
archiveArtifacts artifacts: 'test_results.csv', allowEmptyArchive: true
- print(output)
g = new Gerrit()
def (boot_result, boot_output) = getResult(output, 'BOOT_RESULT: ')
if (boot_result) {
@@ -599,7 +636,7 @@
success = false
} finally {
archiveArtifacts artifacts: 'tf-m-ci-scripts/lava_artifacts/**', allowEmptyArchive: true
- emailNotification(success, 'Test')
+ emailNotification(success, 'test', filterFailedTest(output))
cleanWs()
if (!success) {
error("There was an Error waiting for LAVA jobs")
diff --git a/lava_helper/lava_wait_jobs.py b/lava_helper/lava_wait_jobs.py
index d0578b2..1cd3345 100755
--- a/lava_helper/lava_wait_jobs.py
+++ b/lava_helper/lava_wait_jobs.py
@@ -72,6 +72,7 @@
print_lava_urls(finished_jobs, user_args)
boot_report(finished_jobs, user_args)
test_report(finished_jobs, user_args, lava)
+ failure_report(finished_jobs, user_args)
csv_report(finished_jobs)
def fetch_artifacts(jobs, user_args, lava):
@@ -148,6 +149,14 @@
else:
print("BOOT_RESULT: +1")
+def failure_report(jobs, user_args):
+ failed_report = "FAILURE_TESTS:"
+ for job, info in jobs.items():
+ if info['health'] != "Complete" or info['state'] != "Finished":
+ failed_report += " {}:{}".format(info['metadata']['build_name'],
+ lava_id_to_url(job, user_args))
+ print(failed_report)
+
def remove_lava_dupes(results):
for result in results:
if result['result'] != 'pass':