jenkins/ci.jpl

#!/usr/bin/env groovy
//-------------------------------------------------------------------------------
// Copyright (c) 2020-2021, Arm Limited and Contributors. All rights reserved.
//
// SPDX-License-Identifier: BSD-3-Clause
//
//-------------------------------------------------------------------------------

@Library('trustedfirmware') _
import org.trustedfirmware.Gerrit
import org.trustedfirmware.Summary

mapPlatform = ["cypress/psoc64":            "psoc64",
               "mps2/an519":                "AN519",
               "mps2/an521":                "AN521",
               "mps2/an539":                "AN539",
               "mps2/sse-200_aws":          "SSE-200_AWS",
               "mps3/an524":                "AN524",
               "musca_b1/sse_200":          "MUSCA_B1",
               "musca_b1/secure_enclave":   "MUSCA_B1_SE",
               "musca_s1":                  "MUSCA_S1"]

mapCompiler = ["toolchain_GNUARM.cmake":   "GNUARM",
               "toolchain_ARMCLANG.cmake": "ARMCLANG"]

mapBL2 = ["True":  "--bl2",
          "False": ""]

mapTestPsaApi = ["OFF":                      "",
                 "INTERNAL_TRUSTED_STORAGE": "ITS",
                 "PROTECTED_STORAGE":        "PS",
                 "CRYPTO":                   "Crypto",
                 "INITIAL_ATTESTATION":      "Attest",
                 "IPC":                      "FF"]

// BL2, NS, PSA_API, ISOLATION_LEVEL, TEST_REG, TEST_PSA_API, PROFILE, CONFIG_NAME
mapConfigs = [
  ["True", "True", "False", "1", "False", "OFF",                      "N.A",            "Default"],
  ["True", "True", "True",  "1", "False", "OFF",                      "N.A",            "CoreIPC"],
  ["True", "True", "True",  "2", "False", "OFF",                      "N.A",            "CoreIPCTfmLevel2"],
  ["True", "True", "True",  "3", "False", "OFF",                      "N.A",            "CoreIPCTfmLevel3"],
  ["True", "True", "False", "1", "False", "OFF",                      "profile_small",  "DefaultProfileS"],
  ["True", "True", "True",  "2", "False", "OFF",                      "profile_medium", "DefaultProfileM"],
  ["True", "True", "True",  "3", "False", "OFF",                      "profile_large",  "DefaultProfileL"],
  ["True", "True", "False", "1", "True",  "OFF",                      "N.A",            "Regression"],
  ["True", "True", "True",  "1", "True",  "OFF",                      "N.A",            "RegressionIPC"],
  ["True", "True", "True",  "2", "True",  "OFF",                      "N.A",            "RegressionIPCTfmLevel2"],
  ["True", "True", "True",  "3", "True",  "OFF",                      "N.A",            "RegressionIPCTfmLevel3"],
  ["True", "True", "False", "1", "True",  "OFF",                      "profile_small",  "RegressionProfileS"],
  ["True", "True", "True",  "2", "True",  "OFF",                      "profile_medium", "RegressionProfileM"],
  ["True", "True", "True",  "3", "True",  "OFF",                      "profile_large",  "RegressionProfileL"],
  ["True", "True", "False", "1", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTest (ITS)"],
  ["True", "True", "False", "1", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTest (PS)"],
  ["True", "True", "False", "1", "False", "CRYPTO",                   "N.A",            "PsaApiTest (Crypto)"],
  ["True", "True", "False", "1", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTest (Attest)"],
  ["True", "True", "False", "1", "False", "IPC",                      "N.A",            "PsaApiTest (FF)"],
  ["True", "True", "True",  "1", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTestIPC (ITS)"],
  ["True", "True", "True",  "1", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTestIPC (PS)"],
  ["True", "True", "True",  "1", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPC (Crypto)"],
  ["True", "True", "True",  "1", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPC (Attest)"],
  ["True", "True", "True",  "1", "False", "IPC",                      "N.A",            "PsaApiTestIPC (FF)"],
  ["True", "True", "True",  "2", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTestIPCTfmLevel2 (ITS)"],
  ["True", "True", "True",  "2", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTestIPCTfmLevel2 (PS)"],
  ["True", "True", "True",  "2", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPCTfmLevel2 (Crypto)"],
  ["True", "True", "True",  "2", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPCTfmLevel2 (Attest)"],
  ["True", "True", "True",  "2", "False", "IPC",                      "N.A",            "PsaApiTestIPCTfmLevel2 (FF)"],
  ["True", "True", "True",  "3", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTestIPCTfmLevel3 (ITS)"],
  ["True", "True", "True",  "3", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTestIPCTfmLevel3 (PS)"],
  ["True", "True", "True",  "3", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPCTfmLevel3 (Crypto)"],
  ["True", "True", "True",  "3", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPCTfmLevel3 (Attest)"],
  ["True", "True", "True",  "3", "False", "IPC",                      "N.A",            "PsaApiTestIPCTfmLevel3 (FF)"],
]

cfgs = ["Default", "CoreIPC", "CoreIPCTfmLevel2", "CoreIPCTfmLevel3",
        "Regression", "RegressionIPC",
        "RegressionIPCTfmLevel2", "RegressionIPCTfmLevel3",
        "DefaultProfileS", "RegressionProfileS",
        "DefaultProfileM", "RegressionProfileM", "RegressionProfileM PSOFF",
        "DefaultProfileL", "RegressionProfileL",
        "PsaApiTest (Attest)", "PsaApiTestIPC (Attest)",
        "PsaApiTestIPCTfmLevel2 (Attest)",
        "PsaApiTest (Crypto)", "PsaApiTestIPC (Crypto)",
        "PsaApiTestIPCTfmLevel2 (Crypto)",
        "PsaApiTest (PS)", "PsaApiTestIPC (PS)",
        "PsaApiTestIPCTfmLevel2 (PS)",
        "PsaApiTest (ITS)", "PsaApiTestIPC (ITS)",
        "PsaApiTestIPCTfmLevel2 (ITS)",
        "PsaApiTestIPC (FF)",
        "PsaApiTestIPCTfmLevel2 (FF)",
        "PsaApiTestIPCTfmLevel3 (ITS)", "PsaApiTestIPCTfmLevel3 (PS)",
        "PsaApiTestIPCTfmLevel3 (Crypto)", "PsaApiTestIPCTfmLevel3 (Attest)",
        "PsaApiTestIPCTfmLevel3 (FF)"]

@NonCPS
def generateLavaParam(build_params) {
  def params = []
  if (build_params["TFM_PLATFORM"] == "musca_b1/sse_200" && \
      build_params["OTP"] == "ENABLED") {
    params += string(name: "TARGET_PLATFORM", value: "MUSCA_B1_OTP")
  }
  else {
    params += string(name: "TARGET_PLATFORM", \
                     value: mapPlatform[build_params["TFM_PLATFORM"]])
  }
  params += string(name: "COMPILER", \
                   value: mapCompiler[build_params["TOOLCHAIN_FILE"]])
  params += string(name: "PSA_API_SUITE", \
                   value: mapTestPsaApi[build_params["TEST_PSA_API"]])

  configName = "Config"
  config_params = [build_params["BL2"], build_params["NS"], \
                   build_params["PSA_API"], build_params["ISOLATION_LEVEL"], \
                   build_params["TEST_REGRESSION"], build_params["TEST_PSA_API"], \
                   build_params["PROFILE"]]
  for (config in mapConfigs) {
    if (config_params == config[0..6]) {
      configName += config[7].replace(' (', '_').replace(')', '')
      break
    }
  }
  if (configName == "Config") {
    configName = "ConfigDefault"
  }
  params += string(name: "PROJ_CONFIG", value: configName)
  print("Params of ${configName} :")
  print(config_params)
  return params
}

def listConfigs(ci_scripts_dir, config_list, filter_group) {
  dir(ci_scripts_dir) {
    echo "Obtaining list of configs."
    echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}"
    def build_config_list_raw = sh(script: """\
python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}
""", returnStdout: true).trim()
    def build_config_list = build_config_list_raw.tokenize('\n')
    config_list.addAll(build_config_list)
  }
}

def buildConfig(ci_scripts_dir, config, filter_group, results) {
  def params = []
  def params_collection = [:]
  def build_config_params
  dir(ci_scripts_dir) {
    echo "Obtaining build configuration for config ${config}"
    echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}"
    build_config_params = sh(script: """\
python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}
""", returnStdout: true).trim()
  }
  def lines = build_config_params.tokenize('\n')
  for (String line : lines) {
    def key, value
    (key, value) = line.tokenize('=')
    params += string(name: key, value: value)
    params_collection[key] = value
  }
  params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
  params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
  params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
  params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
  params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
  params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
  params += string(name: 'CODE_REPO', value: env.CODE_REPO)
  params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
  params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
  if (env.JOB_NAME.equals("tf-m-nightly")) { //Setting the Memory footprint gathering.
     params += string(name: 'SQUAD_CONFIGURATIONS', value: env.SQUAD_CONFIGURATIONS)
  }
  return { -> results
    def build_res = build(job: 'tf-m-build-config', parameters: params, propagate: false)
    def build_info = [build_res, config, params_collection]
    results['builds'][build_res.number] = build_info
    def build_url = build_res.getAbsoluteUrl()
    print("${build_res.number}: ${config} ${build_res.result} ${build_url}")
    failure_states = ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]
    if (build_res.result in failure_states) {
      error("Build failed at ${build_url}")
    }
    else if (params_collection["NS"] == "False" || \
             (params_collection["TFM_PLATFORM"].contains("musca_b1") && \
             env.JOB_NAME.equals("tf-m-build-and-test")) || \
             (params_collection["PROFILE"] == "profile_medium" && \
              params_collection["PARTITION_PS"] == "OFF")) {
      print("LAVA is not needed for ${build_url}")
    }
    // Only submit LAVA test for a specified OTP enabled config in nightly or release job
    else if (params_collection["OTP"] == "ENABLED" && \
             params_collection["CONFIG_NAME"] != "MUSCA_B1_ARMCLANG_PSA_3_REG_Debug_OTP_BL2_NS") {
      print("LAVA is not needed for ${build_url}")
    }
    else if (params_collection["CONFIG_NAME"] == "MUSCA_B1_ARMCLANG_PSA_3_REG_Debug_OTP_BL2_NS" && \
             !(env.JOB_NAME.equals("tf-m-nightly") || env.JOB_NAME.equals("tf-m-release"))) {
      print("LAVA is not needed for ${build_url}")
    }
    else {
      print("Doing LAVA stuff for ${build_url}")
      params += generateLavaParam(params_collection)
      params += string(name: 'BUILD_NUMBER', value: "${build_res.number}")
      params += string(name: 'BUILD_URL', value: build_url)
      params += string(name: 'LAVA_URL', value: env.LAVA_URL)
      params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
      params += string(name: 'LAVA_CREDENTIALS', value: env.LAVA_CREDENTIALS)
      def lava_res = build(job: 'tf-m-lava-submit', parameters: params, propagate: false)
      def lava_resubmitted = false
      if (lava_res.result in failure_states) {
        error("LAVA Create and Submit failed at ${lava_res.getAbsoluteUrl()}")
      }
      else {
        lava_des = lava_res.getDescription()
        if (lava_des.contains(" Submitted twice!")) {
          lava_resubmitted = true
          lava_des = lava_des - " Submitted twice!"
        }
        results['lava_jobs'] += lava_des
      }
      links = "Build Config: ${config}\n"
      links += "Build URL: ${build_url}\n"
      links += "LAVA Submit: ${lava_res.getAbsoluteUrl()}"
      if (lava_resubmitted) {
        links += "\nLAVA Job Re-Submitted!"
      }
      print(links)
    }
  }
}

def buildDocs(results) {
  def params = []
  params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
  params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
  params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
  params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
  params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
  params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
  params += string(name: 'CODE_REPO', value: env.CODE_REPO)
  params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
  return { -> results
    def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false)
    print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}")
    results['docs'] = [res.number, res.result, params]
    if (res.result in ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]) {
      error("Build failed at ${res.getAbsoluteUrl()}")
    }
  }
}

def generateEmailBody(stage, failed_jobs) {
  body = "Check console output at ${env.BUILD_URL} \n\n"

  body += "Failed Jobs:\n"
  failed_jobs.each { job ->
    body += "${job.key}  ${job.value}\n"
  }

  body += "\nFor detailed ${stage} results please refer to \
           ${env.BUILD_URL}artifact/${stage}_results.csv \n"
  return body
}

def emailNotification(results, stage, failed_jobs) {
  script {
   if (env.JOB_NAME.equals("tf-m-nightly") && !env.EMAIL_NOTIFICATION.equals('')) {
      def result = "Fail."
      if (results == true) {
        result = "Success."
        print("Skip sending as ${result} for ${stage}")
      }
      else {
        emailext (
            subject: ("Job ${env.JOB_NAME} ${stage} ${env.BUILD_NUMBER} ${result}"),
            body: generateEmailBody(stage, failed_jobs),
            to: "${EMAIL_NOTIFICATION}"
        )
      }
    }
  } /* script */
}

def filterFailedBuild(results) {
  def failed_builds = [:]
  results.each { result ->
    if (result.value[0].getResult() == "FAILURE") {
      failed_builds[result.value[1]] = result.value[0].getAbsoluteUrl()
    }
  }
  return failed_builds
}

def filterFailedTest(string) {
  def failed_tests = [:]
  line = lineInString(string, "FAILURE_TESTS:")
  a = line.split(' ')
  if (a.size() > 1) {
    a = line.split(' ')[1..-1]
    a.each { fail_test ->
      config_link = fail_test.split(':')
      failed_tests[config_link[0]] = config_link[1..-1].join(':')
  }
  }
  return failed_tests
}

@NonCPS
def generateCsvContent(results) {
  def resultsParam = []
  results.each { result ->
    if (result.value[2]['BL2'] == "True") {
      resultsParam.add([result.value[1], \
                        result.value[0].getResult(), \
                        result.value[2]['TFM_PLATFORM'], \
                        result.value[2]['TOOLCHAIN_FILE'], \
                        result.value[2]['CMAKE_BUILD_TYPE'], \
                        result.value[2]['BL2'], \
                        result.value[2]['NS'], \
                        result.value[2]['PSA_API'], \
                        result.value[2]['ISOLATION_LEVEL'], \
                        result.value[2]['TEST_REGRESSION'], \
                        result.value[2]['TEST_PSA_API'], \
                        result.value[2]['PROFILE'], \
                        result.value[2]['PARTITION_PS'], \
                        result.value[2]['OTP']])
    }
  }
  resultsParam.each { result ->
    if (result[2] == 'musca_b1/sse_200') {
      if (result[13] != 'off') {
        result[2] = 'musca_b1/sse_200_OTP'
      }
    }
    result[3] = mapCompiler[result[3]]
    build_params = result[5..12]
    configName = ""
    for (map_cfg in mapConfigs) {
      if (build_params[0..6] == map_cfg[0..6]) {
        configName = map_cfg[7]
        break
      }
    }
    if (configName == "") {
      configName = "Default"
    }
    else if (configName == "RegressionProfileM") {
      if (build_params[7] == "OFF") {
        configName = "RegressionProfileM PSOFF"
      }
    }
    result.add(configName)
  }
  def csvContent = []
  resultsParam.each { result ->
    current_row = result[2..4]
    cfgs.each {cfg ->
      if (cfg == result[14]) {
        current_row.add(cfg)
        current_row.add(result[1])
      }
    }
    csvContent.add(current_row)
  }
  csvContent.sort{a,b -> a[0] <=> b[0] ?: a[1] <=> b[1] ?: a[2] <=> b[2]}
  build_summary = []
  current_platform = ""
  current_compiler = ""
  current_build_type = ""
  csvContent.each { build_cfg ->
    if (current_platform != build_cfg[0] || \
        current_compiler != build_cfg[1] || \
        current_build_type != build_cfg[2]) {
          current_platform = build_cfg[0]
          current_compiler = build_cfg[1]
          current_build_type = build_cfg[2]
          csv_line = [current_platform, current_compiler, current_build_type]
          cfgs.each {
            csv_line.add("N.A.")
          }
          build_summary.add(csv_line)
        }
    i = 0
    cfgs.each { cfg ->
      if (cfg == build_cfg[3]) {
        build_summary[-1][3+i] = build_cfg[4]
      }
      i += 1
    }
  }
  build_summary.add(0, ['Platform', 'Compiler', 'Cmake Build Type'])
  build_summary[0] += cfgs
  return build_summary
}

def generateBuildCsv(results) {
  def csvContent = generateCsvContent(results)
  node("master") {
    writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
    archiveArtifacts 'build_results.csv'
  }
}

def buildCsv(results) {
  def summary = new Summary();
  def csvContent = summary.getBuildCsv(results)
  node("master") {
    writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
    archiveArtifacts 'build_results.csv'
  }
}

def writeSummary(results) {
  def summary = new Summary();
  def buildLinks = summary.getLinks(results)
  node("master") {
    writeFile file: "build_links.html", text: buildLinks
    archiveArtifacts 'build_links.html'
  }
}

def lineInString(string, match) {
  def lines = string.split("\n")
  def result = lines.findAll { it.contains(match) }
  return result[0]
}

def showLinks(string) {
  def lines = string.split("\n")
  def result = lines.findAll { it.contains("Build Config: ")}
  links = result.join("\n")
  print(links)
}

def getResult(string, match) {
  line = lineInString(string, match)
  a = line.split(match)[1].split(' ')
  score = a[0]
  if (a.size() > 1)
  {
    fail_text = a[1..-1].join(" ")
    return [score, fail_text]
  }
  return [score, ""]
}

def submitJobsToList(results) {
  def all_jobs = []
  for (String result : results){
    jobs_s = result.split('JOBS: ')
    if (jobs_s.size() > 1) {
      all_jobs += jobs_s[1]
    }
  }
  return(all_jobs)
}

def configs = []
def builds = [:]
def results = [:]

timestamps {
  node("docker-amd64-tf-m-bionic") {
    stage("Init") {
      cleanWs()
      dir("tf-m-ci-scripts") {
        checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
      }
    }
    stage("Configs") {
      // Populate configs
      listConfigs('tf-m-ci-scripts', configs, env.FILTER_GROUP)
      results['builds'] = [:]
      results['lava_jobs'] = []
      for (config in configs) {
        builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP, results)
      }
      builds["docs"] = buildDocs(results)
    }
  }

  stage("Builds") {
    def verify = 1
    def success = true
    try {
      parallel(builds)
    } catch (Exception e) {
      print(e)
      manager.buildFailure()
      verify = -1
      success = false
    } finally {
      print("Verifying status")
      def failed_builds = filterFailedBuild(results['builds'])
      emailNotification(success, 'build', failed_builds)
      g = new Gerrit()
      g.verifyStatus(verify, 'tf-m-build', 'build')
      print("Building CSV")
      generateBuildCsv(results['builds'])
      writeSummary(results['builds'])
    }
  }

  node("docker-amd64-tf-m-bionic") {
    stage("Tests") {
      dir("tf-m-ci-scripts") {
        checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
      }
      def all_jobs = []
      def success = true
      print("Wait for LAVA results here...")
      try {
        all_jobs = submitJobsToList(results['lava_jobs'])
        if (all_jobs.size() > 0) {
          dir("tf-m-ci-scripts") {
            withCredentials([usernamePassword(credentialsId: env.LAVA_CREDENTIALS, passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) {
              output = sh(script: """./lava_helper/lava_wait_jobs.py --job-ids ${all_jobs.join(",")} \
    --lava-url ${env.LAVA_URL} --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} \
    --artifacts-path lava_artifacts --lava-timeout 7200 \
    """, returnStdout: true).trim()
              showLinks(output)
              archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true
              archiveArtifacts artifacts: 'test_results.csv', allowEmptyArchive: true
              g = new Gerrit()
              def (boot_result, boot_output) = getResult(output, 'BOOT_RESULT: ')
              if (boot_result) {
                g.verifyStatus(boot_result, "lava_boot", "test")
              }
              def (test_result, test_output) = getResult(output, 'TEST_RESULT: ')
              if (test_result) {
                g.verifyStatus(test_result, "lava_test", "test")
              }
              if (boot_result.toInteger() < 1 || test_result.toInteger() < 1) {
                error("Marking job as failed due to failed boots: ${boot_output} or tests: ${test_output}")
              }
            }
          }
        }
        else {
          print("There were no LAVA jobs to test.")
        }
      }
      catch (Exception e) {
        print("ERROR: ${e}")
        success = false
      } finally {
        archiveArtifacts artifacts: 'tf-m-ci-scripts/lava_artifacts/**', allowEmptyArchive: true
        if (all_jobs.size() > 0) {
          emailNotification(success, 'test', filterFailedTest(output))
        }
        cleanWs()
        if (!success) {
          error("There was an Error waiting for LAVA jobs")
        }
      }
    }
  }
}