jenkins/ci.jpl

#!/usr/bin/env groovy
//-------------------------------------------------------------------------------
// Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
//
// SPDX-License-Identifier: BSD-3-Clause
//
//-------------------------------------------------------------------------------

@Library('trustedfirmware') _
import org.trustedfirmware.Gerrit
import org.trustedfirmware.Summary

mapPlatform = ["cypress/psoc64":   "psoc64",
               "mps2/an519":       "AN519",
               "mps2/an521":       "AN521",
               "mps2/an539":       "AN539",
               "mps2/sse-200_aws": "SSE-200_AWS",
               "mps3/an524":       "AN524",
               "musca_a":          "MUSCA_A",
               "musca_b1":         "MUSCA_B1",
               "musca_s1":         "MUSCA_S1"]

mapCompiler = ["toolchain_GNUARM.cmake":   "GNUARM",
               "toolchain_ARMCLANG.cmake": "ARMCLANG"]

mapBL2 = ["True":  "--bl2",
          "False": ""]

mapTestPsaApi = ["OFF":                      "",
                 "INTERNAL_TRUSTED_STORAGE": "ITS",
                 "PROTECTED_STORAGE":        "PS",
                 "CRYPTO":                   "Crypto",
                 "INITIAL_ATTESTATION":      "Attest",
                 "IPC":                      "FF"]

// BL2, NS, PSA_API, ISOLATION_LEVEL, TEST_REG, TEST_PSA_API, PROFILE, PARTITION_PS, OTP
mapConfigs = [
  ["True", "True", "False", "1", "False", "OFF",                      "N.A",            "Default"],
  ["True", "True", "True",  "1", "False", "OFF",                      "N.A",            "CoreIPC"],
  ["True", "True", "True",  "2", "False", "OFF",                      "N.A",            "CoreIPCTfmLevel2"],
  ["True", "True", "True",  "3", "False", "OFF",                      "N.A",            "CoreIPCTfmLevel3"],
  ["True", "True", "False", "1", "False", "OFF",                      "profile_small",  "DefaultProfileS"],
  ["True", "True", "True",  "2", "False", "OFF",                      "profile_medium", "DefaultProfileM"],
  ["True", "True", "False", "1", "True",  "OFF",                      "N.A",            "Regression"],
  ["True", "True", "True",  "1", "True",  "OFF",                      "N.A",            "RegressionIPC"],
  ["True", "True", "True",  "2", "True",  "OFF",                      "N.A",            "RegressionIPCTfmLevel2"],
  ["True", "True", "True",  "3", "True",  "OFF",                      "N.A",            "RegressionIPCTfmLevel3"],
  ["True", "True", "False", "1", "True",  "OFF",                      "profile_small",  "RegressionProfileS"],
  ["True", "True", "True",  "2", "True",  "OFF",                      "profile_medium", "RegressionProfileM"],
  ["True", "True", "False", "1", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTest (ITS)"],
  ["True", "True", "False", "1", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTest (PS)"],
  ["True", "True", "False", "1", "False", "CRYPTO",                   "N.A",            "PsaApiTest (Crypto)"],
  ["True", "True", "False", "1", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTest (Attest)"],
  ["True", "True", "False", "1", "False", "IPC",                      "N.A",            "PsaApiTest (FF)"],
  ["True", "True", "True",  "1", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTestIPC (ITS)"],
  ["True", "True", "True",  "1", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTestIPC (PS)"],
  ["True", "True", "True",  "1", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPC (Crypto)"],
  ["True", "True", "True",  "1", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPC (Attest)"],
  ["True", "True", "True",  "1", "False", "IPC",                      "N.A",            "PsaApiTestIPC (FF)"],
  ["True", "True", "True",  "2", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTestIPCTfmLevel2 (ITS)"],
  ["True", "True", "True",  "2", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTestIPCTfmLevel2 (PS)"],
  ["True", "True", "True",  "2", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPCTfmLevel2 (Crypto)"],
  ["True", "True", "True",  "2", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPCTfmLevel2 (Attest)"],
  ["True", "True", "True",  "2", "False", "IPC",                      "N.A",            "PsaApiTestIPCTfmLevel2 (FF)"],
  ["True", "True", "True",  "3", "False", "INTERNAL_TRUSTED_STORAGE", "N.A",            "PsaApiTestIPCTfmLevel3 (ITS)"],
  ["True", "True", "True",  "3", "False", "PROTECTED_STORAGE",        "N.A",            "PsaApiTestIPCTfmLevel3 (PS)"],
  ["True", "True", "True",  "3", "False", "CRYPTO",                   "N.A",            "PsaApiTestIPCTfmLevel3 (Crypto)"],
  ["True", "True", "True",  "3", "False", "INITIAL_ATTESTATION",      "N.A",            "PsaApiTestIPCTfmLevel3 (Attest)"],
  ["True", "True", "True",  "3", "False", "IPC",                      "N.A",            "PsaApiTestIPCTfmLevel3 (FF)"],
]

cfgs = ["Default", "CoreIPC", "CoreIPCTfmLevel2", "CoreIPCTfmLevel3",
        "Regression", "RegressionIPC",
        "RegressionIPCTfmLevel2", "RegressionIPCTfmLevel3",
        "DefaultProfileS", "RegressionProfileS",
        "DefaultProfileM", "RegressionProfileM", "RegressionProfileM PSOFF",
        "PsaApiTest (Attest)", "PsaApiTestIPC (Attest)",
        "PsaApiTestIPCTfmLevel2 (Attest)",
        "PsaApiTest (Crypto)", "PsaApiTestIPC (Crypto)",
        "PsaApiTestIPCTfmLevel2 (Crypto)",
        "PsaApiTest (PS)", "PsaApiTestIPC (PS)",
        "PsaApiTestIPCTfmLevel2 (PS)",
        "PsaApiTest (ITS)", "PsaApiTestIPC (ITS)",
        "PsaApiTestIPCTfmLevel2 (ITS)",
        "PsaApiTestIPC (FF)",
        "PsaApiTestIPCTfmLevel2 (FF)",
        "PsaApiTestIPCTfmLevel3 (ITS)", "PsaApiTestIPCTfmLevel3 (PS)",
        "PsaApiTestIPCTfmLevel3 (Crypto)", "PsaApiTestIPCTfmLevel3 (Attest)",
        "PsaApiTestIPCTfmLevel3 (FF)"]

def generateLavaParam(build_params) {
  def params = []
  params += string(name: "TARGET_PLATFORM", \
                   value: mapPlatform[build_params["TFM_PLATFORM"]])
  params += string(name: "COMPILER", \
                   value: mapCompiler[build_params["TOOLCHAIN_FILE"]])
  params += string(name: "PSA_API_SUITE", \
                   value: mapTestPsaApi[build_params["TEST_PSA_API"]])

  if (build_params["BL2"]             == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "False" && \
      build_params["ISOLATION_LEVEL"] == "1"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigDefault")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "1"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigCoreIPC")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "2"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigCoreIPCTfmLevel2")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "3"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigCoreIPCTfmLevel3")
      }
  else if (build_params["BL2"]        == "True"          && \
      build_params["NS"]              == "True"          && \
      build_params["PSA_API"]         == "False"         && \
      build_params["ISOLATION_LEVEL"] == "1"             && \
      build_params["PROFILE"]         == "profile_small" && \
      build_params["TEST_REGRESSION"] == "False"         && \
      build_params["TEST_PSA_API"]    == "OFF") {
        params += string(name: "PROJ_CONFIG", value: "ConfigDefaultProfileS")
      }
  else if (build_params["BL2"]        == "True"          && \
      build_params["NS"]              == "True"          && \
      build_params["PSA_API"]         == "True"          && \
      build_params["ISOLATION_LEVEL"] == "2"             && \
      build_params["PROFILE"]         == "profile_medium"&& \
      build_params["TEST_REGRESSION"] == "False"         && \
      build_params["TEST_PSA_API"]    == "OFF") {
        params += string(name: "PROJ_CONFIG", value: "ConfigDefaultProfileM")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "False" && \
      build_params["ISOLATION_LEVEL"] == "1"     && \
      build_params["TEST_REGRESSION"] == "True"  && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigRegression")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "1"     && \
      build_params["TEST_REGRESSION"] == "True"  && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigRegressionIPC")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "2"     && \
      build_params["TEST_REGRESSION"] == "True"  && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigRegressionIPCTfmLevel2")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "3"     && \
      build_params["TEST_REGRESSION"] == "True"  && \
      build_params["TEST_PSA_API"]    == "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigRegressionIPCTfmLevel3")
      }
  else if (build_params["BL2"]        == "True"          && \
      build_params["NS"]              == "True"          && \
      build_params["PSA_API"]         == "False"         && \
      build_params["ISOLATION_LEVEL"] == "1"             && \
      build_params["PROFILE"]         == "profile_small" && \
      build_params["TEST_REGRESSION"] == "True"          && \
      build_params["TEST_PSA_API"]    == "OFF") {
        params += string(name: "PROJ_CONFIG", value: "ConfigRegressionProfileS")
      }
  else if (build_params["BL2"]        == "True"          && \
      build_params["NS"]              == "True"          && \
      build_params["PSA_API"]         == "True"         && \
      build_params["ISOLATION_LEVEL"] == "2"             && \
      build_params["PROFILE"]         == "profile_medium"&& \
      build_params["TEST_REGRESSION"] == "True"          && \
      build_params["TEST_PSA_API"]    == "OFF") {
        params += string(name: "PROJ_CONFIG", value: "ConfigRegressionProfileM")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "False" && \
      build_params["ISOLATION_LEVEL"] == "1"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    != "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigPsaApiTest")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "1"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    != "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigPsaApiTestIPC")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "2"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    != "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigPsaApiTestIPCTfmLevel2")
      }
  else if (build_params["BL2"]        == "True"  && \
      build_params["NS"]              == "True"  && \
      build_params["PSA_API"]         == "True"  && \
      build_params["ISOLATION_LEVEL"] == "3"     && \
      build_params["TEST_REGRESSION"] == "False" && \
      build_params["TEST_PSA_API"]    != "OFF"   && \
      build_params["PROFILE"]         == "N.A") {
        params += string(name: "PROJ_CONFIG", value: "ConfigPsaApiTestIPCTfmLevel3")
      }
  else {
    params += string(name: "PROJ_CONFIG", value: "ConfigDefault")
  }
  return params
}

def listConfigs(ci_scripts_dir, config_list, filter_group) {
  dir(ci_scripts_dir) {
    echo "Obtaining list of configs."
    echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}"
    def build_config_list_raw = sh(script: """\
python3 ./configs.py -g ${filter_group.replace(" ", " -g ")}
""", returnStdout: true).trim()
    def build_config_list = build_config_list_raw.tokenize('\n')
    config_list.addAll(build_config_list)
  }
}

def buildConfig(ci_scripts_dir, config, filter_group, results) {
  def params = []
  def params_collection = [:]
  def build_config_params
  dir(ci_scripts_dir) {
    echo "Obtaining build configuration for config ${config}"
    echo "Running: python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}"
    build_config_params = sh(script: """\
python3 ./configs.py -g ${filter_group.replace(" ", " -g ")} ${config}
""", returnStdout: true).trim()
  }
  def lines = build_config_params.tokenize('\n')
  for (String line : lines) {
    def key, value
    (key, value) = line.tokenize('=')
    params += string(name: key, value: value)
    params_collection[key] = value
  }
  params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
  params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
  params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
  params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
  params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
  params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
  params += string(name: 'CODE_REPO', value: env.CODE_REPO)
  params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
  return { -> results
    def build_res = build(job: 'tf-m-build-config', parameters: params, propagate: false)
    def build_info = [build_res, config, params_collection]
    results['builds'][build_res.number] = build_info
    def build_url = build_res.getAbsoluteUrl()
    print("${build_res.number}: ${config} ${build_res.result} ${build_url}")
    failure_states = ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]
    if (build_res.result in failure_states) {
      error("Build failed at ${build_url}")
    }
    else if (params_collection["NS"] == "False" || \
               (params_collection["PROFILE"] == "profile_medium" && \
               params_collection["PARTITION_PS"] == "OFF")) {
      print("LAVA is not needed for ${build_url}")
    }
    else {
      print("Doing LAVA stuff for ${build_url}")
      params += generateLavaParam(params_collection)
      params += string(name: 'BUILD_NUMBER', value: "${build_res.number}")
      params += string(name: 'BUILD_URL', value: build_url)
      params += string(name: 'LAVA_URL', value: env.LAVA_URL)
      params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
      params += string(name: 'LAVA_CREDENTIALS', value: env.LAVA_CREDENTIALS)
      def lava_res = build(job: 'tf-m-lava-submit', parameters: params, propagate: false)
      if (lava_res.result in failure_states) {
        error("LAVA Create and Submit failed at ${lava_res.getAbsoluteUrl()}")
      }
      else {
        results['lava_jobs'] += lava_res.getDescription()
      }
    }
  }
}

def buildDocs(results) {
  def params = []
  params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
  params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
  params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
  params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
  params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
  params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
  params += string(name: 'CODE_REPO', value: env.CODE_REPO)
  return { -> results
    def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false)
    print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}")
    results['docs'] = [res.number, res.result, params]
    if (res.result in ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]) {
      error("Build failed at ${res.getAbsoluteUrl()}")
    }
  }
}

def generateEmailBody(stage, failed_jobs) {
  body = "Check console output at ${env.BUILD_URL} \n\n"

  body += "Failed Jobs:\n"
  failed_jobs.each { job ->
    body += "${job.key}  ${job.value}\n"
  }

  body += "\nFor detailed ${stage} results please refer to \
           ${env.BUILD_URL}artifact/${stage}_results.csv \n"
  return body
}

def emailNotification(results, stage, failed_jobs) {
  script {
   if (env.JOB_NAME.equals("tf-m-nightly") && !env.EMAIL_NOTIFICATION.equals('')) {
      def result = "Fail."
      if (results == true) {
        result = "Success."
        print("Skip sending as ${result} for ${stage}")
      }
      else {
        emailext (
            subject: ("Job ${env.JOB_NAME} ${stage} ${env.BUILD_NUMBER} ${result}"),
            body: generateEmailBody(stage, failed_jobs),
            to: "${EMAIL_NOTIFICATION}"
        )
      }
    }
  } /* script */
}

def filterFailedBuild(results) {
  def failed_builds = [:]
  results.each { result ->
    if (result.value[0].getResult() == "FAILURE") {
      failed_builds[result.value[1]] = result.value[0].getAbsoluteUrl()
    }
  }
  return failed_builds
}

def filterFailedTest(string) {
  def failed_tests = [:]
  line = lineInString(string, "FAILURE_TESTS:")
  a = line.split(' ')
  if (a.size() > 1) {
    a = line.split(' ')[1..-1]
    a.each { fail_test ->
      config_link = fail_test.split(':')
      failed_tests[config_link[0]] = config_link[1..-1].join(':')
  }
  }
  return failed_tests
}

@NonCPS
def generateCsvContent(results) {
  def resultsParam = []
  results.each { result ->
    if (result.value[2]['BL2'] == "True") {
      resultsParam.add([result.value[1], \
                        result.value[0].getResult(), \
                        result.value[2]['TFM_PLATFORM'], \
                        result.value[2]['TOOLCHAIN_FILE'], \
                        result.value[2]['CMAKE_BUILD_TYPE'], \
                        result.value[2]['BL2'], \
                        result.value[2]['NS'], \
                        result.value[2]['PSA_API'], \
                        result.value[2]['ISOLATION_LEVEL'], \
                        result.value[2]['TEST_REGRESSION'], \
                        result.value[2]['TEST_PSA_API'], \
                        result.value[2]['PROFILE'], \
                        result.value[2]['PARTITION_PS'], \
                        result.value[2]['OTP']])
    }
  }
  resultsParam.each { result ->
    if (result[2] == 'musca_b1') {
      if (result[13] != 'off') {
        result[2] = 'musca_b1_OTP'
      }
    }
    result[3] = mapCompiler[result[3]]
    build_params = result[5..12]
    configName = ""
    for (map_cfg in mapConfigs) {
      if (build_params[0..6] == map_cfg[0..6]) {
        configName = map_cfg[7]
        break
      }
    }
    if (configName == "") {
      configName = "Default"
    }
    else if (configName == "RegressionProfileM") {
      if (build_params[7] == "OFF") {
        configName = "RegressionProfileM PSOFF"
      }
    }
    result.add(configName)
  }
  def csvContent = []
  resultsParam.each { result ->
    current_row = result[2..4]
    cfgs.each {cfg ->
      if (cfg == result[14]) {
        current_row.add(cfg)
        current_row.add(result[1])
      }
    }
    csvContent.add(current_row)
  }
  csvContent.sort{a,b -> a[0] <=> b[0] ?: a[1] <=> b[1] ?: a[2] <=> b[2]}
  build_summary = []
  current_platform = ""
  current_compiler = ""
  current_build_type = ""
  csvContent.each { build_cfg ->
    if (current_platform != build_cfg[0] || \
        current_compiler != build_cfg[1] || \
        current_build_type != build_cfg[2]) {
          current_platform = build_cfg[0]
          current_compiler = build_cfg[1]
          current_build_type = build_cfg[2]
          csv_line = [current_platform, current_compiler, current_build_type]
          cfgs.each {
            csv_line.add("N.A.")
          }
          build_summary.add(csv_line)
        }
    i = 0
    cfgs.each { cfg ->
      if (cfg == build_cfg[3]) {
        build_summary[-1][3+i] = build_cfg[4]
      }
      i += 1
    }
  }
  build_summary.add(0, ['Platform', 'Compiler', 'Cmake Build Type'])
  build_summary[0] += cfgs
  return build_summary
}

def generateBuildCsv(results) {
  def csvContent = generateCsvContent(results)
  node("master") {
    writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
    archiveArtifacts 'build_results.csv'
  }
}

def buildCsv(results) {
  def summary = new Summary();
  def csvContent = summary.getBuildCsv(results)
  node("master") {
    writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
    archiveArtifacts 'build_results.csv'
  }
}

def writeSummary(results) {
  def summary = new Summary();
  def buildLinks = summary.getLinks(results)
  node("master") {
    writeFile file: "build_links.html", text: buildLinks
    archiveArtifacts 'build_links.html'
  }
}

def lineInString(string, match) {
  def lines = string.split("\n")
  def result = lines.findAll { it.contains(match) }
  return result[0]
}

def getResult(string, match) {
  line = lineInString(string, match)
  a = line.split(match)[1].split(' ')
  score = a[0]
  if (a.size() > 1)
  {
    fail_text = a[1..-1].join(" ")
    return [score, fail_text]
  }
  return [score, ""]
}

def submitJobsToList(results) {
  def all_jobs = []
  for (String result : results){
    jobs_s = result.split('JOBS: ')
    if (jobs_s.size() > 1) {
      all_jobs += jobs_s[1]
    }
  }
  return(all_jobs)
}

def configs = []
def builds = [:]
def results = [:]

node("docker-amd64-tf-m-bionic") {
  stage("Init") {
    cleanWs()
    dir("tf-m-ci-scripts") {
      git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
    }
  }
  stage("Configs") {
    // Populate configs
    listConfigs('tf-m-ci-scripts', configs, env.FILTER_GROUP)
    results['builds'] = [:]
    results['lava_jobs'] = []
    for (config in configs) {
      builds[config] = buildConfig("tf-m-ci-scripts", config, env.FILTER_GROUP, results)
    }
    builds["docs"] = buildDocs(results)
  }
}

stage("Builds") {
  def verify = 1
  def success = true
  try {
    parallel(builds)
  } catch (Exception e) {
    print(e)
    manager.buildFailure()
    verify = -1
    success = false
  } finally {
    print("Verifying status")
    def failed_builds = filterFailedBuild(results['builds'])
    emailNotification(success, 'build', failed_builds)
    g = new Gerrit()
    g.verifyStatus(verify, 'tf-m-build', 'build')
    print("Building CSV")
    generateBuildCsv(results['builds'])
    writeSummary(results['builds'])
  }
}

node("docker-amd64-tf-m-bionic") {
  stage("Copy Docs") {
    if (env.JOB_NAME.equals("tf-m-build-and-test")) {
      step([$class: 'CopyArtifact', projectName: 'tf-m-build-docs',
      selector: specific("${results['docs'][0]}"), target: './docs/',
      optional: true])
      archiveArtifacts artifacts: 'docs/**', allowEmptyArchive: true
    }
    else {
      print("No doc copy for job: ${env.JOB_NAME}")
    }
  }
  stage("Tests") {
    dir("tf-m-ci-scripts") {
      git url: '$CI_SCRIPTS_REPO', branch: '$CI_SCRIPTS_BRANCH', credentialsId: 'GIT_SSH_KEY'
    }
    def all_jobs = []
    def success = true
    print("Wait for LAVA results here...")
    try {
      all_jobs = submitJobsToList(results['lava_jobs'])
      if (all_jobs.size() > 0) {
        dir("tf-m-ci-scripts") {
          withCredentials([usernamePassword(credentialsId: env.LAVA_CREDENTIALS, passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) {
            output = sh(script: """./lava_helper/lava_wait_jobs.py --job-ids ${all_jobs.join(",")} \
  --lava-url ${env.LAVA_URL} --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} \
  --artifacts-path lava_artifacts --lava-timeout 7200 \
  """, returnStdout: true).trim()
            archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true
            archiveArtifacts artifacts: 'test_results.csv', allowEmptyArchive: true
            g = new Gerrit()
            def (boot_result, boot_output) = getResult(output, 'BOOT_RESULT: ')
            if (boot_result) {
              g.verifyStatus(boot_result, "lava_boot", "test")
            }
            def (test_result, test_output) = getResult(output, 'TEST_RESULT: ')
            if (test_result) {
              g.verifyStatus(test_result, "lava_test", "test")
            }
            if (boot_result.toInteger() < 1 || test_result.toInteger() < 1) {
              error("Marking job as failed due to failed boots: ${boot_output} or tests: ${test_output}")
            }
          }
        }
      }
      else {
        print("There were no LAVA jobs to test.")
      }
    }
    catch (Exception e) {
      print("ERROR: ${e}")
      success = false
    } finally {
      archiveArtifacts artifacts: 'tf-m-ci-scripts/lava_artifacts/**', allowEmptyArchive: true
      emailNotification(success, 'test', filterFailedTest(output))
      cleanWs()
      if (!success) {
        error("There was an Error waiting for LAVA jobs")
      }
    }
  }
}