Various CI fixes
* Output CSV on build stage
* Check more failure states in pipeline
* Allow configs.py to use multiple groups
* Add build log as artifact
* Add links to particular build configs
* Host CMSIS_5 pack file internally
* Adding mbedcrypto url as a param
* Move the LAVA job generation into a new jenkins job
* Make job_ids strings for adding to job description
Change-Id: I801a1a5d15a7f55e25477ad371e8ec59eb14fd7f
Signed-off-by: Dean Birch <dean.birch@arm.com>
diff --git a/lava_helper/lava_submit_jobs.py b/lava_helper/lava_submit_jobs.py
new file mode 100755
index 0000000..b52bed1
--- /dev/null
+++ b/lava_helper/lava_submit_jobs.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+"""
+Script for submitting multiple LAVA definitions
+"""
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+import os
+import glob
+import sys
+import shutil
+import argparse
+from copy import deepcopy
+from collections import OrderedDict
+from jinja2 import Environment, FileSystemLoader
+from lava_helper_configs import *
+
+try:
+ from tfm_ci_pylib.utils import (
+ save_json,
+ load_json,
+ sort_dict,
+ load_yaml,
+ test,
+ print_test,
+ )
+ from tfm_ci_pylib.lava_rpc_connector import LAVA_RPC_connector
+except ImportError:
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ sys.path.append(os.path.join(dir_path, "../"))
+ from tfm_ci_pylib.utils import (
+ save_json,
+ load_json,
+ sort_dict,
+ load_yaml,
+ test,
+ print_test,
+ )
+ from tfm_ci_pylib.lava_rpc_connector import LAVA_RPC_connector
+
+
+def test_lava_dispatch_credentials(user_args):
+ """ Will validate if provided token/credentials are valid. It will return
+ a valid connection or exit program if not"""
+
+ # Collect the authentication tokens
+ try:
+ if user_args.token_from_env:
+ usr = os.environ['LAVA_USER']
+ secret = os.environ['LAVA_TOKEN']
+ elif user_args.lava_token and user_args.lava_user:
+ usr = user_args.lava_user
+ secret = user_args.lava_token
+
+ # Do not submit job without complete credentials
+ if not len(usr) or not len(secret):
+ raise Exception("Credentials not set")
+
+ lava = LAVA_RPC_connector(usr,
+ secret,
+ user_args.lava_url)
+
+ # Test the credentials againist the backend
+ if not lava.test_credentials():
+ raise Exception("Server rejected user authentication")
+ except Exception as e:
+ print("Credential validation failed with : %s" % e)
+ print("Did you set set --lava-token, --lava-user?")
+ sys.exit(1)
+ return lava
+
+def list_files_from_dir(user_args):
+ file_list = []
+ for filename in glob.iglob(user_args.job_dir + '**/*.yaml', recursive=True):
+ file_list.append(filename)
+ print("Found job {}".format(filename))
+ return file_list
+
+def lava_dispatch(user_args):
+ """ Submit a job to LAVA backend, block untill it is completed, and
+ fetch the results files if successful. If not, calls sys exit with 1
+ return code """
+
+ lava = test_lava_dispatch_credentials(user_args)
+ file_list = list_files_from_dir(user_args)
+ job_id_list = []
+ for job_file in file_list:
+ job_id, job_url = lava.submit_job(job_file)
+
+ # The reason of failure will be reported to user by LAVA_RPC_connector
+ if job_id is None and job_url is None:
+ print("Job failed")
+ else:
+ print("Job submitted at: " + job_url)
+ job_id_list.append(job_id)
+
+ print("\n".join(str(x) for x in job_id_list))
+
+def main(user_args):
+ lava_dispatch(user_args)
+
+
+def get_cmd_args():
+ """ Parse command line arguments """
+
+ # Parse command line arguments to override config
+ parser = argparse.ArgumentParser(description="Lava Create Jobs")
+ cmdargs = parser.add_argument_group("Create LAVA Jobs")
+
+ # Configuration control
+ cmdargs.add_argument(
+ "--lava-url", dest="lava_url", action="store", help="LAVA lab URL (without RPC2)"
+ )
+ cmdargs.add_argument(
+ "--job-dir", dest="job_dir", action="store", help="LAVA jobs directory"
+ )
+ cmdargs.add_argument(
+ "--lava-token", dest="lava_token", action="store", help="LAVA auth token"
+ )
+ cmdargs.add_argument(
+ "--lava-user", dest="lava_user", action="store", help="LAVA username"
+ )
+ cmdargs.add_argument(
+ "--use-env", dest="token_from_env", action="store_true", default=False, help="LAVA username"
+ )
+ cmdargs.add_argument(
+ "--lava-timeout", dest="dispatch_timeout", action="store", default=3600, help="LAVA username"
+ )
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ main(get_cmd_args())