Open CI Scripts: Initial Commit
* build_helper: Python script which builds sets
of configurations from a json file input
* checkpatch: Bash scripts helping with running checkpatch
* cppcheck: Bash script helping with running cppcheck
* lava_helper: Python script which generates a lava job
definition and parses the output of a lava dispatcher
* tfm_ci_pylib: Generic Python module for Open CI
* configs: Directory storing reference configurations
Change-Id: Ibda0cbfeb5b004b35fef3c2af4cb5c012f2672b4
Signed-off-by: Galanakis, Minos <minos.galanakis@linaro.org>
diff --git a/tfm_ci_pylib/tfm_build_manager.py b/tfm_ci_pylib/tfm_build_manager.py
new file mode 100644
index 0000000..dcf75de
--- /dev/null
+++ b/tfm_ci_pylib/tfm_build_manager.py
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+
+""" tfm_build_manager.py:
+
+ Controlling class managing multiple build configruations for tfm """
+
+from __future__ import print_function
+
+__copyright__ = """
+/*
+ * Copyright (c) 2018-2019, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+__author__ = "Minos Galanakis"
+__email__ = "minos.galanakis@linaro.org"
+__project__ = "Trusted Firmware-M Open CI"
+__status__ = "stable"
+__version__ = "1.0"
+
+import os
+import sys
+from pprint import pprint
+from copy import deepcopy
+from .utils import gen_cfg_combinations, list_chunks, load_json,\
+ save_json, print_test
+from .structured_task import structuredTask
+from .tfm_builder import TFM_Builder
+
+
+class TFM_Build_Manager(structuredTask):
+ """ Class that will load a configuration out of a json file, schedule
+ the builds, and produce a report """
+
+ def __init__(self,
+ tfm_dir, # TFM root directory
+ work_dir, # Current working directory(ie logs)
+ cfg_dict, # Input config dictionary of the following form
+ # input_dict = {"PROJ_CONFIG": "ConfigRegression",
+ # "TARGET_PLATFORM": "MUSCA_A",
+ # "COMPILER": "ARMCLANG",
+ # "CMAKE_BUILD_TYPE": "Debug"}
+ report=None, # File to produce report
+ parallel_builds=3, # Number of builds to run in parallel
+ build_threads=4, # Number of threads used per build
+ markdown=True, # Create markdown report
+ html=True, # Create html report
+ ret_code=True, # Set ret_code of script if build failed
+ install=False): # Install libraries after build
+
+ self._tbm_build_threads = build_threads
+ self._tbm_conc_builds = parallel_builds
+ self._tbm_install = install
+ self._tbm_markdown = markdown
+ self._tbm_html = html
+ self._tbm_ret_code = ret_code
+
+ # Required by other methods, always set working directory first
+ self._tbm_work_dir = os.path.abspath(os.path.expanduser(work_dir))
+
+ self._tbm_tfm_dir = os.path.abspath(os.path.expanduser(tfm_dir))
+
+ # Entries will be filled after sanity test on cfg_dict dring pre_exec
+ self._tbm_build_dir = None
+ self._tbm_report = report
+
+ # TODO move them to pre_eval
+ self._tbm_cfg = self.load_config(cfg_dict, self._tbm_work_dir)
+ self._tbm_build_cfg_list = self.parse_config(self._tbm_cfg)
+
+ super(TFM_Build_Manager, self).__init__(name="TFM_Build_Manager")
+
+ def pre_eval(self):
+ """ Tests that need to be run in set-up state """
+ return True
+
+ def pre_exec(self, eval_ret):
+ """ """
+
+ def task_exec(self):
+ """ Create a build pool and execute them in parallel """
+
+ build_pool = []
+ for i in self._tbm_build_cfg_list:
+
+ name = "%s_%s_%s_%s_%s" % (i.TARGET_PLATFORM,
+ i.COMPILER,
+ i.PROJ_CONFIG,
+ i.CMAKE_BUILD_TYPE,
+ "BL2" if i.WITH_MCUBOOT else "NOBL2")
+ print("Loading config %s" % name)
+ build_pool.append(TFM_Builder(name,
+ self._tbm_tfm_dir,
+ self._tbm_work_dir,
+ dict(i._asdict()),
+ self._tbm_install,
+ self._tbm_build_threads))
+
+ status_rep = {}
+ full_rep = {}
+ print("Build: Running %d parallel build jobs" % self._tbm_conc_builds)
+ for build_pool_slice in list_chunks(build_pool, self._tbm_conc_builds):
+
+ # Start the builds
+ for build in build_pool_slice:
+ # Only produce output for the first build
+ if build_pool_slice.index(build) != 0:
+ build.mute()
+ print("Build: Starting %s" % build.get_name())
+ build.start()
+
+ # Wait for the builds to complete
+ for build in build_pool_slice:
+ # Wait for build to finish
+ build.join()
+ # Similarly print the logs of the other builds as they complete
+ if build_pool_slice.index(build) != 0:
+ build.log()
+ print("Build: Finished %s" % build.get_name())
+
+ # Store status in report
+ status_rep[build.get_name()] = build.get_status()
+ full_rep[build.get_name()] = build.report()
+ # Store the report
+ self.stash("Build Status", status_rep)
+ self.stash("Build Report", full_rep)
+
+ if self._tbm_report:
+ print("Exported build report to file:", self._tbm_report)
+ save_json(self._tbm_report, full_rep)
+
+ def post_eval(self):
+ """ If a single build failed fail the test """
+ try:
+ retcode_sum = sum(self.unstash("Build Status").values())
+ if retcode_sum != 0:
+ raise Exception()
+ return True
+ except Exception as e:
+ return False
+
+ def post_exec(self, eval_ret):
+ """ Generate a report and fail the script if build == unsuccessfull"""
+
+ self.print_summary()
+ if not eval_ret:
+ print("ERROR: ====> Build Failed! %s" % self.get_name())
+ self.set_status(1)
+ else:
+ print("SUCCESS: ====> Build Complete!")
+ self.set_status(0)
+
+ def get_report(self):
+ """ Expose the internal report to a new object for external classes """
+ return deepcopy(self.unstash("Build Report"))
+
+ def print_summary(self):
+ """ Print an comprehensive list of the build jobs with their status """
+
+ full_rep = self.unstash("Build Report")
+
+ # Filter out build jobs based on status
+ fl = ([k for k, v in full_rep.items() if v['status'] == 'Failed'])
+ ps = ([k for k, v in full_rep.items() if v['status'] == 'Success'])
+
+ print_test(t_list=fl, status="failed", tname="Builds")
+ print_test(t_list=ps, status="passed", tname="Builds")
+
+ def gen_cfg_comb(self, platform_l, compiler_l, config_l, build_l, boot_l):
+ """ Generate all possible configuration combinations from a group of
+ lists of compiler options"""
+ return gen_cfg_combinations("TFM_Build_CFG",
+ ("TARGET_PLATFORM COMPILER PROJ_CONFIG"
+ " CMAKE_BUILD_TYPE WITH_MCUBOOT"),
+ platform_l,
+ compiler_l,
+ config_l,
+ build_l,
+ boot_l)
+
+ def load_config(self, config, work_dir):
+ try:
+ # passing config_name param supersseeds fileparam
+ if isinstance(config, dict):
+ ret_cfg = deepcopy(config)
+ elif isinstance(config, str):
+ # If the string does not descrive a file try to look for it in
+ # work directory
+ if not os.path.isfile(config):
+ # remove path from file
+ config_2 = os.path.split(config)[-1]
+ # look in the current working directory
+ config_2 = os.path.join(work_dir, config_2)
+ if not os.path.isfile(config_2):
+ m = "Could not find cfg in %s or %s " % (config,
+ config_2)
+ raise Exception(m)
+ # If fille exists in working directory
+ else:
+ config = config_2
+ ret_cfg = load_json(config)
+
+ else:
+ raise Exception("Need to provide a valid config name or file."
+ "Please use --config/--config-file parameter.")
+ except Exception as e:
+ print("Error:%s \nCould not load a valid config" % e)
+ sys.exit(1)
+
+ pprint(ret_cfg)
+ return ret_cfg
+
+ def parse_config(self, cfg):
+ """ Parse a valid configuration file into a set of build dicts """
+
+ # Generate a list of all possible confugration combinations
+ full_cfg = self.gen_cfg_comb(cfg["platform"],
+ cfg["compiler"],
+ cfg["config"],
+ cfg["build"],
+ cfg["with_mcuboot"])
+
+ # Generate a list of all invalid combinations
+ rejection_cfg = []
+
+ for k in cfg["invalid"]:
+ # Pad the omitted values with wildcard char *
+ res_list = list(k) + ["*"] * (5 - len(k))
+
+ print("Working on rejection input: %s" % (res_list))
+
+ # Key order matters. Use index to retrieve default values When
+ # wildcard * char is present
+ _cfg_keys = ["platform",
+ "compiler",
+ "config",
+ "build",
+ "with_mcuboot"]
+
+ # Replace wildcard ( "*") entries with every inluded in cfg variant
+ for n in range(len(res_list)):
+ res_list[n] = [res_list[n]] if res_list[n] != "*" \
+ else cfg[_cfg_keys[n]]
+
+ rejection_cfg += self.gen_cfg_comb(*res_list)
+
+ # Notfy the user for the rejected configuations
+ for i in rejection_cfg:
+
+ name = "%s_%s_%s_%s_%s" % (i.TARGET_PLATFORM,
+ i.COMPILER,
+ i.PROJ_CONFIG,
+ i.CMAKE_BUILD_TYPE,
+ "BL2" if i.WITH_MCUBOOT else "NOBL2")
+ print("Rejecting config %s" % name)
+
+ # Subtract the two lists and convert to dictionary
+ return list(set(full_cfg) - set(rejection_cfg))