blob: 451ef41b6c62e4656686ab0977a58498a8039407 [file] [log] [blame]
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +01001#!/usr/bin/env python3
2
3""" tfm_build_manager.py:
4
5 Controlling class managing multiple build configruations for tfm """
6
7from __future__ import print_function
Xinyu Zhang433771e2022-04-01 16:49:17 +08008from json import tool
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +01009
10__copyright__ = """
11/*
Feder Liang357b1602022-01-11 16:47:49 +080012 * Copyright (c) 2018-2022, Arm Limited. All rights reserved.
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010013 *
14 * SPDX-License-Identifier: BSD-3-Clause
15 *
16 */
17 """
Karl Zhang08681e62020-10-30 13:56:03 +080018
19__author__ = "tf-m@lists.trustedfirmware.org"
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010020__project__ = "Trusted Firmware-M Open CI"
Xinyu Zhang06286a92021-07-22 14:00:51 +080021__version__ = "1.4.0"
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010022
23import os
24import sys
Karl Zhangaff558a2020-05-15 14:28:23 +010025from .utils import *
Minos Galanakisea421232019-06-20 17:11:28 +010026from time import time
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010027from copy import deepcopy
28from .utils import gen_cfg_combinations, list_chunks, load_json,\
Minos Galanakisea421232019-06-20 17:11:28 +010029 save_json, print_test, show_progress, \
30 resolve_rel_path
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010031from .structured_task import structuredTask
32from .tfm_builder import TFM_Builder
33
34
Xinyu Zhang38b76742021-11-11 13:57:56 +080035mapPlatform = {"cypress/psoc64": "psoc64",
36 "arm/mps2/an519": "AN519",
37 "arm/mps2/an521": "AN521",
38 "arm/mps2/an539": "AN539",
39 "arm/mps2/sse-200_aws": "SSE-200_AWS",
40 "arm/mps3/an524": "AN524",
41 "arm/musca_b1/sse_200": "MUSCA_B1",
42 "arm/musca_b1/secure_enclave": "MUSCA_B1_SE",
43 "arm/musca_s1": "MUSCA_S1",
44 "stm/stm32l562e_dk": "stm32l562e_dk",
45 "arm/corstone1000": "corstone1000",
46 "nxp/lpcxpresso55s69": "lpcxpresso55s69",
47 "arm/mps3/an547": "AN547",
Bence Balogh8731a092022-05-24 17:24:54 +020048 "arm/mps3/corstone310_fvp" : "corstone310",
Bence Balogh176b78f2022-02-22 13:49:34 +010049 "arm/mps3/an552": "AN552",
Xinyu Zhang38b76742021-11-11 13:57:56 +080050 "lairdconnectivity/bl5340_dvk_cpuapp": "BL5340",
51 "nordic_nrf/nrf5340dk_nrf5340_cpuapp": "nrf5340dk",
52 "nordic_nrf/nrf9160dk_nrf9160": "nrf9160dk",
53 "nuvoton/m2351": "M2351",
54 "nuvoton/m2354": "M2354",
55 "stm/b_u585i_iot02a": "b_u585i_iot02a",
56 "stm/nucleo_l552ze_q": "nucleo_l552ze_q"}
Xinyu Zhang1078e812020-10-15 11:52:36 +080057
Xinyu Zhangc371af62020-10-21 10:41:57 +080058mapTestPsaApi = {"IPC": "FF",
Xinyu Zhang1078e812020-10-15 11:52:36 +080059 "CRYPTO": "CRYPTO",
Xinyu Zhang1078e812020-10-15 11:52:36 +080060 "INITIAL_ATTESTATION": "ATTEST",
Xinyu Zhang39acb412021-07-09 20:35:19 +080061 "STORAGE": "STORAGE"}
Xinyu Zhang1078e812020-10-15 11:52:36 +080062
Xinyu Zhang9fd74242020-10-22 11:30:50 +080063mapProfile = {"profile_small": "SMALL",
Xinyu Zhang9b1aef92021-03-12 15:36:44 +080064 "profile_medium": "MEDIUM",
65 "profile_large": "LARGE"}
Xinyu Zhang1078e812020-10-15 11:52:36 +080066
Xinyu Zhangfd2e1152021-12-17 18:09:01 +080067mapExtraParams = {"": "",
68 "CRYPTO_OFF": ("-DTEST_S_CRYPTO=OFF "
69 "-DTEST_NS_CRYPTO=OFF "),
Xinyu Zhang7c8d3372021-12-22 11:15:42 +080070 "CRYPTO_ON": ("-DTEST_S_CRYPTO=ON "
71 "-DTEST_NS_CRYPTO=ON "),
Xinyu Zhang3bb01af2021-12-20 14:45:49 +080072 "NSCE": "-DTFM_NS_MANAGE_NSID=ON ",
Feder Liang357b1602022-01-11 16:47:49 +080073 "MMIO": "-DPSA_FRAMEWORK_HAS_MM_IOVEC=ON ",
74 "FPSOFT": "-DCONFIG_TFM_FP=soft ",
75 "FPHARD": "-DCONFIG_TFM_FP=hard ",
76 "FPHARD_LOFF": ("-DCONFIG_TFM_FP=hard "
Xinyu Zhang6071f962022-02-07 15:56:39 +080077 "-DCONFIG_TFM_LAZY_STACKING=OFF "),
78 "FVP": "-DPLATFORM_IS_FVP=True",
Xinyu Zhang6d2dd7c2022-02-07 17:22:55 +080079 "FPGA": "-DPLATFORM_IS_FVP=False",
David Huda27ae72022-03-28 15:32:19 +080080 "CC_DRIVER_PSA": "-DCC312_LEGACY_DRIVER_API_ENABLED=OFF",
81 "SFN_ENABLE": "-DCONFIG_TFM_SPM_BACKEND=SFN"
Feder Liang357b1602022-01-11 16:47:49 +080082 }
Xinyu Zhangfd2e1152021-12-17 18:09:01 +080083
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010084class TFM_Build_Manager(structuredTask):
85 """ Class that will load a configuration out of a json file, schedule
86 the builds, and produce a report """
87
88 def __init__(self,
89 tfm_dir, # TFM root directory
90 work_dir, # Current working directory(ie logs)
91 cfg_dict, # Input config dictionary of the following form
92 # input_dict = {"PROJ_CONFIG": "ConfigRegression",
93 # "TARGET_PLATFORM": "MUSCA_A",
94 # "COMPILER": "ARMCLANG",
95 # "CMAKE_BUILD_TYPE": "Debug"}
96 report=None, # File to produce report
97 parallel_builds=3, # Number of builds to run in parallel
Minos Galanakisea421232019-06-20 17:11:28 +010098 build_threads=3, # Number of threads used per build
99 install=False, # Install libraries after build
100 img_sizes=False, # Use arm-none-eabi-size for size info
101 relative_paths=False): # Store relative paths in report
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100102 self._tbm_build_threads = build_threads
103 self._tbm_conc_builds = parallel_builds
104 self._tbm_install = install
Minos Galanakisea421232019-06-20 17:11:28 +0100105 self._tbm_img_sizes = img_sizes
106 self._tbm_relative_paths = relative_paths
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100107
108 # Required by other methods, always set working directory first
109 self._tbm_work_dir = os.path.abspath(os.path.expanduser(work_dir))
110
111 self._tbm_tfm_dir = os.path.abspath(os.path.expanduser(tfm_dir))
112
Karl Zhangaff558a2020-05-15 14:28:23 +0100113 print("bm param tfm_dir %s" % tfm_dir)
114 print("bm %s %s %s" % (work_dir, cfg_dict, self._tbm_work_dir))
Minos Galanakisea421232019-06-20 17:11:28 +0100115 # Internal flag to tag simple (non combination formatted configs)
116 self.simple_config = False
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100117 self._tbm_report = report
118
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100119 self._tbm_cfg = self.load_config(cfg_dict, self._tbm_work_dir)
Minos Galanakisea421232019-06-20 17:11:28 +0100120 self._tbm_build_cfg, \
121 self.tbm_common_cfg = self.parse_config(self._tbm_cfg)
Karl Zhangaff558a2020-05-15 14:28:23 +0100122 self._tfb_code_base_updated = False
123 self._tfb_log_f = "CodeBasePrepare.log"
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100124
125 super(TFM_Build_Manager, self).__init__(name="TFM_Build_Manager")
126
Xinyu Zhang433771e2022-04-01 16:49:17 +0800127 def choose_toolchain(self, compiler):
128 toolchain = ""
129 if "GCC"in compiler:
130 toolchain = "toolchain_GNUARM.cmake"
131 elif "ARMCLANG" in compiler:
132 toolchain = "toolchain_ARMCLANG.cmake"
Xinyu Zhangff5d7712022-01-14 13:48:59 +0800133
Xinyu Zhang433771e2022-04-01 16:49:17 +0800134 return toolchain
135
136 def get_compiler_name(self, compiler):
137 compiler_name = ""
138 if "GCC"in compiler:
139 compiler_name = "arm-none-eabi-gcc"
140 elif "ARMCLANG" in compiler:
141 compiler_name = "armclang"
142
143 return compiler_name
Xinyu Zhangff5d7712022-01-14 13:48:59 +0800144
Dean Bircha6ede7e2020-03-13 14:00:33 +0000145 def get_config(self):
146 return list(self._tbm_build_cfg.keys())
Dean Birch5cb5a882020-01-24 11:37:13 +0000147
Dean Bircha6ede7e2020-03-13 14:00:33 +0000148 def print_config_environment(self, config, silence_stderr=False):
Dean Birch5cb5a882020-01-24 11:37:13 +0000149 """
150 For a given build configuration from output of print_config
151 method, print environment variables to build.
152 """
153 if config not in self._tbm_build_cfg:
Dean Bircha6ede7e2020-03-13 14:00:33 +0000154 if not silence_stderr:
155 print("Error: no such config {}".format(config), file=sys.stderr)
Dean Birch5cb5a882020-01-24 11:37:13 +0000156 sys.exit(1)
157 config_details = self._tbm_build_cfg[config]
158 argument_list = [
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000159 "CONFIG_NAME={}",
Xinyu Zhangb708f572020-09-15 11:43:46 +0800160 "TFM_PLATFORM={}",
Xinyu Zhang433771e2022-04-01 16:49:17 +0800161 "COMPILER={}",
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800162 "LIB_MODEL={}",
Xinyu Zhangb708f572020-09-15 11:43:46 +0800163 "ISOLATION_LEVEL={}",
164 "TEST_REGRESSION={}",
165 "TEST_PSA_API={}",
Dean Birch5cb5a882020-01-24 11:37:13 +0000166 "CMAKE_BUILD_TYPE={}",
167 "BL2={}",
Xinyu Zhangb708f572020-09-15 11:43:46 +0800168 "NS={}",
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800169 "PROFILE={}",
Xinyu Zhang9bfe8a92021-10-28 16:27:12 +0800170 "PARTITION_PS={}",
Xinyu Zhangfd2e1152021-12-17 18:09:01 +0800171 "EXTRA_PARAMS={}"
Dean Birch5cb5a882020-01-24 11:37:13 +0000172 ]
173 print(
174 "\n".join(argument_list)
175 .format(
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000176 config,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800177 config_details.tfm_platform,
Xinyu Zhang433771e2022-04-01 16:49:17 +0800178 config_details.compiler,
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800179 config_details.lib_model,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800180 config_details.isolation_level,
181 config_details.test_regression,
182 config_details.test_psa_api,
Dean Birch5cb5a882020-01-24 11:37:13 +0000183 config_details.cmake_build_type,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800184 config_details.with_bl2,
185 config_details.with_ns,
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800186 "N.A" if not config_details.profile else config_details.profile,
Xinyu Zhang9bfe8a92021-10-28 16:27:12 +0800187 config_details.partition_ps,
Xinyu Zhangfd2e1152021-12-17 18:09:01 +0800188 "N.A" if not config_details.extra_params else config_details.extra_params,
Dean Birch5cb5a882020-01-24 11:37:13 +0000189 )
190 .strip()
191 )
192
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000193 def print_build_commands(self, config, silence_stderr=False):
194 config_details = self._tbm_build_cfg[config]
195 codebase_dir = os.path.join(os.getcwd(),"trusted-firmware-m")
Xinyu Zhangb708f572020-09-15 11:43:46 +0800196 build_dir=os.path.join(os.getcwd(),"trusted-firmware-m/build")
Xinyu Zhang433771e2022-04-01 16:49:17 +0800197 build_config = self.get_build_config(config_details, config, \
198 silence=silence_stderr, \
199 build_dir=build_dir, \
200 codebase_dir=codebase_dir)
201 build_commands = [build_config["set_compiler_path"], \
202 build_config["config_template"]]
Xinyu Zhang694eb492020-11-04 18:29:08 +0800203 for command in build_config["build_cmds"]:
204 build_commands.append(command)
Xinyu Zhangb708f572020-09-15 11:43:46 +0800205 print(" ;\n".join(build_commands))
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000206
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100207 def pre_eval(self):
208 """ Tests that need to be run in set-up state """
209 return True
210
211 def pre_exec(self, eval_ret):
212 """ """
213
Minos Galanakisea421232019-06-20 17:11:28 +0100214 def override_tbm_cfg_params(self, config, override_keys, **params):
215 """ Using a dictionay as input, for each key defined in
216 override_keys it will replace the config[key] entries with
217 the key=value parameters provided """
218
219 for key in override_keys:
220 if isinstance(config[key], list):
221 config[key] = [n % params for n in config[key]]
222 elif isinstance(config[key], str):
223 config[key] = config[key] % params
224 else:
225 raise Exception("Config does not contain key %s "
226 "of type %s" % (key, config[key]))
227 return config
228
Karl Zhangaff558a2020-05-15 14:28:23 +0100229 def pre_build(self, build_cfg):
230 print("pre_build start %s \r\nself._tfb_cfg %s\r\n" %
231 (self, build_cfg))
232
233 try:
234 if self._tfb_code_base_updated:
235 print("Code base has been updated")
236 return True
237
238 self._tfb_code_base_updated = True
239
240 if "build_psa_api" in build_cfg:
241 # FF IPC build needs repo manifest update for TFM and PSA arch test
242 if "build_ff_ipc" in build_cfg:
243 print("Checkout to FF IPC code base")
244 os.chdir(build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
245 _api_test_manifest = "git checkout . ; python3 tools/scripts/manifest_update.py"
246 if subprocess_log(_api_test_manifest,
247 self._tfb_log_f,
248 append=True,
249 prefix=_api_test_manifest):
250
251 raise Exception("Python Failed please check log: %s" %
252 self._tfb_log_f)
253
254 _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py -m tools/tfm_psa_ff_test_manifest_list.yaml append"
255 os.chdir(build_cfg["codebase_root_dir"])
256 if subprocess_log(_api_test_manifest_tfm,
257 self._tfb_log_f,
258 append=True,
259 prefix=_api_test_manifest_tfm):
260
261 raise Exception("Python TFM Failed please check log: %s" %
262 self._tfb_log_f)
263 return True
264
265 print("Checkout to default code base")
266 os.chdir(build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
267 _api_test_manifest = "git checkout ."
268 if subprocess_log(_api_test_manifest,
269 self._tfb_log_f,
270 append=True,
271 prefix=_api_test_manifest):
272
273 raise Exception("Python Failed please check log: %s" %
274 self._tfb_log_f)
275
276 _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py"
277 os.chdir(build_cfg["codebase_root_dir"])
278 if subprocess_log(_api_test_manifest_tfm,
279 self._tfb_log_f,
280 append=True,
281 prefix=_api_test_manifest_tfm):
282
283 raise Exception("Python TFM Failed please check log: %s" %
284 self._tfb_log_f)
285 finally:
286 print("python pass after builder prepare")
287 os.chdir(build_cfg["codebase_root_dir"] + "/../")
288
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100289 def task_exec(self):
290 """ Create a build pool and execute them in parallel """
291
292 build_pool = []
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100293
Minos Galanakisea421232019-06-20 17:11:28 +0100294 # When a config is flagged as a single build config.
295 # Name is evaluated by config type
296 if self.simple_config:
297
298 build_cfg = deepcopy(self.tbm_common_cfg)
299
300 # Extract the common for all elements of config
301 for key in ["build_cmds", "required_artefacts"]:
302 try:
303 build_cfg[key] = build_cfg[key]["all"]
304 except KeyError:
305 build_cfg[key] = []
306 name = build_cfg["config_type"]
307
308 # Override _tbm_xxx paths in commands
309 # plafrom in not guaranteed without seeds so _tbm_target_platform
310 # is ignored
311 over_dict = {"_tbm_build_dir_": os.path.join(self._tbm_work_dir,
312 name),
313 "_tbm_code_dir_": build_cfg["codebase_root_dir"]}
314
315 build_cfg = self.override_tbm_cfg_params(build_cfg,
316 ["build_cmds",
317 "required_artefacts",
318 "artifact_capture_rex"],
319 **over_dict)
320
321 # Overrides path in expected artefacts
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100322 print("Loading config %s" % name)
Minos Galanakisea421232019-06-20 17:11:28 +0100323
324 build_pool.append(TFM_Builder(
325 name=name,
326 work_dir=self._tbm_work_dir,
327 cfg_dict=build_cfg,
328 build_threads=self._tbm_build_threads,
329 img_sizes=self._tbm_img_sizes,
330 relative_paths=self._tbm_relative_paths))
331 # When a seed pool is provided iterate through the entries
332 # and update platform spefific parameters
333 elif len(self._tbm_build_cfg):
Karl Zhangaff558a2020-05-15 14:28:23 +0100334 print("\r\n_tbm_build_cfg %s\r\n tbm_common_cfg %s\r\n" \
335 % (self._tbm_build_cfg, self.tbm_common_cfg))
Minos Galanakisea421232019-06-20 17:11:28 +0100336 for name, i in self._tbm_build_cfg.items():
337 # Do not modify the original config
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000338 build_cfg = self.get_build_config(i, name)
Karl Zhangaff558a2020-05-15 14:28:23 +0100339 self.pre_build(build_cfg)
Minos Galanakisea421232019-06-20 17:11:28 +0100340 # Overrides path in expected artefacts
341 print("Loading config %s" % name)
342
343 build_pool.append(TFM_Builder(
344 name=name,
345 work_dir=self._tbm_work_dir,
346 cfg_dict=build_cfg,
347 build_threads=self._tbm_build_threads,
348 img_sizes=self._tbm_img_sizes,
349 relative_paths=self._tbm_relative_paths))
350 else:
351 print("Could not find any configuration. Check the rejection list")
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100352
353 status_rep = {}
Minos Galanakisea421232019-06-20 17:11:28 +0100354 build_rep = {}
355 completed_build_count = 0
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100356 print("Build: Running %d parallel build jobs" % self._tbm_conc_builds)
357 for build_pool_slice in list_chunks(build_pool, self._tbm_conc_builds):
358
359 # Start the builds
360 for build in build_pool_slice:
361 # Only produce output for the first build
362 if build_pool_slice.index(build) != 0:
363 build.mute()
364 print("Build: Starting %s" % build.get_name())
365 build.start()
366
367 # Wait for the builds to complete
368 for build in build_pool_slice:
369 # Wait for build to finish
370 build.join()
371 # Similarly print the logs of the other builds as they complete
372 if build_pool_slice.index(build) != 0:
373 build.log()
Minos Galanakisea421232019-06-20 17:11:28 +0100374 completed_build_count += 1
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100375 print("Build: Finished %s" % build.get_name())
Minos Galanakisea421232019-06-20 17:11:28 +0100376 print("Build Progress:")
377 show_progress(completed_build_count, len(build_pool))
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100378
379 # Store status in report
380 status_rep[build.get_name()] = build.get_status()
Minos Galanakisea421232019-06-20 17:11:28 +0100381 build_rep[build.get_name()] = build.report()
382
383 # Include the original input configuration in the report
384
385 metadata = {"input_build_cfg": self._tbm_cfg,
386 "build_dir": self._tbm_work_dir
387 if not self._tbm_relative_paths
388 else resolve_rel_path(self._tbm_work_dir),
389 "time": time()}
390
391 full_rep = {"report": build_rep,
392 "_metadata_": metadata}
393
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100394 # Store the report
395 self.stash("Build Status", status_rep)
396 self.stash("Build Report", full_rep)
397
398 if self._tbm_report:
399 print("Exported build report to file:", self._tbm_report)
400 save_json(self._tbm_report, full_rep)
401
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000402 def get_build_config(self, i, name, silence=False, build_dir=None, codebase_dir=None):
403 psa_build_dir = self._tbm_work_dir + "/" + name + "/BUILD"
404 if not build_dir:
405 build_dir = os.path.join(self._tbm_work_dir, name)
406 else:
407 psa_build_dir = os.path.join(build_dir, "../../psa-arch-tests/api-tests/build")
408 build_cfg = deepcopy(self.tbm_common_cfg)
409 if not codebase_dir:
410 codebase_dir = build_cfg["codebase_root_dir"]
411 else:
412 # Would prefer to do all with the new variable
413 # However, many things use this from build_cfg elsewhere
414 build_cfg["codebase_root_dir"] = codebase_dir
415 # Extract the common for all elements of config
416 for key in ["build_cmds", "required_artefacts"]:
417 try:
418 build_cfg[key] = deepcopy(self.tbm_common_cfg[key]
419 ["all"])
420 except KeyError as E:
421 build_cfg[key] = []
422 # Extract the platform specific elements of config
423 for key in ["build_cmds", "required_artefacts"]:
424 try:
Xinyu Zhang694eb492020-11-04 18:29:08 +0800425 if i.tfm_platform in self.tbm_common_cfg[key].keys() and i.with_ns:
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000426 build_cfg[key] += deepcopy(self.tbm_common_cfg[key]
Xinyu Zhangb708f572020-09-15 11:43:46 +0800427 [i.tfm_platform])
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000428 except Exception as E:
429 pass
Karl Zhang1eed6322020-07-01 15:38:10 +0800430
431 if os.cpu_count() >= 8:
432 #run in a serviver with scripts, parallel build will use CPU numbers
433 thread_no = " -j 2"
434 else:
435 #run in a docker, usually docker with CPUs less than 8
436 thread_no = " -j " + str(os.cpu_count())
Xinyu Zhangb708f572020-09-15 11:43:46 +0800437 build_cfg["build_cmds"][0] += thread_no
Xinyu Zhang433771e2022-04-01 16:49:17 +0800438
439 # Overwrite command lines to set compiler
440 build_cfg["set_compiler_path"] %= {"compiler": i.compiler}
441 build_cfg["set_compiler_path"] += " ;\n{} --version".format(self.get_compiler_name(i.compiler))
442
443 # Overwrite command lines of cmake
Xinyu Zhangb708f572020-09-15 11:43:46 +0800444 overwrite_params = {"codebase_root_dir": build_cfg["codebase_root_dir"],
445 "tfm_platform": i.tfm_platform,
Xinyu Zhang433771e2022-04-01 16:49:17 +0800446 "compiler": self.choose_toolchain(i.compiler),
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800447 "lib_model": i.lib_model,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800448 "isolation_level": i.isolation_level,
449 "test_regression": i.test_regression,
450 "test_psa_api": i.test_psa_api,
451 "cmake_build_type": i.cmake_build_type,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800452 "with_bl2": i.with_bl2,
453 "with_ns": i.with_ns,
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800454 "profile": "" if i.profile=="N.A" else i.profile,
Xinyu Zhang9bfe8a92021-10-28 16:27:12 +0800455 "partition_ps": i.partition_ps,
Xinyu Zhangfd2e1152021-12-17 18:09:01 +0800456 "extra_params": mapExtraParams[i.extra_params]}
Xinyu Zhanga0086022020-11-10 18:11:12 +0800457 if i.test_psa_api == "IPC":
Xinyu Zhangcd1ed962020-11-11 16:00:52 +0800458 overwrite_params["test_psa_api"] += " -DINCLUDE_PANIC_TESTS=1"
Xinyu Zhang5f9fa962022-04-12 16:54:35 +0800459 if i.test_psa_api == "CRYPTO" and "musca" in i.tfm_platform:
460 overwrite_params["test_psa_api"] += " -DCC312_LEGACY_DRIVER_API_ENABLED=OFF"
Xinyu Zhang6ac0eb02022-03-31 13:19:07 +0800461 if i.tfm_platform == "arm/musca_b1/sse_200":
462 overwrite_params["test_psa_api"] += " -DITS_RAM_FS=ON -DPS_RAM_FS=ON"
Feder Liang357b1602022-01-11 16:47:49 +0800463 if i.extra_params == "FPHARD" or i.extra_params == "FPHARD_LOFF":
Feder Liang567e8c22021-10-26 14:16:21 +0800464 overwrite_params["test_psa_api"] += " -DTEST_S_FPU=ON -DTEST_NS_FPU=ON"
Xinyu Zhangb708f572020-09-15 11:43:46 +0800465 build_cfg["config_template"] %= overwrite_params
Xinyu Zhang694eb492020-11-04 18:29:08 +0800466 if len(build_cfg["build_cmds"]) > 1:
467 overwrite_build_dir = {"_tbm_build_dir_": build_dir}
468 build_cfg["build_cmds"][1] %= overwrite_build_dir
Xinyu Zhang433771e2022-04-01 16:49:17 +0800469
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000470 return build_cfg
471
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100472 def post_eval(self):
473 """ If a single build failed fail the test """
474 try:
Minos Galanakisea421232019-06-20 17:11:28 +0100475 status_dict = self.unstash("Build Status")
476 if not status_dict:
477 raise Exception()
478 retcode_sum = sum(status_dict.values())
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100479 if retcode_sum != 0:
480 raise Exception()
481 return True
482 except Exception as e:
483 return False
484
485 def post_exec(self, eval_ret):
486 """ Generate a report and fail the script if build == unsuccessfull"""
487
488 self.print_summary()
489 if not eval_ret:
490 print("ERROR: ====> Build Failed! %s" % self.get_name())
491 self.set_status(1)
492 else:
493 print("SUCCESS: ====> Build Complete!")
494 self.set_status(0)
495
496 def get_report(self):
497 """ Expose the internal report to a new object for external classes """
498 return deepcopy(self.unstash("Build Report"))
499
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100500 def load_config(self, config, work_dir):
501 try:
502 # passing config_name param supersseeds fileparam
503 if isinstance(config, dict):
504 ret_cfg = deepcopy(config)
505 elif isinstance(config, str):
506 # If the string does not descrive a file try to look for it in
507 # work directory
508 if not os.path.isfile(config):
509 # remove path from file
510 config_2 = os.path.split(config)[-1]
511 # look in the current working directory
512 config_2 = os.path.join(work_dir, config_2)
513 if not os.path.isfile(config_2):
514 m = "Could not find cfg in %s or %s " % (config,
515 config_2)
516 raise Exception(m)
517 # If fille exists in working directory
518 else:
519 config = config_2
520 ret_cfg = load_json(config)
521
522 else:
523 raise Exception("Need to provide a valid config name or file."
524 "Please use --config/--config-file parameter.")
525 except Exception as e:
526 print("Error:%s \nCould not load a valid config" % e)
527 sys.exit(1)
528
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100529 return ret_cfg
530
531 def parse_config(self, cfg):
532 """ Parse a valid configuration file into a set of build dicts """
533
Minos Galanakisea421232019-06-20 17:11:28 +0100534 ret_cfg = {}
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100535
Minos Galanakisea421232019-06-20 17:11:28 +0100536 # Config entries which are not subject to changes during combinations
537 static_cfg = cfg["common_params"]
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100538
Minos Galanakisea421232019-06-20 17:11:28 +0100539 # Converth the code path to absolute path
540 abs_code_dir = static_cfg["codebase_root_dir"]
541 abs_code_dir = os.path.abspath(os.path.expanduser(abs_code_dir))
542 static_cfg["codebase_root_dir"] = abs_code_dir
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100543
Minos Galanakisea421232019-06-20 17:11:28 +0100544 # seed_params is an optional field. Do not proccess if it is missing
545 if "seed_params" in cfg:
546 comb_cfg = cfg["seed_params"]
547 # Generate a list of all possible confugration combinations
548 ret_cfg = TFM_Build_Manager.generate_config_list(comb_cfg,
549 static_cfg)
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100550
Xinyu Zhang2c63ce72021-07-23 14:01:59 +0800551 # valid is an optional field. Do not proccess if it is missing
552 if "valid" in cfg:
553 # Valid configurations(Need to build)
554 valid_cfg = cfg["valid"]
555 # Add valid configs to build list
556 ret_cfg.update(TFM_Build_Manager.generate_optional_list(
557 comb_cfg,
558 static_cfg,
559 valid_cfg))
560
Minos Galanakisea421232019-06-20 17:11:28 +0100561 # invalid is an optional field. Do not proccess if it is missing
562 if "invalid" in cfg:
563 # Invalid configurations(Do not build)
564 invalid_cfg = cfg["invalid"]
565 # Remove the rejected entries from the test list
Xinyu Zhang0581b082021-05-17 10:46:57 +0800566 rejection_cfg = TFM_Build_Manager.generate_optional_list(
Minos Galanakisea421232019-06-20 17:11:28 +0100567 comb_cfg,
568 static_cfg,
569 invalid_cfg)
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100570
Minos Galanakisea421232019-06-20 17:11:28 +0100571 # Subtract the two configurations
572 ret_cfg = {k: v for k, v in ret_cfg.items()
573 if k not in rejection_cfg}
574 self.simple_config = False
575 else:
576 self.simple_config = True
577 return ret_cfg, static_cfg
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100578
Minos Galanakisea421232019-06-20 17:11:28 +0100579 # ----- Override bellow methods when subclassing for other projects ----- #
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100580
Minos Galanakisea421232019-06-20 17:11:28 +0100581 def print_summary(self):
582 """ Print an comprehensive list of the build jobs with their status """
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100583
Minos Galanakisea421232019-06-20 17:11:28 +0100584 try:
585 full_rep = self.unstash("Build Report")["report"]
586 fl = ([k for k, v in full_rep.items() if v['status'] == 'Failed'])
587 ps = ([k for k, v in full_rep.items() if v['status'] == 'Success'])
588 except Exception as E:
Karl Zhangaff558a2020-05-15 14:28:23 +0100589 print("No report generated", E)
Minos Galanakisea421232019-06-20 17:11:28 +0100590 return
591 if fl:
592 print_test(t_list=fl, status="failed", tname="Builds")
593 if ps:
594 print_test(t_list=ps, status="passed", tname="Builds")
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100595
Minos Galanakisea421232019-06-20 17:11:28 +0100596 @staticmethod
597 def generate_config_list(seed_config, static_config):
598 """ Generate all possible configuration combinations from a group of
599 lists of compiler options"""
600 config_list = []
601
602 if static_config["config_type"] == "tf-m":
603 cfg_name = "TFM_Build_CFG"
604 # Ensure the fieds are sorted in the desired order
605 # seed_config can be a subset of sort order for configurations with
606 # optional parameters.
607 tags = [n for n in static_config["sort_order"]
608 if n in seed_config.keys()]
Karl Zhangaff558a2020-05-15 14:28:23 +0100609 print("!!!!!!!!!!!gen list %s\r\n" % tags)
Minos Galanakisea421232019-06-20 17:11:28 +0100610
611 data = []
612 for key in tags:
613 data.append(seed_config[key])
614 config_list = gen_cfg_combinations(cfg_name,
615 " ".join(tags),
616 *data)
617 else:
618 print("Not information for project type: %s."
619 " Please check config" % static_config["config_type"])
620
621 ret_cfg = {}
622 # Notify the user for the rejected configuations
623 for i in config_list:
Xinyu Zhang1078e812020-10-15 11:52:36 +0800624 # Convert named tuples to string in a brief format
625 config_param = []
626 config_param.append(mapPlatform[list(i)[0]])
Xinyu Zhang433771e2022-04-01 16:49:17 +0800627 config_param.append(list(i)[1].split("_")[0])
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800628 if list(i)[2]: # LIB_MODEL
629 config_param.append("LIB")
630 else:
631 config_param.append("IPC")
Xinyu Zhang1078e812020-10-15 11:52:36 +0800632 config_param.append(list(i)[3]) # ISOLATION_LEVEL
633 if list(i)[4]: # TEST_REGRESSION
634 config_param.append("REG")
635 if list(i)[5] != "OFF": #TEST_PSA_API
636 config_param.append(mapTestPsaApi[list(i)[5]])
637 config_param.append(list(i)[6]) # BUILD_TYPE
Xinyu Zhang589fd052022-04-19 17:54:16 +0800638 if list(i)[7]: # BL2
Xinyu Zhang1078e812020-10-15 11:52:36 +0800639 config_param.append("BL2")
Xinyu Zhang589fd052022-04-19 17:54:16 +0800640 if list(i)[8]: # NS
Xinyu Zhang1078e812020-10-15 11:52:36 +0800641 config_param.append("NS")
Xinyu Zhang589fd052022-04-19 17:54:16 +0800642 if list(i)[9]: # PROFILE
643 config_param.append(mapProfile[list(i)[9]])
644 if list(i)[10] == "OFF": #PARTITION_PS
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800645 config_param.append("PSOFF")
Xinyu Zhang589fd052022-04-19 17:54:16 +0800646 if list(i)[11]: # EXTRA_PARAMS
647 config_param.append(list(i)[11])
Xinyu Zhang1078e812020-10-15 11:52:36 +0800648 i_str = "_".join(config_param)
Karl Zhangaff558a2020-05-15 14:28:23 +0100649 ret_cfg[i_str] = i
Minos Galanakisea421232019-06-20 17:11:28 +0100650 return ret_cfg
651
652 @staticmethod
Xinyu Zhang0581b082021-05-17 10:46:57 +0800653 def generate_optional_list(seed_config,
654 static_config,
655 optional_list):
656 optional_cfg = {}
Minos Galanakisea421232019-06-20 17:11:28 +0100657
658 if static_config["config_type"] == "tf-m":
659
Xinyu Zhang0581b082021-05-17 10:46:57 +0800660 # If optional list is empty do nothing
661 if not optional_list:
662 return optional_cfg
Minos Galanakisea421232019-06-20 17:11:28 +0100663
664 tags = [n for n in static_config["sort_order"]
665 if n in seed_config.keys()]
666 sorted_default_lst = [seed_config[k] for k in tags]
667
Xinyu Zhang0581b082021-05-17 10:46:57 +0800668 # If tags are not alligned with optional list entries quit
669 if len(tags) != len(optional_list[0]):
670 print(len(tags), len(optional_list[0]))
Minos Galanakisea421232019-06-20 17:11:28 +0100671 print("Error, tags should be assigned to each "
Xinyu Zhang0581b082021-05-17 10:46:57 +0800672 "of the optional inputs")
Minos Galanakisea421232019-06-20 17:11:28 +0100673 return []
674
675 # Replace wildcard ( "*") entries with every
676 # inluded in cfg variant
Xinyu Zhang0581b082021-05-17 10:46:57 +0800677 for k in optional_list:
Minos Galanakisea421232019-06-20 17:11:28 +0100678 # Pad the omitted values with wildcard char *
679 res_list = list(k) + ["*"] * (5 - len(k))
Xinyu Zhang0581b082021-05-17 10:46:57 +0800680 print("Working on optional input: %s" % (res_list))
Minos Galanakisea421232019-06-20 17:11:28 +0100681
682 for n in range(len(res_list)):
683
684 res_list[n] = [res_list[n]] if res_list[n] != "*" \
685 else sorted_default_lst[n]
686
687 # Generate a configuration and a name for the completed array
Xinyu Zhang0581b082021-05-17 10:46:57 +0800688 op_cfg = TFM_Build_Manager.generate_config_list(
Minos Galanakisea421232019-06-20 17:11:28 +0100689 dict(zip(tags, res_list)),
690 static_config)
691
692 # Append the configuration to the existing ones
Xinyu Zhang0581b082021-05-17 10:46:57 +0800693 optional_cfg = dict(optional_cfg, **op_cfg)
Minos Galanakisea421232019-06-20 17:11:28 +0100694
Xinyu Zhang0581b082021-05-17 10:46:57 +0800695 # Notify the user for the optional configuations
696 for i in optional_cfg.keys():
697 print("Generating optional config %s" % i)
Minos Galanakisea421232019-06-20 17:11:28 +0100698 else:
699 print("Not information for project type: %s."
700 " Please check config" % static_config["config_type"])
Xinyu Zhang0581b082021-05-17 10:46:57 +0800701 return optional_cfg