blob: 8b7841d10dd981621343d9bdaa77bade292a5637 [file] [log] [blame]
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +01001#!/usr/bin/env python3
2
3""" tfm_build_manager.py:
4
5 Controlling class managing multiple build configruations for tfm """
6
7from __future__ import print_function
8
9__copyright__ = """
10/*
Xinyu Zhang7c038b02021-04-20 10:27:49 +080011 * Copyright (c) 2018-2021, Arm Limited. All rights reserved.
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010012 *
13 * SPDX-License-Identifier: BSD-3-Clause
14 *
15 */
16 """
Karl Zhang08681e62020-10-30 13:56:03 +080017
18__author__ = "tf-m@lists.trustedfirmware.org"
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010019__project__ = "Trusted Firmware-M Open CI"
Xinyu Zhang06286a92021-07-22 14:00:51 +080020__version__ = "1.4.0"
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010021
22import os
23import sys
Karl Zhangaff558a2020-05-15 14:28:23 +010024from .utils import *
Minos Galanakisea421232019-06-20 17:11:28 +010025from time import time
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010026from copy import deepcopy
27from .utils import gen_cfg_combinations, list_chunks, load_json,\
Minos Galanakisea421232019-06-20 17:11:28 +010028 save_json, print_test, show_progress, \
29 resolve_rel_path
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010030from .structured_task import structuredTask
31from .tfm_builder import TFM_Builder
32
33
Xinyu Zhang38b76742021-11-11 13:57:56 +080034mapPlatform = {"cypress/psoc64": "psoc64",
35 "arm/mps2/an519": "AN519",
36 "arm/mps2/an521": "AN521",
37 "arm/mps2/an539": "AN539",
38 "arm/mps2/sse-200_aws": "SSE-200_AWS",
39 "arm/mps3/an524": "AN524",
40 "arm/musca_b1/sse_200": "MUSCA_B1",
41 "arm/musca_b1/secure_enclave": "MUSCA_B1_SE",
42 "arm/musca_s1": "MUSCA_S1",
43 "stm/stm32l562e_dk": "stm32l562e_dk",
44 "arm/corstone1000": "corstone1000",
45 "nxp/lpcxpresso55s69": "lpcxpresso55s69",
46 "arm/mps3/an547": "AN547",
47 "arm/mps3/corstone_polaris": "Polaris",
48 "lairdconnectivity/bl5340_dvk_cpuapp": "BL5340",
49 "nordic_nrf/nrf5340dk_nrf5340_cpuapp": "nrf5340dk",
50 "nordic_nrf/nrf9160dk_nrf9160": "nrf9160dk",
51 "nuvoton/m2351": "M2351",
52 "nuvoton/m2354": "M2354",
53 "stm/b_u585i_iot02a": "b_u585i_iot02a",
54 "stm/nucleo_l552ze_q": "nucleo_l552ze_q"}
Xinyu Zhang1078e812020-10-15 11:52:36 +080055
56mapCompiler = {"toolchain_GNUARM.cmake": "GNUARM",
57 "toolchain_ARMCLANG.cmake": "ARMCLANG"}
58
Xinyu Zhangc371af62020-10-21 10:41:57 +080059mapTestPsaApi = {"IPC": "FF",
Xinyu Zhang1078e812020-10-15 11:52:36 +080060 "CRYPTO": "CRYPTO",
Xinyu Zhang1078e812020-10-15 11:52:36 +080061 "INITIAL_ATTESTATION": "ATTEST",
Xinyu Zhang39acb412021-07-09 20:35:19 +080062 "STORAGE": "STORAGE"}
Xinyu Zhang1078e812020-10-15 11:52:36 +080063
Xinyu Zhang9fd74242020-10-22 11:30:50 +080064mapProfile = {"profile_small": "SMALL",
Xinyu Zhang9b1aef92021-03-12 15:36:44 +080065 "profile_medium": "MEDIUM",
66 "profile_large": "LARGE"}
Xinyu Zhang1078e812020-10-15 11:52:36 +080067
Feder Liang567e8c22021-10-26 14:16:21 +080068mapSFPOption = {"0": "SFP0",
69 "1": "SFP1",
70 "2": "SFP2"}
Xinyu Zhang1078e812020-10-15 11:52:36 +080071
Xinyu Zhangfd2e1152021-12-17 18:09:01 +080072mapExtraParams = {"": "",
73 "CRYPTO_OFF": ("-DTEST_S_CRYPTO=OFF "
74 "-DTEST_NS_CRYPTO=OFF "),
75 "NS_ATTEST_ON": ("-DTEST_S_CRYPTO=OFF "
76 "-DTEST_NS_CRYPTO=OFF "
77 "-DTEST_S_ITS=OFF "
78 "-DTEST_NS_ITS=OFF "
79 "-DTEST_S_PLATFORM=OFF "
80 "-DTEST_NS_PLATFORM=OFF "
81 "-DTEST_NS_ATTESTATION=ON "
82 "-DTEST_S_ATTESTATION=OFF "
83 "-DTEST_NS_QCBOR=OFF "
84 "-DTEST_S_QCBOR=OFF "
85 "-DTEST_NS_T_COSE=OFF "
86 "-DTEST_S_T_COSE=OFF "
87 "-DTEST_NS_PS=OFF "
88 "-DTEST_S_PS=OFF ")}
89
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +010090class TFM_Build_Manager(structuredTask):
91 """ Class that will load a configuration out of a json file, schedule
92 the builds, and produce a report """
93
94 def __init__(self,
95 tfm_dir, # TFM root directory
96 work_dir, # Current working directory(ie logs)
97 cfg_dict, # Input config dictionary of the following form
98 # input_dict = {"PROJ_CONFIG": "ConfigRegression",
99 # "TARGET_PLATFORM": "MUSCA_A",
100 # "COMPILER": "ARMCLANG",
101 # "CMAKE_BUILD_TYPE": "Debug"}
102 report=None, # File to produce report
103 parallel_builds=3, # Number of builds to run in parallel
Minos Galanakisea421232019-06-20 17:11:28 +0100104 build_threads=3, # Number of threads used per build
105 install=False, # Install libraries after build
106 img_sizes=False, # Use arm-none-eabi-size for size info
107 relative_paths=False): # Store relative paths in report
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100108 self._tbm_build_threads = build_threads
109 self._tbm_conc_builds = parallel_builds
110 self._tbm_install = install
Minos Galanakisea421232019-06-20 17:11:28 +0100111 self._tbm_img_sizes = img_sizes
112 self._tbm_relative_paths = relative_paths
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100113
114 # Required by other methods, always set working directory first
115 self._tbm_work_dir = os.path.abspath(os.path.expanduser(work_dir))
116
117 self._tbm_tfm_dir = os.path.abspath(os.path.expanduser(tfm_dir))
118
Karl Zhangaff558a2020-05-15 14:28:23 +0100119 print("bm param tfm_dir %s" % tfm_dir)
120 print("bm %s %s %s" % (work_dir, cfg_dict, self._tbm_work_dir))
Minos Galanakisea421232019-06-20 17:11:28 +0100121 # Internal flag to tag simple (non combination formatted configs)
122 self.simple_config = False
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100123 self._tbm_report = report
124
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100125 self._tbm_cfg = self.load_config(cfg_dict, self._tbm_work_dir)
Minos Galanakisea421232019-06-20 17:11:28 +0100126 self._tbm_build_cfg, \
127 self.tbm_common_cfg = self.parse_config(self._tbm_cfg)
Karl Zhangaff558a2020-05-15 14:28:23 +0100128 self._tfb_code_base_updated = False
129 self._tfb_log_f = "CodeBasePrepare.log"
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100130
131 super(TFM_Build_Manager, self).__init__(name="TFM_Build_Manager")
132
Dean Bircha6ede7e2020-03-13 14:00:33 +0000133 def get_config(self):
134 return list(self._tbm_build_cfg.keys())
Dean Birch5cb5a882020-01-24 11:37:13 +0000135
Dean Bircha6ede7e2020-03-13 14:00:33 +0000136 def print_config_environment(self, config, silence_stderr=False):
Dean Birch5cb5a882020-01-24 11:37:13 +0000137 """
138 For a given build configuration from output of print_config
139 method, print environment variables to build.
140 """
141 if config not in self._tbm_build_cfg:
Dean Bircha6ede7e2020-03-13 14:00:33 +0000142 if not silence_stderr:
143 print("Error: no such config {}".format(config), file=sys.stderr)
Dean Birch5cb5a882020-01-24 11:37:13 +0000144 sys.exit(1)
145 config_details = self._tbm_build_cfg[config]
146 argument_list = [
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000147 "CONFIG_NAME={}",
Xinyu Zhangb708f572020-09-15 11:43:46 +0800148 "TFM_PLATFORM={}",
149 "TOOLCHAIN_FILE={}",
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800150 "LIB_MODEL={}",
Xinyu Zhangb708f572020-09-15 11:43:46 +0800151 "ISOLATION_LEVEL={}",
152 "TEST_REGRESSION={}",
153 "TEST_PSA_API={}",
Dean Birch5cb5a882020-01-24 11:37:13 +0000154 "CMAKE_BUILD_TYPE={}",
Xinyu Zhangb708f572020-09-15 11:43:46 +0800155 "OTP={}",
Dean Birch5cb5a882020-01-24 11:37:13 +0000156 "BL2={}",
Xinyu Zhangb708f572020-09-15 11:43:46 +0800157 "NS={}",
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800158 "PROFILE={}",
Xinyu Zhang9bfe8a92021-10-28 16:27:12 +0800159 "PARTITION_PS={}",
Xinyu Zhanga1088e22021-11-11 18:02:45 +0800160 "NSCE={}",
Feder Liang567e8c22021-10-26 14:16:21 +0800161 "MMIO={}",
162 "FP={}",
Xinyu Zhangfd2e1152021-12-17 18:09:01 +0800163 "LAZY={}",
164 "EXTRA_PARAMS={}"
Dean Birch5cb5a882020-01-24 11:37:13 +0000165 ]
166 print(
167 "\n".join(argument_list)
168 .format(
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000169 config,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800170 config_details.tfm_platform,
171 config_details.toolchain_file,
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800172 config_details.lib_model,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800173 config_details.isolation_level,
174 config_details.test_regression,
175 config_details.test_psa_api,
Dean Birch5cb5a882020-01-24 11:37:13 +0000176 config_details.cmake_build_type,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800177 config_details.with_otp,
178 config_details.with_bl2,
179 config_details.with_ns,
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800180 "N.A" if not config_details.profile else config_details.profile,
Xinyu Zhang9bfe8a92021-10-28 16:27:12 +0800181 config_details.partition_ps,
Xinyu Zhanga1088e22021-11-11 18:02:45 +0800182 config_details.nsce,
Feder Liang567e8c22021-10-26 14:16:21 +0800183 config_details.mmio,
184 config_details.fp,
Xinyu Zhangfd2e1152021-12-17 18:09:01 +0800185 config_details.lazy,
186 "N.A" if not config_details.extra_params else config_details.extra_params,
Dean Birch5cb5a882020-01-24 11:37:13 +0000187 )
188 .strip()
189 )
190
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000191 def print_build_commands(self, config, silence_stderr=False):
192 config_details = self._tbm_build_cfg[config]
193 codebase_dir = os.path.join(os.getcwd(),"trusted-firmware-m")
Xinyu Zhangb708f572020-09-15 11:43:46 +0800194 build_dir=os.path.join(os.getcwd(),"trusted-firmware-m/build")
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000195 build_config = self.get_build_config(config_details, config, silence=silence_stderr, build_dir=build_dir, codebase_dir=codebase_dir)
Xinyu Zhang694eb492020-11-04 18:29:08 +0800196 build_commands = [build_config["config_template"]]
197 for command in build_config["build_cmds"]:
198 build_commands.append(command)
Xinyu Zhangb708f572020-09-15 11:43:46 +0800199 print(" ;\n".join(build_commands))
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000200
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100201 def pre_eval(self):
202 """ Tests that need to be run in set-up state """
203 return True
204
205 def pre_exec(self, eval_ret):
206 """ """
207
Minos Galanakisea421232019-06-20 17:11:28 +0100208 def override_tbm_cfg_params(self, config, override_keys, **params):
209 """ Using a dictionay as input, for each key defined in
210 override_keys it will replace the config[key] entries with
211 the key=value parameters provided """
212
213 for key in override_keys:
214 if isinstance(config[key], list):
215 config[key] = [n % params for n in config[key]]
216 elif isinstance(config[key], str):
217 config[key] = config[key] % params
218 else:
219 raise Exception("Config does not contain key %s "
220 "of type %s" % (key, config[key]))
221 return config
222
Karl Zhangaff558a2020-05-15 14:28:23 +0100223 def pre_build(self, build_cfg):
224 print("pre_build start %s \r\nself._tfb_cfg %s\r\n" %
225 (self, build_cfg))
226
227 try:
228 if self._tfb_code_base_updated:
229 print("Code base has been updated")
230 return True
231
232 self._tfb_code_base_updated = True
233
234 if "build_psa_api" in build_cfg:
235 # FF IPC build needs repo manifest update for TFM and PSA arch test
236 if "build_ff_ipc" in build_cfg:
237 print("Checkout to FF IPC code base")
238 os.chdir(build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
239 _api_test_manifest = "git checkout . ; python3 tools/scripts/manifest_update.py"
240 if subprocess_log(_api_test_manifest,
241 self._tfb_log_f,
242 append=True,
243 prefix=_api_test_manifest):
244
245 raise Exception("Python Failed please check log: %s" %
246 self._tfb_log_f)
247
248 _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py -m tools/tfm_psa_ff_test_manifest_list.yaml append"
249 os.chdir(build_cfg["codebase_root_dir"])
250 if subprocess_log(_api_test_manifest_tfm,
251 self._tfb_log_f,
252 append=True,
253 prefix=_api_test_manifest_tfm):
254
255 raise Exception("Python TFM Failed please check log: %s" %
256 self._tfb_log_f)
257 return True
258
259 print("Checkout to default code base")
260 os.chdir(build_cfg["codebase_root_dir"] + "/../psa-arch-tests/api-tests")
261 _api_test_manifest = "git checkout ."
262 if subprocess_log(_api_test_manifest,
263 self._tfb_log_f,
264 append=True,
265 prefix=_api_test_manifest):
266
267 raise Exception("Python Failed please check log: %s" %
268 self._tfb_log_f)
269
270 _api_test_manifest_tfm = "python3 tools/tfm_parse_manifest_list.py"
271 os.chdir(build_cfg["codebase_root_dir"])
272 if subprocess_log(_api_test_manifest_tfm,
273 self._tfb_log_f,
274 append=True,
275 prefix=_api_test_manifest_tfm):
276
277 raise Exception("Python TFM Failed please check log: %s" %
278 self._tfb_log_f)
279 finally:
280 print("python pass after builder prepare")
281 os.chdir(build_cfg["codebase_root_dir"] + "/../")
282
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100283 def task_exec(self):
284 """ Create a build pool and execute them in parallel """
285
286 build_pool = []
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100287
Minos Galanakisea421232019-06-20 17:11:28 +0100288 # When a config is flagged as a single build config.
289 # Name is evaluated by config type
290 if self.simple_config:
291
292 build_cfg = deepcopy(self.tbm_common_cfg)
293
294 # Extract the common for all elements of config
295 for key in ["build_cmds", "required_artefacts"]:
296 try:
297 build_cfg[key] = build_cfg[key]["all"]
298 except KeyError:
299 build_cfg[key] = []
300 name = build_cfg["config_type"]
301
302 # Override _tbm_xxx paths in commands
303 # plafrom in not guaranteed without seeds so _tbm_target_platform
304 # is ignored
305 over_dict = {"_tbm_build_dir_": os.path.join(self._tbm_work_dir,
306 name),
307 "_tbm_code_dir_": build_cfg["codebase_root_dir"]}
308
309 build_cfg = self.override_tbm_cfg_params(build_cfg,
310 ["build_cmds",
311 "required_artefacts",
312 "artifact_capture_rex"],
313 **over_dict)
314
315 # Overrides path in expected artefacts
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100316 print("Loading config %s" % name)
Minos Galanakisea421232019-06-20 17:11:28 +0100317
318 build_pool.append(TFM_Builder(
319 name=name,
320 work_dir=self._tbm_work_dir,
321 cfg_dict=build_cfg,
322 build_threads=self._tbm_build_threads,
323 img_sizes=self._tbm_img_sizes,
324 relative_paths=self._tbm_relative_paths))
325 # When a seed pool is provided iterate through the entries
326 # and update platform spefific parameters
327 elif len(self._tbm_build_cfg):
Karl Zhangaff558a2020-05-15 14:28:23 +0100328 print("\r\n_tbm_build_cfg %s\r\n tbm_common_cfg %s\r\n" \
329 % (self._tbm_build_cfg, self.tbm_common_cfg))
Minos Galanakisea421232019-06-20 17:11:28 +0100330 for name, i in self._tbm_build_cfg.items():
331 # Do not modify the original config
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000332 build_cfg = self.get_build_config(i, name)
Karl Zhangaff558a2020-05-15 14:28:23 +0100333 self.pre_build(build_cfg)
Minos Galanakisea421232019-06-20 17:11:28 +0100334 # Overrides path in expected artefacts
335 print("Loading config %s" % name)
336
337 build_pool.append(TFM_Builder(
338 name=name,
339 work_dir=self._tbm_work_dir,
340 cfg_dict=build_cfg,
341 build_threads=self._tbm_build_threads,
342 img_sizes=self._tbm_img_sizes,
343 relative_paths=self._tbm_relative_paths))
344 else:
345 print("Could not find any configuration. Check the rejection list")
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100346
347 status_rep = {}
Minos Galanakisea421232019-06-20 17:11:28 +0100348 build_rep = {}
349 completed_build_count = 0
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100350 print("Build: Running %d parallel build jobs" % self._tbm_conc_builds)
351 for build_pool_slice in list_chunks(build_pool, self._tbm_conc_builds):
352
353 # Start the builds
354 for build in build_pool_slice:
355 # Only produce output for the first build
356 if build_pool_slice.index(build) != 0:
357 build.mute()
358 print("Build: Starting %s" % build.get_name())
359 build.start()
360
361 # Wait for the builds to complete
362 for build in build_pool_slice:
363 # Wait for build to finish
364 build.join()
365 # Similarly print the logs of the other builds as they complete
366 if build_pool_slice.index(build) != 0:
367 build.log()
Minos Galanakisea421232019-06-20 17:11:28 +0100368 completed_build_count += 1
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100369 print("Build: Finished %s" % build.get_name())
Minos Galanakisea421232019-06-20 17:11:28 +0100370 print("Build Progress:")
371 show_progress(completed_build_count, len(build_pool))
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100372
373 # Store status in report
374 status_rep[build.get_name()] = build.get_status()
Minos Galanakisea421232019-06-20 17:11:28 +0100375 build_rep[build.get_name()] = build.report()
376
377 # Include the original input configuration in the report
378
379 metadata = {"input_build_cfg": self._tbm_cfg,
380 "build_dir": self._tbm_work_dir
381 if not self._tbm_relative_paths
382 else resolve_rel_path(self._tbm_work_dir),
383 "time": time()}
384
385 full_rep = {"report": build_rep,
386 "_metadata_": metadata}
387
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100388 # Store the report
389 self.stash("Build Status", status_rep)
390 self.stash("Build Report", full_rep)
391
392 if self._tbm_report:
393 print("Exported build report to file:", self._tbm_report)
394 save_json(self._tbm_report, full_rep)
395
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000396 def get_build_config(self, i, name, silence=False, build_dir=None, codebase_dir=None):
397 psa_build_dir = self._tbm_work_dir + "/" + name + "/BUILD"
398 if not build_dir:
399 build_dir = os.path.join(self._tbm_work_dir, name)
400 else:
401 psa_build_dir = os.path.join(build_dir, "../../psa-arch-tests/api-tests/build")
402 build_cfg = deepcopy(self.tbm_common_cfg)
403 if not codebase_dir:
404 codebase_dir = build_cfg["codebase_root_dir"]
405 else:
406 # Would prefer to do all with the new variable
407 # However, many things use this from build_cfg elsewhere
408 build_cfg["codebase_root_dir"] = codebase_dir
409 # Extract the common for all elements of config
410 for key in ["build_cmds", "required_artefacts"]:
411 try:
412 build_cfg[key] = deepcopy(self.tbm_common_cfg[key]
413 ["all"])
414 except KeyError as E:
415 build_cfg[key] = []
416 # Extract the platform specific elements of config
417 for key in ["build_cmds", "required_artefacts"]:
418 try:
Xinyu Zhang694eb492020-11-04 18:29:08 +0800419 if i.tfm_platform in self.tbm_common_cfg[key].keys() and i.with_ns:
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000420 build_cfg[key] += deepcopy(self.tbm_common_cfg[key]
Xinyu Zhangb708f572020-09-15 11:43:46 +0800421 [i.tfm_platform])
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000422 except Exception as E:
423 pass
Karl Zhang1eed6322020-07-01 15:38:10 +0800424
425 if os.cpu_count() >= 8:
426 #run in a serviver with scripts, parallel build will use CPU numbers
427 thread_no = " -j 2"
428 else:
429 #run in a docker, usually docker with CPUs less than 8
430 thread_no = " -j " + str(os.cpu_count())
Xinyu Zhangb708f572020-09-15 11:43:46 +0800431 build_cfg["build_cmds"][0] += thread_no
432 overwrite_params = {"codebase_root_dir": build_cfg["codebase_root_dir"],
433 "tfm_platform": i.tfm_platform,
434 "toolchain_file": i.toolchain_file,
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800435 "lib_model": i.lib_model,
Xinyu Zhangb708f572020-09-15 11:43:46 +0800436 "isolation_level": i.isolation_level,
437 "test_regression": i.test_regression,
438 "test_psa_api": i.test_psa_api,
439 "cmake_build_type": i.cmake_build_type,
440 "with_otp": i.with_otp,
441 "with_bl2": i.with_bl2,
442 "with_ns": i.with_ns,
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800443 "profile": "" if i.profile=="N.A" else i.profile,
Xinyu Zhang9bfe8a92021-10-28 16:27:12 +0800444 "partition_ps": i.partition_ps,
Xinyu Zhanga1088e22021-11-11 18:02:45 +0800445 "nsce": i.nsce,
Feder Liang567e8c22021-10-26 14:16:21 +0800446 "mmio": i.mmio,
447 "fp": i.fp,
Xinyu Zhangfd2e1152021-12-17 18:09:01 +0800448 "lazy": i.lazy,
449 "extra_params": mapExtraParams[i.extra_params]}
Xinyu Zhanga0086022020-11-10 18:11:12 +0800450 if i.test_psa_api == "IPC":
Xinyu Zhangcd1ed962020-11-11 16:00:52 +0800451 overwrite_params["test_psa_api"] += " -DINCLUDE_PANIC_TESTS=1"
Xinyu Zhang8cee3312021-11-12 11:06:39 +0800452 if i.tfm_platform == "arm/musca_b1/sse_200":
453 overwrite_params["test_psa_api"] += " -DITS_RAM_FS=ON -DPS_RAM_FS=ON"
Feder Liang567e8c22021-10-26 14:16:21 +0800454 if i.fp == "1" or i.fp == "2":
455 overwrite_params["test_psa_api"] += " -DTEST_S_FPU=ON -DTEST_NS_FPU=ON"
Xinyu Zhangb708f572020-09-15 11:43:46 +0800456 build_cfg["config_template"] %= overwrite_params
Xinyu Zhang694eb492020-11-04 18:29:08 +0800457 if len(build_cfg["build_cmds"]) > 1:
458 overwrite_build_dir = {"_tbm_build_dir_": build_dir}
459 build_cfg["build_cmds"][1] %= overwrite_build_dir
Dean Birchd0f9f8c2020-03-26 11:10:33 +0000460 return build_cfg
461
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100462 def post_eval(self):
463 """ If a single build failed fail the test """
464 try:
Minos Galanakisea421232019-06-20 17:11:28 +0100465 status_dict = self.unstash("Build Status")
466 if not status_dict:
467 raise Exception()
468 retcode_sum = sum(status_dict.values())
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100469 if retcode_sum != 0:
470 raise Exception()
471 return True
472 except Exception as e:
473 return False
474
475 def post_exec(self, eval_ret):
476 """ Generate a report and fail the script if build == unsuccessfull"""
477
478 self.print_summary()
479 if not eval_ret:
480 print("ERROR: ====> Build Failed! %s" % self.get_name())
481 self.set_status(1)
482 else:
483 print("SUCCESS: ====> Build Complete!")
484 self.set_status(0)
485
486 def get_report(self):
487 """ Expose the internal report to a new object for external classes """
488 return deepcopy(self.unstash("Build Report"))
489
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100490 def load_config(self, config, work_dir):
491 try:
492 # passing config_name param supersseeds fileparam
493 if isinstance(config, dict):
494 ret_cfg = deepcopy(config)
495 elif isinstance(config, str):
496 # If the string does not descrive a file try to look for it in
497 # work directory
498 if not os.path.isfile(config):
499 # remove path from file
500 config_2 = os.path.split(config)[-1]
501 # look in the current working directory
502 config_2 = os.path.join(work_dir, config_2)
503 if not os.path.isfile(config_2):
504 m = "Could not find cfg in %s or %s " % (config,
505 config_2)
506 raise Exception(m)
507 # If fille exists in working directory
508 else:
509 config = config_2
510 ret_cfg = load_json(config)
511
512 else:
513 raise Exception("Need to provide a valid config name or file."
514 "Please use --config/--config-file parameter.")
515 except Exception as e:
516 print("Error:%s \nCould not load a valid config" % e)
517 sys.exit(1)
518
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100519 return ret_cfg
520
521 def parse_config(self, cfg):
522 """ Parse a valid configuration file into a set of build dicts """
523
Minos Galanakisea421232019-06-20 17:11:28 +0100524 ret_cfg = {}
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100525
Minos Galanakisea421232019-06-20 17:11:28 +0100526 # Config entries which are not subject to changes during combinations
527 static_cfg = cfg["common_params"]
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100528
Minos Galanakisea421232019-06-20 17:11:28 +0100529 # Converth the code path to absolute path
530 abs_code_dir = static_cfg["codebase_root_dir"]
531 abs_code_dir = os.path.abspath(os.path.expanduser(abs_code_dir))
532 static_cfg["codebase_root_dir"] = abs_code_dir
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100533
Minos Galanakisea421232019-06-20 17:11:28 +0100534 # seed_params is an optional field. Do not proccess if it is missing
535 if "seed_params" in cfg:
536 comb_cfg = cfg["seed_params"]
537 # Generate a list of all possible confugration combinations
538 ret_cfg = TFM_Build_Manager.generate_config_list(comb_cfg,
539 static_cfg)
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100540
Xinyu Zhang2c63ce72021-07-23 14:01:59 +0800541 # valid is an optional field. Do not proccess if it is missing
542 if "valid" in cfg:
543 # Valid configurations(Need to build)
544 valid_cfg = cfg["valid"]
545 # Add valid configs to build list
546 ret_cfg.update(TFM_Build_Manager.generate_optional_list(
547 comb_cfg,
548 static_cfg,
549 valid_cfg))
550
Minos Galanakisea421232019-06-20 17:11:28 +0100551 # invalid is an optional field. Do not proccess if it is missing
552 if "invalid" in cfg:
553 # Invalid configurations(Do not build)
554 invalid_cfg = cfg["invalid"]
555 # Remove the rejected entries from the test list
Xinyu Zhang0581b082021-05-17 10:46:57 +0800556 rejection_cfg = TFM_Build_Manager.generate_optional_list(
Minos Galanakisea421232019-06-20 17:11:28 +0100557 comb_cfg,
558 static_cfg,
559 invalid_cfg)
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100560
Minos Galanakisea421232019-06-20 17:11:28 +0100561 # Subtract the two configurations
562 ret_cfg = {k: v for k, v in ret_cfg.items()
563 if k not in rejection_cfg}
564 self.simple_config = False
565 else:
566 self.simple_config = True
567 return ret_cfg, static_cfg
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100568
Minos Galanakisea421232019-06-20 17:11:28 +0100569 # ----- Override bellow methods when subclassing for other projects ----- #
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100570
Minos Galanakisea421232019-06-20 17:11:28 +0100571 def print_summary(self):
572 """ Print an comprehensive list of the build jobs with their status """
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100573
Minos Galanakisea421232019-06-20 17:11:28 +0100574 try:
575 full_rep = self.unstash("Build Report")["report"]
576 fl = ([k for k, v in full_rep.items() if v['status'] == 'Failed'])
577 ps = ([k for k, v in full_rep.items() if v['status'] == 'Success'])
578 except Exception as E:
Karl Zhangaff558a2020-05-15 14:28:23 +0100579 print("No report generated", E)
Minos Galanakisea421232019-06-20 17:11:28 +0100580 return
581 if fl:
582 print_test(t_list=fl, status="failed", tname="Builds")
583 if ps:
584 print_test(t_list=ps, status="passed", tname="Builds")
Minos Galanakisf4ca6ac2017-12-11 02:39:21 +0100585
Minos Galanakisea421232019-06-20 17:11:28 +0100586 @staticmethod
587 def generate_config_list(seed_config, static_config):
588 """ Generate all possible configuration combinations from a group of
589 lists of compiler options"""
590 config_list = []
591
592 if static_config["config_type"] == "tf-m":
593 cfg_name = "TFM_Build_CFG"
594 # Ensure the fieds are sorted in the desired order
595 # seed_config can be a subset of sort order for configurations with
596 # optional parameters.
597 tags = [n for n in static_config["sort_order"]
598 if n in seed_config.keys()]
Karl Zhangaff558a2020-05-15 14:28:23 +0100599 print("!!!!!!!!!!!gen list %s\r\n" % tags)
Minos Galanakisea421232019-06-20 17:11:28 +0100600
601 data = []
602 for key in tags:
603 data.append(seed_config[key])
604 config_list = gen_cfg_combinations(cfg_name,
605 " ".join(tags),
606 *data)
607 else:
608 print("Not information for project type: %s."
609 " Please check config" % static_config["config_type"])
610
611 ret_cfg = {}
612 # Notify the user for the rejected configuations
613 for i in config_list:
Xinyu Zhang1078e812020-10-15 11:52:36 +0800614 # Convert named tuples to string in a brief format
615 config_param = []
616 config_param.append(mapPlatform[list(i)[0]])
617 config_param.append(mapCompiler[list(i)[1]])
Xinyu Zhang73ed2992021-09-15 11:38:23 +0800618 if list(i)[2]: # LIB_MODEL
619 config_param.append("LIB")
620 else:
621 config_param.append("IPC")
Xinyu Zhang1078e812020-10-15 11:52:36 +0800622 config_param.append(list(i)[3]) # ISOLATION_LEVEL
623 if list(i)[4]: # TEST_REGRESSION
624 config_param.append("REG")
625 if list(i)[5] != "OFF": #TEST_PSA_API
626 config_param.append(mapTestPsaApi[list(i)[5]])
627 config_param.append(list(i)[6]) # BUILD_TYPE
Xinyu Zhanga50432e2020-10-23 18:00:18 +0800628 if list(i)[7] == "ENABLED": # OTP
Xinyu Zhang1078e812020-10-15 11:52:36 +0800629 config_param.append("OTP")
630 if list(i)[8]: # BL2
631 config_param.append("BL2")
632 if list(i)[9]: # NS
633 config_param.append("NS")
634 if list(i)[10]: # PROFILE
635 config_param.append(mapProfile[list(i)[10]])
Xinyu Zhang9fd74242020-10-22 11:30:50 +0800636 if list(i)[11] == "OFF": #PARTITION_PS
637 config_param.append("PSOFF")
Xinyu Zhang9bfe8a92021-10-28 16:27:12 +0800638 if list(i)[12] == "ON":
639 config_param.append("NSCE")
Xinyu Zhanga1088e22021-11-11 18:02:45 +0800640 if list(i)[13] == "ON":
641 config_param.append("MMIO")
Feder Liang567e8c22021-10-26 14:16:21 +0800642 if list(i)[14] == "1" or list(i)[14] == "2":
643 config_param.append(mapSFPOption[list(i)[14]]) #FP
644 if list(i)[15] == "ON": # LAZY
645 config_param.append("SLAZY")
Xinyu Zhangfd2e1152021-12-17 18:09:01 +0800646 if list(i)[16]: # EXTRA_PARAMS
647 config_param.append(list(i)[16])
Xinyu Zhang1078e812020-10-15 11:52:36 +0800648 i_str = "_".join(config_param)
Karl Zhangaff558a2020-05-15 14:28:23 +0100649 ret_cfg[i_str] = i
Minos Galanakisea421232019-06-20 17:11:28 +0100650 return ret_cfg
651
652 @staticmethod
Xinyu Zhang0581b082021-05-17 10:46:57 +0800653 def generate_optional_list(seed_config,
654 static_config,
655 optional_list):
656 optional_cfg = {}
Minos Galanakisea421232019-06-20 17:11:28 +0100657
658 if static_config["config_type"] == "tf-m":
659
Xinyu Zhang0581b082021-05-17 10:46:57 +0800660 # If optional list is empty do nothing
661 if not optional_list:
662 return optional_cfg
Minos Galanakisea421232019-06-20 17:11:28 +0100663
664 tags = [n for n in static_config["sort_order"]
665 if n in seed_config.keys()]
666 sorted_default_lst = [seed_config[k] for k in tags]
667
Xinyu Zhang0581b082021-05-17 10:46:57 +0800668 # If tags are not alligned with optional list entries quit
669 if len(tags) != len(optional_list[0]):
670 print(len(tags), len(optional_list[0]))
Minos Galanakisea421232019-06-20 17:11:28 +0100671 print("Error, tags should be assigned to each "
Xinyu Zhang0581b082021-05-17 10:46:57 +0800672 "of the optional inputs")
Minos Galanakisea421232019-06-20 17:11:28 +0100673 return []
674
675 # Replace wildcard ( "*") entries with every
676 # inluded in cfg variant
Xinyu Zhang0581b082021-05-17 10:46:57 +0800677 for k in optional_list:
Minos Galanakisea421232019-06-20 17:11:28 +0100678 # Pad the omitted values with wildcard char *
679 res_list = list(k) + ["*"] * (5 - len(k))
Xinyu Zhang0581b082021-05-17 10:46:57 +0800680 print("Working on optional input: %s" % (res_list))
Minos Galanakisea421232019-06-20 17:11:28 +0100681
682 for n in range(len(res_list)):
683
684 res_list[n] = [res_list[n]] if res_list[n] != "*" \
685 else sorted_default_lst[n]
686
687 # Generate a configuration and a name for the completed array
Xinyu Zhang0581b082021-05-17 10:46:57 +0800688 op_cfg = TFM_Build_Manager.generate_config_list(
Minos Galanakisea421232019-06-20 17:11:28 +0100689 dict(zip(tags, res_list)),
690 static_config)
691
692 # Append the configuration to the existing ones
Xinyu Zhang0581b082021-05-17 10:46:57 +0800693 optional_cfg = dict(optional_cfg, **op_cfg)
Minos Galanakisea421232019-06-20 17:11:28 +0100694
Xinyu Zhang0581b082021-05-17 10:46:57 +0800695 # Notify the user for the optional configuations
696 for i in optional_cfg.keys():
697 print("Generating optional config %s" % i)
Minos Galanakisea421232019-06-20 17:11:28 +0100698 else:
699 print("Not information for project type: %s."
700 " Please check config" % static_config["config_type"])
Xinyu Zhang0581b082021-05-17 10:46:57 +0800701 return optional_cfg