blob: 9ef6f8109d48307aadd1643e30d05086e92320fb [file] [log] [blame]
Basil Eljuse4b14afb2020-09-30 13:07:23 +01001# !/usr/bin/env python
2###############################################################################
Jelle Sels83f141e2022-08-01 15:17:40 +00003# Copyright (c) 2020-2022, ARM Limited and Contributors. All rights reserved.
Basil Eljuse4b14afb2020-09-30 13:07:23 +01004#
5# SPDX-License-Identifier: BSD-3-Clause
6###############################################################################
7
8###############################################################################
9# FILE: intermediate_layer.py
10#
11# DESCRIPTION: Creates an intermediate json file with information provided
12# by the configuration json file, dwarf signatures and trace
13# files.
14#
15###############################################################################
16
17import os
18import re
19import glob
20import argparse
21import subprocess
22import json
23from argparse import RawTextHelpFormatter
24import logging
25import time
Saul Romero884d2142023-01-16 10:31:22 +000026from typing import Dict
27from typing import List
Basil Eljuse4b14afb2020-09-30 13:07:23 +010028
Saul Romero884d2142023-01-16 10:31:22 +000029__version__ = "7.0"
Basil Eljuse4b14afb2020-09-30 13:07:23 +010030
31# Static map that defines the elf file source type in the intermediate json
32ELF_MAP = {
33 "bl1": 0,
34 "bl2": 1,
35 "bl31": 2,
36 "bl32": 3,
37 "scp_ram": 10,
38 "scp_rom": 11,
39 "mcp_rom": 12,
40 "mcp_ram": 13,
Saul Romero884d2142023-01-16 10:31:22 +000041 "secure_hafnium": 14,
42 "hafium": 15,
Basil Eljuse4b14afb2020-09-30 13:07:23 +010043 "custom_offset": 100
44}
45
46
47def os_command(command, show_command=False):
48 """
49 Function that execute an os command, on fail exit the program
50
51 :param command: OS command as string
52 :param show_command: Optional argument to print the command in stdout
53 :return: The string output of the os command
54 """
Basil Eljuse4b14afb2020-09-30 13:07:23 +010055 try:
56 if show_command:
57 print("OS command: {}".format(command))
58 out = subprocess.check_output(
59 command, stderr=subprocess.STDOUT, shell=True)
60 except subprocess.CalledProcessError as ex:
61 raise Exception(
62 "Exception running command '{}': {}({})".format(
63 command, ex.output, ex.returncode))
64 return out.decode("utf8")
65
66
67def load_stats_from_traces(trace_globs):
68 """
69 Function to process and consolidate statistics from trace files
70
71 :param trace_globs: List of trace file patterns
72 :return: Dictionary with stats from trace files i.e.
73 {mem address in decimal}=(times executed, inst size)
74 """
75 stats = {}
76 stat_size = {}
77
78 # Make a list of unique trace files
79 trace_files = []
80 for tg in trace_globs:
81 trace_files.extend(glob.glob(tg))
82 trace_files = set(trace_files)
83
84 if not trace_files:
85 raise Exception("No trace files found for '{}'".format(trace_globs))
86 # Load stats from the trace files
87 for trace_file in trace_files:
88 try:
89 with open(trace_file, 'r') as f:
90 for line in f:
91 data = line.split()
92 address = int(data[0], 16)
93 stat = int(data[1])
94 size = int(data[2])
95 stat_size[address] = size
96 if address in stats:
97 stats[address] += stat
98 else:
99 stats[address] = stat
100 except Exception as ex:
101 logger.error("@Loading stats from trace files:{}".format(ex))
102 # Merge the two dicts
103 for address in stats:
104 stats[address] = (stats[address], stat_size[address])
105 return stats
106
107
108def get_code_sections_for_binary(elf_name):
109 """
110 Function to return the ranges of memory address for sections of code
111 in the elf file
112
113 :param elf_name: Elf binary file name
114 :return: List of code sections tuples, i.e. (section type, initial
115 address, end address)
116 """
117 command = """%s -h %s | grep -B 1 CODE | grep -v CODE \
118 | awk '{print $2" "$4" "$3}'""" % (OBJDUMP, elf_name)
119 text_out = os_command(command)
120 sections = text_out.split('\n')
121 sections.pop()
122 secs = []
123 for sec in sections:
124 try:
125 d = sec.split()
126 secs.append((d[0], int(d[1], 16), int(d[2], 16)))
127 except Exception as ex:
128 logger.error(
129 "@Returning memory address code sections:".format(ex))
130 return secs
131
132
133def get_executable_ranges_for_binary(elf_name):
134 """
135 Get function ranges from an elf file
136
137 :param elf_name: Elf binary file name
138 :return: List of tuples for ranges i.e. (range start, range end)
139 """
140 # Parse all $x / $d symbols
141 symbol_table = []
Saul Romero884d2142023-01-16 10:31:22 +0000142 address = None
143 _type = None
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100144 command = r"""%s -s %s | awk '/\$[xatd]/ {print $2" "$8}'""" % (
145 READELF, elf_name)
146 text_out = os_command(command)
147 lines = text_out.split('\n')
148 lines.pop()
149 for line in lines:
150 try:
151 data = line.split()
152 address = int(data[0], 16)
153 _type = 'X' if data[1] in ['$x', '$t', '$a'] else 'D'
154 except Exception as ex:
155 logger.error("@Getting executable ranges:".format(ex))
156 symbol_table.append((address, _type))
157
158 # Add markers for end of code sections
159 sections = get_code_sections_for_binary(elf_name)
160 for sec in sections:
161 symbol_table.append((sec[1] + sec[2], 'S'))
162
163 # Sort by address
164 symbol_table = sorted(symbol_table, key=lambda tup: tup[0])
165
166 # Create ranges (list of START/END tuples)
167 ranges = []
168 range_start = symbol_table[0][0]
169 rtype = symbol_table[0][1]
170 for sym in symbol_table:
171 if sym[1] != rtype:
172 if rtype == 'X':
Saul Romero884d2142023-01-16 10:31:22 +0000173 # Subtract one because the first address of the
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100174 # next range belongs to the next range.
175 ranges.append((range_start, sym[0] - 1))
176 range_start = sym[0]
177 rtype = sym[1]
178 return ranges
179
180
Saul Romero884d2142023-01-16 10:31:22 +0000181def list_of_functions_for_binary(elf_name: str) -> Dict[str, Dict[str, any]]:
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100182 """
183 Get an array of the functions in the elf file
184
185 :param elf_name: Elf binary file name
186 :return: An array of function address start, function address end,
Saul Romero884d2142023-01-16 10:31:22 +0000187 function dwarf signature (sources) indexed by function name
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100188 """
189 _functions = {}
190 command = "%s -t %s | awk 'NR>4' | sed /^$/d" % (OBJDUMP, elf_name)
191 symbols_output = os_command(command)
192 rex = r'([0-9a-fA-F]+) (.{7}) ([^ ]+)[ \t]([0-9a-fA-F]+) (.*)'
193 symbols = symbols_output.split('\n')[:-1]
194 for sym in symbols:
195 try:
196 symbol_details = re.findall(rex, sym)
197 symbol_details = symbol_details[0]
198 if 'F' not in symbol_details[1]:
199 continue
200 function_name = symbol_details[4]
201 # We don't want the .hidden for hidden functions
202 if function_name.startswith('.hidden '):
203 function_name = function_name[len('.hidden '):]
204 if function_name not in _functions:
205 _functions[function_name] = {'start': symbol_details[0],
206 'end': symbol_details[3],
207 'sources': False}
208 else:
209 logger.warning("'{}' duplicated in '{}'".format(
210 function_name,
211 elf_name))
212 except Exception as ex:
213 logger.error("@Listing functions at file {}: {}".format(
214 elf_name,
215 ex))
216 return _functions
217
218
219def apply_functions_exclude(elf_config, functions):
220 """
221 Remove excluded functions from the list of functions
222
223 :param elf_config: Config for elf binary file
224 :param functions: Array of functions in the binary elf file
225 :return: Tuple with included and excluded functions
226 """
227 if 'exclude_functions' not in elf_config:
228 return functions, []
229 incl = {}
230 excl = {}
231 for fname in functions:
232 exclude = False
233 for rex in elf_config['exclude_functions']:
234 if re.match(rex, fname):
235 exclude = True
236 excl[fname] = functions[fname]
237 break
238 if not exclude:
239 incl[fname] = functions[fname]
240 return incl, excl
241
242
243def remove_workspace(path, workspace):
244 """
245 Get the relative path to a given workspace
246
247 :param path: Path relative to the workspace to be returned
248 :param workspace: Path.
249 """
250 ret = path if workspace is None else os.path.relpath(path, workspace)
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100251 return ret
252
253
Saul Romero884d2142023-01-16 10:31:22 +0000254def get_function_line_numbers(source_file: str) -> Dict[str, int]:
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100255 """
256 Using ctags get all the function names with their line numbers
257 within the source_file
258
259 :return: Dictionary with function name as key and line number as value
260 """
Saul Romeroc1aa68d2021-07-22 16:56:07 +0100261 command = "ctags -x --c-kinds=f {}".format(source_file)
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100262 fln = {}
263 try:
Saul Romeroc1aa68d2021-07-22 16:56:07 +0100264 function_lines = os_command(command).split("\n")
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100265 for line in function_lines:
266 cols = line.split()
267 if len(cols) < 3:
268 continue
269 if cols[1] == "function":
270 fln[cols[0]] = int(cols[2])
271 elif cols[1] == "label" and cols[0] == "func":
272 fln[cols[-1]] = int(cols[2])
273 except BaseException:
274 logger.warning("Warning: Can't get all function line numbers from %s" %
275 source_file)
Saul Romeroc1aa68d2021-07-22 16:56:07 +0100276 except Exception as ex:
Saul Romero884d2142023-01-16 10:31:22 +0000277 logger.warning(f"Warning: Unknown error '{ex}' when executing command "
278 f"'{command}'")
Saul Romeroc1aa68d2021-07-22 16:56:07 +0100279 return {}
280
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100281 return fln
282
283
284class FunctionLineNumbers(object):
Saul Romero884d2142023-01-16 10:31:22 +0000285 """Helper class used to get a function start line number within
286 a source code file"""
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100287
Saul Romero884d2142023-01-16 10:31:22 +0000288 def __init__(self, workspace: str):
289 """
290 Initialise dictionary to allocate source code files with the
291 corresponding function start line numbers.
292
293 :param workspace: The folder where the source files are deployed
294 """
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100295 self.filenames = {}
296 self.workspace = workspace
297
Saul Romero884d2142023-01-16 10:31:22 +0000298 def get_line_number(self, filename: str, function_name: str) -> int:
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100299 if not FUNCTION_LINES_ENABLED:
300 return 0
301 if filename not in self.filenames:
302 newp = os.path.join(self.workspace, filename)
303 self.filenames[filename] = get_function_line_numbers(newp)
304 return 0 if function_name not in self.filenames[filename] else \
305 self.filenames[filename][function_name]
306
307
Saul Romero884d2142023-01-16 10:31:22 +0000308class BinaryParser(object):
309 """Class used to create an instance to parse the binary files with a
310 dwarf signature in order to produce logical information to be matched with
311 traces and produce a code coverage report"""
312
313 def __init__(self, dump: str, function_list: Dict[str, Dict[str, any]],
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000314 _workspace: str, _remove_workspace: bool,
315 function_line_numbers: FunctionLineNumbers):
Saul Romero884d2142023-01-16 10:31:22 +0000316 """
317 Initialisation of the instance to parse binary files.
318
319 :param dump: Binary dump (string) containing assembly code and source
320 code metadata, i.e. source code location and line number.
321 :param function_list: Dictionary of functions defined in the binary
322 dump.
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000323 :param _workspace: Workspace (folder) where the source files were built.
324 :param _remove_workspace: Boolean to indicate if the building of
325 source files is local (false) or in a CI (true).
Saul Romero884d2142023-01-16 10:31:22 +0000326 :param function_line_numbers: Object instance to get a function line
327 number within a source code file.
328 """
329 self.dump = dump
330 self.function_list = function_list
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000331 self.workspace = _workspace
332 self.remove_workspace = _remove_workspace
Saul Romero884d2142023-01-16 10:31:22 +0000333 self.function_definition = None
334 self.function_line_numbers = function_line_numbers
335
336 class FunctionBlock(object):
337 """Class used to parse and obtain a function block from the
338 binary dump file that corresponds to a function declaration within
339 the binary assembly code.
340 The function block has the following components:
341 - Function start address in memory (hexadecimal).
342 - Function name.
343 - Function code.
344 """
345
346 def __init__(self, function_group: List[str]):
347 """
348 Create an instance of a function block within a binary dump.
349
350 :param function_group: List containing the function start
351 address, name and code in the function block.
352 """
353 self.start, self.name, self.code = function_group
354 self.source_file = None
355 self.function_line_number = None
356
357 @staticmethod
358 def get(dump: str):
359 """
360 Static method generator to extract a function block from the binary
361 dump.
362
363 :param dump: Binary dump (string) that contains the binary file
364 information.
365 :return: A FunctionBlock object that is a logical representation
366 of a function declaration within the binary dump.
367 """
368 function_groups = re.findall(
369 r"(?s)([0-9a-fA-F]+) <([a-zA-Z0-9_]+)>:\n(.+?)(?=[A-Fa-f0-9]* "
370 r"<[a-zA-Z0-9_]+>:)", dump, re.DOTALL | re.MULTILINE)
371 for group in function_groups:
372 if len(group) != 3:
373 continue
374 function_group = list(group)
375 function_group[-1] += "\n"
376 yield BinaryParser.FunctionBlock(function_group)
377
378 class SourceCodeBlock(object):
379 """Class used to represent a source code block of information within
380 a function block in a binary dump file.
381 The source code block contains the following components:
382 - Optional function name where the source code/assembly code is defined.
383 - Source code file that contains the source code corresponding
384 to the assembly code.
385 - Line number within the source code file corresponding to the source
386 code.
387 - Assembly code block.
388 """
389
390 def __init__(self, source_code_block):
391 """
392 Create an instance of a source code block within a function block.
393
394 :param source_code_block: Tuple of 4 elements that contains the
395 components of a source code block.
396 """
397 self.function_name, self.source_file, self.line, self.asm_code \
398 = source_code_block
399
400 def get_assembly_line(self):
401 """Getter to return and AssemblyLine instance that corresponds to
402 a logical representation of an assembly code line contained
403 within a source code block (assembly code block)"""
404 return BinaryParser.AssemblyLine.get(self)
405
406 class AssemblyLine(object):
407 """Class used to represent an assembly code line within an
408 assembly code block.
409 The assembly line instruction is formed by the following components:
410 - Hexadecimal address of the assembly instruction.
411 - Assembly instruction.
412 """
413
414 def __init__(self, line):
415 """
416 Create an instance representing an assembly code line within an
417 assembly code block.
418
419 :param line: Tuple of 2 elements [Hexadecimal number,
420 and assembly code]
421 """
422 self.hex_line_number, self.opcode = line
423 self.dec_address = int(self.hex_line_number, 16)
424
425 @staticmethod
426 def get(source_code_block):
427 """
428 Static method generator to extract an assembly code line from a
429 assembly code block.
430
431 :param source_code_block: Object that contains the assembly code
432 within the source code block.
433 :return: AssemblyLine object.
434 """
435 lines = re.findall(
436 r"^[\s]+([a-fA-F0-9]+):\t(.+?)\n",
437 source_code_block.asm_code, re.DOTALL | re.MULTILINE)
438 for line in lines:
439 if len(line) != 2:
440 continue
441 yield BinaryParser.AssemblyLine(line)
442
443 class FunctionDefinition(object):
444 """
445 Class used to handle a function definition i.e. function name, source
446 code filename and line number where is declared.
447 """
448
449 def __init__(self, function_name):
450 """
451 Create an instance representing a function definition within a
452 function code block.
453
454 :param function_name: Initial function name
455 """
456 self.function_line_number = None
457 self.function_name = function_name
458 self.source_file: str = None
459
460 def update_sources(self, source_files, function_line_numbers):
461 """
462 Method to update source files dictionary
463
464 :param source_files: Dictionary that contains the representation
465 of the intermediate layer.
466
467 :param function_line_numbers: Object that obtains the start line
468 number for a function definition inside it source file.
469 :return:Nothing
470 """
471 source_files.setdefault(self.source_file, {"functions": {},
472 "lines": {}})
473 if self.function_name not in \
474 source_files[self.source_file]["functions"]:
475 self.function_line_number = \
476 function_line_numbers.get_line_number(
477 self.source_file,
478 self.function_name)
479 source_files[self.source_file]["functions"][
480 self.function_name] = {"covered": False,
481 "line_number":
482 self.function_line_number}
483
484 def get_source_code_block(self, function_block: FunctionBlock):
485 """
486 Generator method to obtain all the source code blocks within a
487 function block.
488
489 :param function_block: FunctionBlock object that contains the code
490 the source code blocks.
491 :return: A SourceCodeBlock object.
492 """
493 # When not present the block function name applies
494 self.function_definition = BinaryParser.FunctionDefinition(
495 function_block.name)
496 pattern = r'(?s)(^[a-zA-Z0-9_]+)?(?:\(\):\n)?(^{0}.+?):([0-9]+)[' \
497 r'^\n]*\n(.+?)(?={0}.+?:[0-9]+.+\n|^[a-zA-Z0-9_]+\(' \
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000498 r'\):\n)'.format(self.workspace)
Saul Romero884d2142023-01-16 10:31:22 +0000499 source_code_blocks = re.findall(pattern,
500 "{}\n{}/:000".format(
501 function_block.code,
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000502 self.workspace),
Saul Romero884d2142023-01-16 10:31:22 +0000503 re.DOTALL |
504 re.MULTILINE)
505 for block in source_code_blocks:
506 if len(block) != 4:
507 continue
508 source_code_block = BinaryParser.SourceCodeBlock(block)
509 if source_code_block.function_name:
510 # Usually in the first iteration function name is not empty
511 # and is the function's name block
512 self.function_definition.function_name = \
513 source_code_block.function_name
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000514 self.function_definition.source_file = source_code_block.source_file
515 if self.remove_workspace:
516 self.function_definition.source_file = remove_workspace(
517 source_code_block.source_file, self.workspace)
Saul Romero884d2142023-01-16 10:31:22 +0000518 yield source_code_block
519
520 def get_function_block(self):
521 """Generator method to obtain all the function blocks contained in
522 the binary dump file.
523 """
524 for function_block in BinaryParser.FunctionBlock.get(self.dump):
525 # Find out if the function block has C source code filename in
526 # the function block code
527 signature_group = re.findall(
528 r"(?s){}\(\):\n(/.+?):[0-9]+.*(?:\r*\n\n|\n$)".format(
529 function_block.name), function_block.code,
530 re.DOTALL | re.MULTILINE)
531 if not signature_group:
532 continue # Function does not have dwarf signature (sources)
533 if function_block.name not in self.function_list:
534 print("Warning:Function '{}' not found in function list!!!".
535 format(function_block.name))
536 continue # Function not found in function list
537 source_code_file = signature_group[0]
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000538 function_block.source_file = source_code_file
539 if self.remove_workspace:
540 function_block.source_file = remove_workspace(
541 source_code_file, self.workspace)
Saul Romero884d2142023-01-16 10:31:22 +0000542 function_block.function_line_number = \
543 self.function_line_numbers.get_line_number(
544 function_block.source_file, function_block.name)
545 yield function_block
546
547
548class IntermediateCodeCoverage(object):
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100549 """Class used to process the trace data along with the dwarf
550 signature files to produce an intermediate layer in json with
551 code coverage in assembly and c source code.
552 """
553
554 def __init__(self, _config, local_workspace):
555 self._data = {}
556 self.config = _config
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000557 self.workspace = self.config['parameters']['workspace']
558 self.remove_workspace = self.config['configuration']['remove_workspace']
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100559 self.local_workspace = local_workspace
560 self.elfs = self.config['elfs']
561 # Dictionary with stats from trace files {address}=(times executed,
562 # inst size)
563 self.traces_stats = {}
564 # Dictionary of unique assembly line memory address against source
565 # file location
566 # {assembly address} = (opcode, source file location, line number in
567 # the source file, times executed)
568 self.asm_lines = {}
569 # Dictionary of {source file location}=>{'lines': {'covered':Boolean,
570 # 'elf_index'; {elf index}=>{assembly address}=>(opcode,
571 # times executed),
572 # 'functions': {function name}=>is covered(boolean)}
573 self.source_files_coverage = {}
574 self.functions = []
575 # Unique set of elf list of files
576 self.elf_map = {}
577 # For elf custom mappings
578 self.elf_custom = None
579
580 def process(self):
581 """
582 Public method to process the trace files and dwarf signatures
583 using the information contained in the json configuration file.
584 This method writes the intermediate json file output linking
585 the trace data and c source and assembly code.
586 """
587 self.source_files_coverage = {}
588 self.asm_lines = {}
589 # Initialize for unknown elf files
590 self.elf_custom = ELF_MAP["custom_offset"]
591 sources_config = {}
592 print("Generating intermediate json layer '{}'...".format(
593 self.config['parameters']['output_file']))
594 for elf in self.elfs:
595 # Gather information
596 elf_name = elf['name']
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100597 # Trace data
598 self.traces_stats = load_stats_from_traces(elf['traces'])
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100599 functions_list = list_of_functions_for_binary(elf_name)
600 (functions_list, excluded_functions) = apply_functions_exclude(
601 elf, functions_list)
602 # Produce code coverage
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000603 self.process_binary(elf_name, functions_list)
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100604 sources_config = self.config['parameters']['sources']
605 # Now check code coverage in the functions with no dwarf signature
606 # (sources)
607 nf = {f: functions_list[f] for f in
608 functions_list if not
609 functions_list[f]["sources"]}
610 self.process_fn_no_sources(nf)
611 # Write to the intermediate json file
612 data = {"source_files": self.source_files_coverage,
613 "configuration": {
614 "sources": sources_config,
615 "metadata": "" if 'metadata' not in
616 self.config['parameters'] else
617 self.config['parameters']['metadata'],
Saul Romero884d2142023-01-16 10:31:22 +0000618 "elf_map": self.elf_map}
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100619 }
620 json_data = json.dumps(data, indent=4, sort_keys=True)
621 with open(self.config['parameters']['output_file'], "w") as f:
622 f.write(json_data)
623
Saul Romero884d2142023-01-16 10:31:22 +0000624 def get_elf_index(self, elf_name: str) -> int:
625 """Obtains the elf index and fills the elf_map instance variable"""
626 if elf_name not in self.elf_map:
627 if elf_name in ELF_MAP:
628 self.elf_map[elf_name] = ELF_MAP[elf_name]
629 else:
630 self.elf_map[elf_name] = ELF_MAP["custom_offset"]
631 ELF_MAP["custom_offset"] += 1
632 return self.elf_map[elf_name]
633
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000634 def process_binary(self, elf_filename: str, function_list):
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100635 """
Saul Romero884d2142023-01-16 10:31:22 +0000636 Process an elf file i.e. match the source code and asm lines against
637 trace files (coverage).
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100638
639 :param elf_filename: Elf binary file name
640 :param function_list: List of functions in the elf file i.e.
641 [(address start, address end, function name)]
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100642 """
Saul Romero884d2142023-01-16 10:31:22 +0000643 command = "%s -Sl %s | tee %s" % (OBJDUMP, elf_filename,
644 elf_filename.replace(".elf", ".dump"))
645 dump = os_command(command, show_command=True)
Jelle Sels83f141e2022-08-01 15:17:40 +0000646 dump += "\n0 <null>:" # For pattern matching the last function
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100647 elf_name = os.path.splitext(os.path.basename(elf_filename))[0]
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100648 function_line_numbers = FunctionLineNumbers(self.local_workspace)
Saul Romero884d2142023-01-16 10:31:22 +0000649 elf_index = self.get_elf_index(elf_name)
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100650 # Pointer to files dictionary
651 source_files = self.source_files_coverage
Saul Romero Dominguezfd4d0c92023-02-15 10:47:59 +0000652 parser = BinaryParser(dump, function_list, self.workspace,
653 self.remove_workspace, function_line_numbers)
Saul Romero884d2142023-01-16 10:31:22 +0000654 for function_block in parser.get_function_block():
655 function_list[function_block.name]["sources"] = True
656 source_files.setdefault(function_block.source_file,
657 {"functions": {},
658 "lines": {}})
659 source_files[function_block.source_file]["functions"][
660 function_block.name] = {"covered": False,
661 "line_number":
662 function_block.function_line_number}
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100663 is_function_block_covered = False
Saul Romero884d2142023-01-16 10:31:22 +0000664 source_code_block: BinaryParser.SourceCodeBlock
665 for source_code_block in parser.get_source_code_block(
666 function_block):
667 if parser.function_definition.function_name in function_list:
668 function_list[parser.function_definition.function_name][
669 "sources"] = True
670 parser.function_definition.update_sources(source_files,
671 function_line_numbers)
672 source_file_ln = \
673 source_files[parser.function_definition.source_file][
674 "lines"].setdefault(source_code_block.line,
675 {"covered": False, "elf_index": {}})
676 for asm_block in source_code_block.get_assembly_line():
677 times_executed = 0 if \
678 asm_block.dec_address not in self.traces_stats else \
679 self.traces_stats[asm_block.dec_address][0]
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100680 if times_executed > 0:
681 is_function_block_covered = True
682 source_file_ln["covered"] = True
Saul Romero884d2142023-01-16 10:31:22 +0000683 source_files[parser.function_definition.source_file][
684 "functions"][
685 parser.function_definition.function_name][
686 "covered"] = True
687 source_file_ln.setdefault("elf_index", {'elf_index': {}})
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100688 if elf_index not in source_file_ln["elf_index"]:
689 source_file_ln["elf_index"][elf_index] = {}
Saul Romero884d2142023-01-16 10:31:22 +0000690 if asm_block.dec_address not in \
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100691 source_file_ln["elf_index"][elf_index]:
Saul Romero884d2142023-01-16 10:31:22 +0000692 source_file_ln["elf_index"][elf_index][
693 asm_block.dec_address] = (
694 asm_block.opcode, times_executed)
695 source_files[function_block.source_file]["functions"][
696 function_block.name]["covered"] |= is_function_block_covered
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100697
698 def process_fn_no_sources(self, function_list):
699 """
700 Checks function coverage for functions with no dwarf signature i.e
701 sources.
702
703 :param function_list: Dictionary of functions to be checked
704 """
705 if not FUNCTION_LINES_ENABLED:
706 return # No source code at the workspace
707 address_seq = sorted(self.traces_stats.keys())
708 for function_name in function_list:
709 # Just check if the start address is in the trace logs
710 covered = function_list[function_name]["start"] in address_seq
711 # Find the source file
712 files = os_command(("grep --include *.c --include *.s -nrw '{}' {}"
713 "| cut -d: -f1").format(function_name,
714 self.local_workspace))
715 unique_files = set(files.split())
716 sources = []
717 line_number = 0
718 for source_file in unique_files:
719 d = get_function_line_numbers(source_file)
720 if function_name in d:
721 line_number = d[function_name]
722 sources.append(source_file)
723 if len(sources) > 1:
724 logger.warning("'{}' declared in {} files:{}".format(
725 function_name, len(sources),
726 ", ".join(sources)))
727 elif len(sources) == 1:
728 source_file = remove_workspace(sources[0],
729 self.local_workspace)
730 if source_file not in self.source_files_coverage:
731 self.source_files_coverage[source_file] = {"functions": {},
732 "lines": {}}
733 if function_name not in \
Saul Romero884d2142023-01-16 10:31:22 +0000734 self.source_files_coverage[source_file]["functions"] \
735 or covered:
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100736 self.source_files_coverage[source_file]["functions"][
737 function_name] = {"covered": covered,
738 "line_number": line_number}
739 else:
740 logger.warning("Function '{}' not found in sources.".format(
741 function_name))
742
743
744json_conf_help = """
745Produces an intermediate json layer for code coverage reporting
746using an input json configuration file.
747
748Input json configuration file format:
749{
750 "configuration":
751 {
752 "remove_workspace": <true if 'workspace' must be from removed from the
753 path of the source files>,
754 "include_assembly": <true to include assembly source code in the
755 intermediate layer>
756 },
757 "parameters":
758 {
759 "objdump": "<Path to the objdump binary to handle dwarf signatures>",
760 "readelf: "<Path to the readelf binary to handle dwarf signatures>",
761 "sources": [ <List of source code origins, one or more of the next
762 options>
763 {
764 "type": "git",
765 "URL": "<URL git repo>",
766 "COMMIT": "<Commit id>",
767 "REFSPEC": "<Refspec>",
768 "LOCATION": "<Folder within 'workspace' where this source
769 is located>"
770 },
771 {
772 "type": "http",
773 "URL": <URL link to file>",
774 "COMPRESSION": "xz",
775 "LOCATION": "<Folder within 'workspace' where this source
776 is located>"
777 }
778 ],
779 "workspace": "<Workspace folder where the source code was located to
780 produce the elf/axf files>",
781 "output_file": "<Intermediate layer output file name and location>",
782 "metadata": {<Metadata objects to be passed to the intermediate json
783 files>}
784 },
785 "elfs": [ <List of elf files to be traced/parsed>
786 {
787 "name": "<Full path name to elf/axf file>",
788 "traces": [ <List of trace files to be parsed for this
789 elf/axf file>
790 "Full path name to the trace file,"
791 ]
792 }
793 ]
794}
795"""
796OBJDUMP = None
797READELF = None
798FUNCTION_LINES_ENABLED = None
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100799
800
801def main():
802 global OBJDUMP
803 global READELF
804 global FUNCTION_LINES_ENABLED
805
806 parser = argparse.ArgumentParser(epilog=json_conf_help,
807 formatter_class=RawTextHelpFormatter)
808 parser.add_argument('--config-json', metavar='PATH',
809 dest="config_json", default='config_file.json',
810 help='JSON configuration file', required=True)
811 parser.add_argument('--local-workspace', default="",
812 help=('Local workspace folder where source code files'
813 ' and folders resides'))
814 args = parser.parse_args()
815 try:
816 with open(args.config_json, 'r') as f:
817 config = json.load(f)
818 except Exception as ex:
819 print("Error at opening and processing JSON: {}".format(ex))
820 return
821 # Setting toolchain binary tools variables
822 OBJDUMP = config['parameters']['objdump']
823 READELF = config['parameters']['readelf']
824 # Checking if are installed
825 os_command("{} --version".format(OBJDUMP))
826 os_command("{} --version".format(READELF))
827
828 if args.local_workspace != "":
829 # Checking ctags installed
830 try:
831 os_command("ctags --version")
832 except BaseException:
833 print("Warning!: ctags not installed/working function line numbers\
834 will be set to 0. [{}]".format(
835 "sudo apt install exuberant-ctags"))
836 else:
837 FUNCTION_LINES_ENABLED = True
838
Saul Romero884d2142023-01-16 10:31:22 +0000839 intermediate_layer = IntermediateCodeCoverage(config, args.local_workspace)
840 intermediate_layer.process()
Basil Eljuse4b14afb2020-09-30 13:07:23 +0100841
842
843if __name__ == '__main__':
844 logging.basicConfig(filename='intermediate_layer.log', level=logging.DEBUG,
845 format=('%(asctime)s %(levelname)s %(name)s '
846 '%(message)s'))
847 logger = logging.getLogger(__name__)
848 start_time = time.time()
849 main()
850 elapsed_time = time.time() - start_time
851 print("Elapsed time: {}s".format(elapsed_time))