Initial commit.
- qa-tools public release which includes:
- trace-based coverage tool
- quality metrics measurement and tracking setup
- associated in-source documentation.
Signed-off-by: Basil Eljuse <basil.eljuse@arm.com>
diff --git a/quality-metrics/broker-component/Dockerfile b/quality-metrics/broker-component/Dockerfile
new file mode 100644
index 0000000..2f960ec
--- /dev/null
+++ b/quality-metrics/broker-component/Dockerfile
@@ -0,0 +1,28 @@
+#======================================================================
+# Copyright (c) 2020, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#======================================================================
+
+#======================================================================
+# FILE: Dockerfile
+#
+# DESCRIPTION: This is used for containerised deployment of the quality
+# metrics broker component.
+#======================================================================
+
+FROM alpine:3.11
+
+RUN apk update && \
+ apk upgrade && \
+ apk add --no-cache python3-dev && \
+ pip3 install --upgrade pip
+
+WORKDIR /metrics_server
+COPY . /metrics_server
+
+RUN pip3 --no-cache-dir install -r requirements.txt
+
+EXPOSE 5000
+
+CMD python3 quality_metrics_server.py
diff --git a/quality-metrics/broker-component/constants.py b/quality-metrics/broker-component/constants.py
new file mode 100644
index 0000000..32ce30f
--- /dev/null
+++ b/quality-metrics/broker-component/constants.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+""" constants.py:
+
+ This file contains the constants required by metrics server.
+
+"""
+
+JWT_EXPIRATION_DAYS = 365
+
+HOST = "<Host Public IP Address>"
+PORT = "8086"
+BUFF_SIZE = 10
+POLL_DELAY = 0.1
+
+LISTEN_ALL_IPS = "0.0.0.0"
+
+VALID_METRICS = [
+ 'tfm_image_size',
+ 'tfa_code_churn',
+ 'tfa_defects_stats',
+ 'tfa_defects_tracking',
+ 'tfa_complexity_stats',
+ 'tfa_complexity_tracking',
+ 'tfa_rtinstr',
+ 'tfa_image_size',
+ 'tfa_misra_defects']
+
+DATABASE_DICT = {
+ "TFM_IMAGE_SIZE": "TFM_ImageSize",
+ "TFA_CODE_CHURN": "TFA_CodeChurn",
+ "TFA_DEFECTS": "TFA_Defects",
+ "TFA_COMPLEXITY": "TFA_Complexity",
+ "TFA_RTINSTR": "TFA_RTINSTR",
+ "TFA_IMAGE_SIZE": "TFA_ImageSize",
+ "TFA_MISRA_DEFECTS": "TFA_MisraDefects"
+}
+
+SUPPORTED_API_VERSIONS = ["1.0"]
diff --git a/quality-metrics/broker-component/credentials.py b/quality-metrics/broker-component/credentials.py
new file mode 100644
index 0000000..7e9793a
--- /dev/null
+++ b/quality-metrics/broker-component/credentials.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+""" credentials.py:
+
+ Credentials class. This is for reference only.
+
+"""
+
+# SECRET_KEY is set for reference purpose only
+# It is recommended to change its value before deployment
+SECRET_KEY = 'SECRET_KEY'
+
+
+class User(object):
+ def __init__(self, id, username, password):
+ self.id = id
+ self.username = username
+ self.password = password
+
+ def __str__(self):
+ return "User(id='%s')" % self.id
+
+
+# User credentials are set for reference purpose only
+# It is recommended to change these value accordingly before deployment
+users = [
+ User(1, 'metrics_1', 'metrics_pass_1'),
+ User(2, 'metrics_2', 'metrics_pass_2'),
+ User(3, 'tfa_metrics', 'tfa_metrics_pass'),
+]
diff --git a/quality-metrics/broker-component/data_converter.py b/quality-metrics/broker-component/data_converter.py
new file mode 100644
index 0000000..5b3c74b
--- /dev/null
+++ b/quality-metrics/broker-component/data_converter.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+""" data_converter.py:
+
+ Data converter class. This class is aimed at converting the received
+ data in the format which InfluxDB understands.
+
+"""
+
+import json
+import constants
+
+
+class DataConverter:
+
+ @staticmethod
+ def convert_tfm_imagesize_data(data):
+ # Transform keys names
+ data['metadata']['DataProducer'] = data['metadata'].pop(
+ 'data_producer')
+
+ data['metadata']['git_info']['CommitTitle'] = data['metadata']['git_info'].pop(
+ 'commit_title')
+ data['metadata']['git_info']['CommitID'] = data['metadata']['git_info'].pop(
+ 'commit_id')
+ data['metadata']['git_info']['GerritID'] = data['metadata']['git_info'].pop(
+ 'gerrit_id')
+ data['metadata']['git_info']['CommitURL'] = data['metadata']['git_info'].pop(
+ 'commit_url')
+ data['metadata']['git_info']['Branch'] = data['metadata']['git_info'].pop(
+ 'branch')
+
+ data['metadata']['build_info']['BuildType'] = data['metadata']['build_info'].pop(
+ 'build_type')
+ data['metadata']['build_info']['CmakeConfig'] = data['metadata']['build_info'].pop(
+ 'cmake_config')
+ data['metadata']['build_info']['Compiler'] = data['metadata']['build_info'].pop(
+ 'compiler')
+ data['metadata']['build_info']['Target'] = data['metadata']['build_info'].pop(
+ 'target')
+
+ ret = {}
+ ret['tags'] = {}
+ ret['fields'] = {}
+
+ ret['measurement'] = 'TFM_ImageSize_Statistics'
+
+ for file_info in data['data']:
+ ret['fields'][file_info['file'].rsplit(
+ '.', 1)[0] + '_b'] = file_info['bss']
+ ret['fields'][file_info['file'].rsplit(
+ '.', 1)[0] + '_d'] = file_info['data']
+ ret['fields'][file_info['file'].rsplit(
+ '.', 1)[0] + '_t'] = file_info['text']
+
+ ret['tags']['DataProducer'] = str(data['metadata']['DataProducer'])
+
+ ret['time'] = str(data['metadata']['git_info']['commit_time'])
+
+ for key in data['metadata']['git_info']:
+ if key == 'commit_time':
+ continue
+ ret['tags'][key] = str(data['metadata']['git_info'][key])
+
+ for key in data['metadata']['build_info']:
+ ret['tags'][key] = str(data['metadata']['build_info'][key])
+
+ print(ret)
+
+ return [ret]
+
+ @staticmethod
+ def convert_data(data):
+ """
+ Convert data to a dictionary containing measurement
+ name, fields and tags. It is required by InfluxDB.
+
+ :param data: data to be converted to InfluxDB format
+ """
+
+ if data['metadata']['metrics'] == 'tfm_imagesize':
+ ret = DataConverter.convert_tfm_imagesize_data(data)
+ else:
+ ret = data['data']
+
+ return ret
diff --git a/quality-metrics/broker-component/data_validator.py b/quality-metrics/broker-component/data_validator.py
new file mode 100644
index 0000000..a67b86b
--- /dev/null
+++ b/quality-metrics/broker-component/data_validator.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+""" data_validator.py:
+
+ JSON data validator class. This class is aimed at validating the JSON
+ data sent in curl command with the JSON schema, so that before pushing
+ the data to the database, it is ensured that required data is received
+ in agreed-upon format.
+
+"""
+
+import sys
+import json
+import os.path
+import constants
+import jsonschema
+from jsonschema import validate
+
+
+class DataValidator:
+ @staticmethod
+ def validate_request_sanity(data_dict):
+ """
+ Input sanitisation/authentication in the application flow
+
+ :param: data_dict: Data to be validated
+ :return: Validation info and error code
+ """
+ if 'metrics' in data_dict['metadata'] and 'api_version' in data_dict and \
+ data_dict['metadata']['metrics'] in constants.VALID_METRICS:
+ if data_dict['api_version'] not in constants.SUPPORTED_API_VERSIONS:
+ return 'Incorrect API version', 401
+
+ filename = 'metrics-schemas/' + data_dict['metadata']['metrics'] + '_schema_' + \
+ data_dict['api_version'].replace(".", "_") + '.json'
+ if not os.path.exists(filename):
+ return filename + ' does not exist', 501
+
+ try:
+ with open(filename, 'r') as handle:
+ parsed = json.load(handle)
+ validate(data_dict, parsed)
+ sys.stdout.write('Record OK\n')
+ return 'OK', 204
+ except jsonschema.exceptions.ValidationError as ve:
+ sys.stdout.write('Record ERROR\n')
+ sys.stderr.write(str(ve) + "\n")
+ return 'Incorrect JSON Schema: ' + \
+ str(ve).split('\n', 1)[0], 400
+ else:
+ return 'Invalid schema - metrics or api version missing\n', 401
diff --git a/quality-metrics/broker-component/db_manager.py b/quality-metrics/broker-component/db_manager.py
new file mode 100644
index 0000000..8f2d77c
--- /dev/null
+++ b/quality-metrics/broker-component/db_manager.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+""" db_manager.py:
+
+ Database interface abstraction class. This class is aimed at providing an
+ asynchronous interface between a blocking IO resource(database) and a
+ public interface designed for high concurrency.
+
+"""
+
+import time
+import threading
+from queue import Queue
+from pprint import pformat
+from influxdb import InfluxDBClient
+
+import constants
+from data_converter import DataConverter
+
+
+class dbManager(object):
+ def __init__(self,
+ host=constants.HOST,
+ port=constants.PORT,
+ user=None,
+ password=None,
+ buff_size=constants.BUFF_SIZE,
+ poll_delay=constants.POLL_DELAY,
+ app=None):
+ self.queue_buff_sz = buff_size
+ self.poll_delay = poll_delay
+
+ self.db_host = host
+ self.db_port = port
+ self.db_user = user
+ self.db_pass = password
+ self.write_queue = Queue(maxsize=self.queue_buff_sz)
+ self.stop_threads = False
+ self.app = app
+
+ for key in constants.DATABASE_DICT:
+ client = InfluxDBClient(host=self.db_host,
+ port=self.db_port,
+ username=self.db_user,
+ password=self.db_pass,
+ database=constants.DATABASE_DICT[key])
+ setattr(self, key.lower() + '_client', client)
+
+ def store(self, data):
+ """
+ Places data in the FIFO to be broadcast when
+ the database is not busy
+
+ :param: data: Data to be placed in FIFO
+ """
+ validation = 'OK'
+ err_code = 204
+ try:
+ self.write_queue.put(data)
+ except Exception as e:
+ validation = "** Write to Queue Failed. ** "
+ err_code = 402
+ print(validation, e)
+ self.app.logger.error(pformat({"error_code": err_code,
+ "info": validation, "exception": e}))
+ return validation, err_code
+
+ def get_db_client(self, metrics):
+ if "stats" in metrics:
+ client = metrics.replace("_stats", "") + "_client"
+ elif "tracking" in metrics:
+ client = metrics.replace("_tracking", "") + "_client"
+ else:
+ client = metrics + "_client"
+ if hasattr(self, client):
+ return getattr(self, client)
+ else:
+ self.app.logger.error("Invalid metrics %" % (metrics))
+
+ def write_db_direct(self, data):
+ """
+ Write data to database ( will block if database is busy )
+
+ :param: data: data to be written to database
+ """
+ db_client = self.get_db_client(data['metadata']['metrics'])
+
+ converted_data = DataConverter.convert_data(data)
+
+ if db_client:
+ if db_client.write_points(converted_data):
+ self.app.logger.info(
+ "Writing to InfluxDB hosted at %s "
+ "has been successful for %s!" %
+ (self.db_host, data['metadata']['metrics']))
+ else:
+ self.app.logger.error(
+ "Writing to InfluxDB hosted at %s "
+ "has FAILED for %s!" %
+ (self.db_host, data['metadata']['metrics']))
+ else:
+ self.app.logger.error(
+ "%s database not connected.." %
+ data['metadata']['metrics'])
+
+ def start_daemon(self):
+ """
+ Spawn a new thread that will consume data in the write FIFO
+ and place it into the databse
+ """
+
+ def write_db_loop():
+
+ while True:
+ try:
+ time.sleep(self.poll_delay)
+ if self.stop_threads:
+ self.app.logger.info(
+ "\n ** Shutting Down Database Writer **")
+ return
+ elif self.write_queue.qsize() > 0:
+ dt = self.write_queue.get()
+ # Write the data to the databse
+ self.write_db_direct(dt)
+ self.write_queue.task_done()
+ except Exception as e:
+ self.app.logger.error(
+ "** DB Writer Thread Failed. ** \n%s" % e)
+
+ self.db_write_thread = threading.Thread(target=write_db_loop)
+ self.db_write_thread.daemon = True
+ self.db_write_thread.start()
+ return self
+
+ def stop(self):
+ """
+ Flag which terminates db_write_threads loop
+ """
+ self.app.logger.info("** Setting stop_threads to True **")
+ self.stop_threads = True
diff --git a/quality-metrics/broker-component/docker-compose.yml b/quality-metrics/broker-component/docker-compose.yml
new file mode 100644
index 0000000..bb91628
--- /dev/null
+++ b/quality-metrics/broker-component/docker-compose.yml
@@ -0,0 +1,58 @@
+#======================================================================
+# Copyright (c) 2020, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#======================================================================
+
+#======================================================================
+# FILE: docker-compose.yml
+#
+# DESCRIPTION: This is a docker compose file for the deployment of all
+# the components involved in the quality metrics collection setup. It
+# brings up Grafana, InfluxDB and the broker component as a multi-
+# container deployment.
+#======================================================================
+
+version: "2"
+services:
+ grafana:
+ image: grafana/grafana
+ container_name: grafana_container
+ restart: always
+ ports:
+ - 3000:3000
+ networks:
+ - metrics_network
+ volumes:
+ - grafana-volume:/var/lib/grafana
+ - ./grafana-provisioning/:/etc/grafana/provisioning
+ environment:
+ - GF_INSTALL_PLUGINS=grafana-piechart-panel
+ influxdb:
+ image: influxdb
+ container_name: influxdb_container
+ restart: always
+ ports:
+ - 8086:8086
+ networks:
+ - metrics_network
+ volumes:
+ - influxdb-volume:/var/lib/influxdb
+ web:
+ build: .
+ ports:
+ - "5000:5000"
+ networks:
+ - metrics_network
+ depends_on:
+ - influxdb
+ links:
+ - influxdb:influx
+networks:
+ metrics_network:
+volumes:
+ grafana-volume:
+ external: true
+ influxdb-volume:
+ external: true
+
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_code_churn_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_code_churn_schema_1_0.json
new file mode 100644
index 0000000..8935e76
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_code_churn_schema_1_0.json
@@ -0,0 +1,58 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Lines_of_Change" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["Lines_of_Change"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Git_Tag_Date" : {
+ "type" : "number"
+ },
+ "Base_Tag" : {
+ "type" : "string"
+ },
+ "Target_Tag" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Git_Tag_Date", "Base_Tag", "Target_Tag"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_complexity_stats_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_complexity_stats_schema_1_0.json
new file mode 100644
index 0000000..0c22160
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_complexity_stats_schema_1_0.json
@@ -0,0 +1,70 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Function_ID" : {
+ "type" : "string"
+ },
+ "Score" : {
+ "type" : "number"
+ },
+ "Whitelisted" : {
+ "type" : "string"
+ },
+ "Threshold" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["Function_ID", "Score", "Whitelisted", "Threshold"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Git_Tag_Date" : {
+ "type" : "number"
+ },
+ "Base_Tag" : {
+ "type" : "string"
+ },
+ "Target_Tag" : {
+ "type" : "string"
+ },
+ "Location" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Git_Tag_Date", "Base_Tag", "Target_Tag", "Location"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
\ No newline at end of file
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_complexity_tracking_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_complexity_tracking_schema_1_0.json
new file mode 100644
index 0000000..509a966
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_complexity_tracking_schema_1_0.json
@@ -0,0 +1,61 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Functions_Exceeding_Threshold_Not_Whitelisted" : {
+ "type" : "number"
+ },
+ "Whitelisted" : {
+ "type" : "string"
+ },
+ "Threshold" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["Functions_Exceeding_Threshold_Not_Whitelisted", "Whitelisted", "Threshold"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Git_Tag_Date" : {
+ "type" : "number"
+ },
+ "Target_Tag" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Git_Tag_Date", "Target_Tag"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_defects_stats_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_defects_stats_schema_1_0.json
new file mode 100644
index 0000000..04dcdf8
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_defects_stats_schema_1_0.json
@@ -0,0 +1,58 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Title" : {
+ "type" : "string"
+ },
+ "Issue_Status" : {
+ "type" : "string"
+ },
+ "URL" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Title", "Issue_Status", "URL"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Defect_ID" : {
+ "type" : "string"
+ },
+ "Measured_Date" : {
+ }
+ },
+ "required" : ["Defect_ID", "Measured_Date"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_defects_tracking_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_defects_tracking_schema_1_0.json
new file mode 100644
index 0000000..7a0d855
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_defects_tracking_schema_1_0.json
@@ -0,0 +1,52 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Issue_Status" : {
+ "type" : "string"
+ },
+ "Number_of_Defects" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["Issue_Status", "Number_of_Defects"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Measured_Date" : {
+ }
+ },
+ "required" : ["Measured_Date"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_image_size_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_image_size_schema_1_0.json
new file mode 100644
index 0000000..e64e1f8
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_image_size_schema_1_0.json
@@ -0,0 +1,58 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "minProperties": 1,
+ "patternProperties" : {
+ "^BL*$" : {
+ "type" : "number"
+ }
+ }
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "BinMode" : {
+ "type" : "string"
+ },
+ "CommitID" : {
+ "type" : "string"
+ },
+ "CommitTitle" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["BinMode", "CommitID", "CommitTitle"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_misra_defects_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_misra_defects_schema_1_0.json
new file mode 100644
index 0000000..9109bd9
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_misra_defects_schema_1_0.json
@@ -0,0 +1,67 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "TotalDefects" : {
+ "type" : "number"
+ },
+ "MandatoryDefects" : {
+ "type" : "number"
+ },
+ "RequiredDefects" : {
+ "type" : "number"
+ },
+ "AdvisoryDefects" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["TotalDefects", "MandatoryDefects", "RequiredDefects", "AdvisoryDefects"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "BinMode" : {
+ "type" : "string"
+ },
+ "CommitID" : {
+ "type" : "string"
+ },
+ "CommitTitle" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["BinMode", "CommitID", "CommitTitle"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfa_rtinstr_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfa_rtinstr_schema_1_0.json
new file mode 100644
index 0000000..1801814
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfa_rtinstr_schema_1_0.json
@@ -0,0 +1,82 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Latency_EL3Entry_EL3Exit" : {
+ "type" : "number"
+ },
+ "Latency_EL3Entry_CPUPowerDown" : {
+ "type" : "number"
+ },
+ "Latency_CPUWakeup_EL3Exit" : {
+ "type" : "number"
+ },
+ "CacheFlush" : {
+ "type" : "number"
+ }
+ },
+ "oneOf": [
+ { "required": [
+ "Latency_EL3Entry_EL3Exit"
+ ]},
+ { "required": [
+ "Latency_EL3Entry_CPUPowerDown",
+ "Latency_CPUWakeup_EL3Exit",
+ "CacheFlush"
+ ]}
+ ]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "CommitID" : {
+ "type" : "string"
+ },
+ "CommitTitle" : {
+ "type" : "string"
+ },
+ "TC_Name" : {
+ "type" : "string"
+ },
+ "Cluster_ID" : {
+ "type" : "number"
+ },
+ "CPU_Core" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["CommitID", "CommitTitle", "TC_Name", "Cluster_ID", "CPU_Core"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfm_code_churn_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfm_code_churn_schema_1_0.json
new file mode 100644
index 0000000..a181c31
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfm_code_churn_schema_1_0.json
@@ -0,0 +1,58 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Lines_of_Change" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["Lines_of_Change"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Git_Tag_Date" : {
+ "type" : "string"
+ },
+ "Base_Tag" : {
+ "type" : "string"
+ },
+ "Target_Tag" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Git_Tag_Date", "Base_Tag", "Target_Tag"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfm_complexity_stats_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfm_complexity_stats_schema_1_0.json
new file mode 100644
index 0000000..8390692
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfm_complexity_stats_schema_1_0.json
@@ -0,0 +1,73 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Function" : {
+ "type" : "string"
+ },
+ "Score" : {
+ "type" : "number"
+ },
+ "Whitelisted" : {
+ "type" : "string"
+ },
+ "Threshold" : {
+ "type" : "number"
+ },
+ "Location" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Function", "Whitelisted", "Threshold", "Score", "Location"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Git_Tag_Date" : {
+ "type" : "string"
+ },
+ "Target_Tag" : {
+ "type" : "string"
+ },
+ "Base_Tag" : {
+ "type" : "string"
+ },
+ "Location_Tag" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Git_Tag_Date", "Target_Tag", "Base_Tag", "Location_Tag"],
+ "additionalProperties": false
+ },
+ "time" : {
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags", "time"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfm_defects_stats_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfm_defects_stats_schema_1_0.json
new file mode 100644
index 0000000..b5f819d
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfm_defects_stats_schema_1_0.json
@@ -0,0 +1,65 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ }
+ }
+ },
+ "data" : {
+ "type" : "array",
+ "minItems": 1,
+ "items" : {
+ "type" : "object",
+ "properties" :{
+ "measurement" : {
+ "type" : "string"
+ },
+ "fields" : {
+ "type" : "object",
+ "properties" : {
+ "Status" : {
+ "type" : "string"
+ },
+ "Priority" : {
+ "type" : "string"
+ },
+ "Summary" : {
+ "type" : "string"
+ },
+ "URL" : {
+ "type" : "string"
+ },
+ "Existing_Defect" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["Status", "Priority", "Summary", "URL", "Existing_Defect"]
+ },
+ "tags" : {
+ "type" : "object",
+ "properties" : {
+ "Defect_ID" : {
+ "type" : "string"
+ },
+ "Measured_Date" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["Defect_ID", "Measured_Date"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required" : ["measurement", "fields", "tags"],
+ "additionalProperties": false
+ }
+ },
+ "required" : ["api_version", "metadata", "data"]
+}
diff --git a/quality-metrics/broker-component/metrics-schemas/tfm_image_size_schema_1_0.json b/quality-metrics/broker-component/metrics-schemas/tfm_image_size_schema_1_0.json
new file mode 100644
index 0000000..30a2ffc
--- /dev/null
+++ b/quality-metrics/broker-component/metrics-schemas/tfm_image_size_schema_1_0.json
@@ -0,0 +1,88 @@
+{
+ "type" : "object",
+ "properties" : {
+ "api_version" : {
+ "type" : "string"
+ },
+ "metadata" : {
+ "type" : "object",
+ "properties" : {
+ "metrics" : {
+ "type" : "string"
+ },
+ "data_producer" : {
+ "type" : "string"
+ },
+ "git_info" : {
+ "type" : "object",
+ "properties" : {
+ "commit_time" : {
+ "type" : "string"
+ },
+ "commit_title" : {
+ "type" : "string"
+ },
+ "commit_id" : {
+ "type" : "string"
+ },
+ "commit_url" : {
+ "type" : "string"
+ },
+ "branch" : {
+ "type" : "string"
+ },
+ "gerrit_id" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["commit_time", "commit_title", "commit_id", "commit_url", "branch"],
+ "additionalProperties": false
+ },
+ "build_info" : {
+ "type" : "object",
+ "properties" : {
+ "build_type" : {
+ "type" : "string"
+ },
+ "cmake_config" : {
+ "type" : "string"
+ },
+ "compiler" : {
+ "type" : "string"
+ },
+ "target" : {
+ "type" : "string"
+ }
+ },
+ "required" : ["build_type", "cmake_config", "compiler", "target"]
+ }
+ },
+ "required" : ["metrics", "build_info", "git_info", "data_producer"],
+ "additionalProperties": false
+ },
+ "data" : {
+ "type" : "array",
+ "minItems" : 1,
+ "items" : {
+ "type" : "object",
+ "properties" : {
+ "file" : {
+ "type" : "string"
+ },
+ "bss" : {
+ "type" : "number"
+ },
+ "data" : {
+ "type" : "number"
+ },
+ "text" : {
+ "type" : "number"
+ }
+ },
+ "required" : ["file", "bss", "data", "text"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "required" : ["metadata", "data", "api_version"]
+}
diff --git a/quality-metrics/broker-component/quality_metrics_server.py b/quality-metrics/broker-component/quality_metrics_server.py
new file mode 100644
index 0000000..f68f4b2
--- /dev/null
+++ b/quality-metrics/broker-component/quality_metrics_server.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+from data_validator import DataValidator
+import credentials
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+""" quality_metrics_server.py:
+
+ This is the broker component which accepts the data from data
+ generator scripts, and performs basic sanity check and pushes
+ the data to Influx-DB for visualisation with Grafana component.
+ It is not mandatory to push data via data generator scripts.
+ The request to push data to database, in this case - InfluxDB,
+ is expected to be be a POST request with right credentials and
+ should be in agreed upon format.
+
+"""
+
+from pprint import pprint
+from pprint import pformat
+from db_manager import dbManager
+from flask_jwt import JWT, jwt_required
+from flask import Flask, jsonify, request
+from werkzeug.security import safe_str_cmp
+from logging.handlers import RotatingFileHandler
+
+import sys
+import json
+import random
+import logging
+import argparse
+import datetime
+
+import constants
+""" It is suggested to keep credentials.py is kept locally in the
+ system where server is running. This file has been provided
+ for reference.
+"""
+
+username_table = {u.username: u for u in credentials.users}
+userid_table = {u.id: u for u in credentials.users}
+
+
+def authenticate(username, password):
+ user = username_table.get(username, None)
+ if user and safe_str_cmp(
+ user.password.encode('utf-8'),
+ password.encode('utf-8')):
+ return user
+
+
+def identity(payload):
+ user_id = payload['identity']
+ return userid_table.get(user_id, None)
+
+
+def setup_logging(app):
+ # maxBytes and backupCount values to allow the file to rollover at a predetermined size.
+ # When the size is about to be exceeded, the file is closed and a new file is silently
+ # opened for output. Rollover occurs whenever the current log file is nearly maxBytes in length.
+ # When backupCount is non-zero, the system will save old log files by appending the extensions
+ # ‘.1’, ‘.2’ etc., to the filename.
+ file_handler = RotatingFileHandler(
+ "./flask.log",
+ maxBytes=1024 * 1024 * 1024 * 5,
+ backupCount=5)
+ file_handler.setFormatter(
+ logging.Formatter(
+ '[%(asctime)s][PID:%(process)d][%(levelname)s]'
+ '[%(lineno)s][%(name)s.%(funcName)s()] %(message)s'))
+ file_handler.setLevel(logging.INFO)
+ loggers = [app.logger]
+ for logger in loggers:
+ logger.addHandler(file_handler)
+ app.logger.setLevel(logging.INFO)
+
+
+app = Flask(__name__)
+
+setup_logging(app)
+
+logger = logging.getLogger(__name__)
+
+app.debug = True
+app.config['SECRET_KEY'] = credentials.SECRET_KEY
+app.config['JWT_EXPIRATION_DELTA'] = datetime.timedelta(
+ days=constants.JWT_EXPIRATION_DAYS)
+
+dbm = dbManager(app=app).start_daemon()
+
+jwt = JWT(app, authenticate, identity)
+
+# ----------------------- Database Methods ----------------------------------#
+
+
+def store_to_db(data_dict):
+ """
+ Use the database manager to asynchronously update the database
+
+ :param: data_dict: Dictionary containing data to be stored
+ """
+ validation, err_code = dbm.store(data_dict)
+ return validation, err_code
+
+# ----------------------- FLASK API Methods ---------------------------------- #
+
+
+@app.route('/', methods=['POST'])
+@jwt_required()
+def add_db_entry():
+ """
+ Store received data to database if validation is okay
+
+ :return: validation information and error code
+ """
+
+ data = request.get_json()
+ app.logger.debug("Received Data (POST)")
+ app.logger.debug(pformat(data))
+ # Make sure the data is valid
+ validation, err_code = DataValidator.validate_request_sanity(data)
+ if validation == "OK":
+ app.logger.info("<<<<VALIDATION OK>>>>")
+ validation, err_code = store_to_db(data)
+ else:
+ app.logger.error("<<<<VALIDATION NOT OK>>>>")
+ app.logger.error(pformat({"data": validation, "error_code": err_code}))
+ info_json = jsonify({"data": validation, "error_code": err_code})
+ return info_json, err_code
+
+
+@app.route("/")
+def home():
+ info_json = jsonify({"type": "INFO", "data": "Quality Metrics"})
+ return info_json, 200
+
+
+if __name__ == '__main__':
+ try:
+ app.run(host=constants.LISTEN_ALL_IPS, port=5000)
+ except Exception as ex:
+ template = "An exception of type {0} occurred. Arguments:\n{1!r}"
+ message = template.format(type(ex).__name__, ex.args)
+ app.logger.error("message")
+ dbm.stop()
diff --git a/quality-metrics/broker-component/requirements.txt b/quality-metrics/broker-component/requirements.txt
new file mode 100644
index 0000000..115caa2
--- /dev/null
+++ b/quality-metrics/broker-component/requirements.txt
@@ -0,0 +1,35 @@
+#======================================================================
+# Copyright (c) 2020, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#======================================================================
+
+#===============================================================================
+# FILE: requirements.txt
+#
+# DESCRIPTION: Requirements for metrics server
+#===============================================================================
+
+attrs==19.3.0
+certifi==2020.4.5.2
+chardet==3.0.4
+click==7.1.2
+Flask==1.1.2
+Flask-JWT==0.3.2
+idna==2.9
+importlib-metadata==1.6.1
+influxdb==5.3.0
+itsdangerous==1.1.0
+Jinja2==2.11.2
+jsonschema==3.2.0
+MarkupSafe==1.1.1
+msgpack==0.6.1
+PyJWT==1.4.2
+pyrsistent==0.16.0
+python-dateutil==2.8.1
+pytz==2020.1
+requests==2.23.0
+six==1.15.0
+urllib3==1.25.9
+Werkzeug==1.0.1
+zipp==3.1.0