Share files: Compresses files before sharing
This compresses the code files before they are shared between jobs in an
attempt to lower the shared storage overhead.
Change-Id: Ic4ad813e47a6cda429141886ed0ec796598d7b85
Signed-off-by: Matthew Dalzell <matthew.dalzell@arm.com>
diff --git a/clone.sh b/clone.sh
index ee0246c..2d902b9 100755
--- a/clone.sh
+++ b/clone.sh
@@ -38,6 +38,8 @@
local REPO_REFSPEC=$3
local SYNC_CMD=$4
+ echo "Refspec when entering is ${REPO_REFSPEC}"
+
echo "Repo: $REPO_URL $REPO_NAME $REPO_REFSPEC $SYNC_CMD"
# In case repository is not defined, just skip it
@@ -46,18 +48,38 @@
return 0
fi
+ # The non-compressed repo should not be in there, removing
if [ ! -d "${SHARE_FOLDER}/${REPO_NAME}" ]; then
+ echo "Removing non-compressed ${REPO_NAME} as it should not be stored."
+ rm -rf "${SHARE_FOLDER}/${REPO_NAME}"
+ fi
+
+ if [ ! -f "${SHARE_FOLDER}/${REPO_NAME}.tar.gz" ]; then
git_clone $REPO_URL "${SHARE_FOLDER}/${REPO_NAME}"
+ # Compress for shared area
+ echo "Refspec at pull is ${REPO_REFSPEC}"
git_checkout "${SHARE_FOLDER}/${REPO_NAME}" $REPO_REFSPEC $SYNC_CMD
- else
- cd "${SHARE_FOLDER}/${REPO_NAME}"
- echo -e "Share Folder ${REPO_NAME} $(git rev-parse --short HEAD)\n"
+ cd ${SHARE_FOLDER}
+ if [ "${REPO_NAME}" = "${TFM_NAME}" ] || [ "${REPO_NAME}" = "${TFM_TESTS_NAME}" ]; then
+ # These two need to remain as directories for now for further usage
+ tar -czf "${REPO_NAME}.tar.gz" "${REPO_NAME}"
+ else
+ tar -czf "${REPO_NAME}.tar.gz" "${REPO_NAME}" --remove-files
+ fi
cd $OLDPWD
fi
# Copy repos into pwd dir (workspace in CI), so each job would work
# on its own workspace
- cp -a -f "${SHARE_FOLDER}/${REPO_NAME}" "${WORKSPACE}/${REPO_NAME}"
+ cp -a -f "${SHARE_FOLDER}/${REPO_NAME}.tar.gz" "${WORKSPACE}/${REPO_NAME}.tar.gz"
+ # De-compress the repo
+ tar -xzf "${WORKSPACE}/${REPO_NAME}.tar.gz" -C "${WORKSPACE}"
+ rm "${WORKSPACE}/${REPO_NAME}.tar.gz"
+
+ # Output the info of the current repo
+ cd "${WORKSPACE}/${REPO_NAME}"
+ echo -e "Local Folder ${REPO_NAME} $(git rev-parse --short HEAD)\n"
+ cd $OLDPWD
}
# Take into consideration non-CI runs where SHARE_FOLDER variable
@@ -122,6 +144,7 @@
QA_TOOLS_REFSPEC="master"
QA_TOOLS_NAME="qa-tools"
+
# Array containing "<repo url>;"<repo name>;<refspec>" elements
dependency_repos=(
"${TFM_TESTS_PROJECT};${TFM_TESTS_NAME};${TFM_TESTS_REFSPEC}"
@@ -151,4 +174,8 @@
PSA_ARCH_TESTS_REFSPEC="${PSA_ARCH_TESTS_VERSION:-"$(parse_version ../tf-m-tests/tests_psa_arch/fetch_repo/CMakeLists.txt set\(PSA_ARCH_TESTS_VERSION \" 2)"}"
PSA_ARCH_TESTS_NAME="psa-arch-tests"
+if [-d "${SHARE_FOLDER}/${TFM_TESTS_NAME}" ]; then
+ rm -rf "${SHARE_FOLDER}/${TFM_TESTS_NAME}"
+fi
+
clone_repo_to_share_folder "${PSA_ARCH_TESTS_PROJECT}" "${PSA_ARCH_TESTS_NAME}" "${PSA_ARCH_TESTS_REFSPEC}"
diff --git a/lava_helper/codecov_helper.py b/lava_helper/codecov_helper.py
index ac60d72..4932c19 100644
--- a/lava_helper/codecov_helper.py
+++ b/lava_helper/codecov_helper.py
@@ -48,6 +48,8 @@
local_workspace = os.path.join(os.getenv("WORKSPACE"),os.getenv("JOB_NAME"),os.getenv("BUILD_NUMBER"))
run("mkdir -p %s" % (local_workspace))
run("cp -ar %s/. %s" % (os.getenv("SHARE_FOLDER"), local_workspace))
+ run("pwd")
+ run("ls -l")
for job_id, info in jobs.items():
job_dir = info["job_dir"]
metadata = info["metadata"]
@@ -66,8 +68,8 @@
dl_artifact("nspe/bin/tfm_ns.axf")
script_dir = os.path.dirname(__file__)
- run("python3 $SHARE_FOLDER/qa-tools/coverage-tool/coverage-reporting/intermediate_layer.py --config-json %s/trace2covjson.json --local-workspace %s" % (script_dir, local_workspace), cwd=job_dir)
- run("python3 $SHARE_FOLDER/qa-tools/coverage-tool/coverage-reporting/generate_info_file.py --workspace %s --json covjson.json" % (local_workspace), cwd=job_dir)
+ run("python3 %s/../../qa-tools/coverage-tool/coverage-reporting/intermediate_layer.py --config-json %s/trace2covjson.json --local-workspace %s" % (script_dir, script_dir, local_workspace), cwd=job_dir)
+ run("python3 %s/../../qa-tools/coverage-tool/coverage-reporting/generate_info_file.py --workspace %s --json covjson.json" % (script_dir, local_workspace), cwd=job_dir)
# Remove sources, coverage of which we're not interested in (e.g.
# 3rd party code).
run(
diff --git a/lava_helper/codecov_merge.sh b/lava_helper/codecov_merge.sh
index 7487624..11c0d8d 100755
--- a/lava_helper/codecov_merge.sh
+++ b/lava_helper/codecov_merge.sh
@@ -12,7 +12,7 @@
output_coverage_file=merge.info
output_json_file=merge.json
-python3 $SHARE_FOLDER/qa-tools/coverage-tool/coverage-reporting/merge.py \
+python3 $WORKSPACE/qa-tools/coverage-tool/coverage-reporting/merge.py \
$(find $input_folder -name "*.info" -exec echo "-a {}" \;) \
-o $output_coverage_file \