Share files: Compresses files before sharing

This compresses the code files before they are shared between jobs in an
attempt to lower the shared storage overhead.

Change-Id: Ic4ad813e47a6cda429141886ed0ec796598d7b85
Signed-off-by: Matthew Dalzell <matthew.dalzell@arm.com>
diff --git a/clone.sh b/clone.sh
index ee0246c..2d902b9 100755
--- a/clone.sh
+++ b/clone.sh
@@ -38,6 +38,8 @@
     local REPO_REFSPEC=$3
     local SYNC_CMD=$4
 
+    echo "Refspec when entering is ${REPO_REFSPEC}"
+
     echo "Repo: $REPO_URL $REPO_NAME $REPO_REFSPEC $SYNC_CMD"
 
     # In case repository is not defined, just skip it
@@ -46,18 +48,38 @@
         return 0
     fi
 
+    # The non-compressed repo should not be in there, removing
     if [ ! -d "${SHARE_FOLDER}/${REPO_NAME}" ]; then
+        echo "Removing non-compressed ${REPO_NAME} as it should not be stored."
+        rm -rf "${SHARE_FOLDER}/${REPO_NAME}"
+    fi
+
+    if [ ! -f "${SHARE_FOLDER}/${REPO_NAME}.tar.gz" ]; then
         git_clone $REPO_URL "${SHARE_FOLDER}/${REPO_NAME}"
+        # Compress for shared area
+        echo "Refspec at pull is ${REPO_REFSPEC}"
         git_checkout "${SHARE_FOLDER}/${REPO_NAME}" $REPO_REFSPEC $SYNC_CMD
-    else
-        cd "${SHARE_FOLDER}/${REPO_NAME}"
-        echo -e "Share Folder ${REPO_NAME} $(git rev-parse --short HEAD)\n"
+        cd ${SHARE_FOLDER}
+        if [ "${REPO_NAME}" = "${TFM_NAME}" ] || [ "${REPO_NAME}" = "${TFM_TESTS_NAME}" ]; then
+            # These two need to remain as directories for now for further usage
+            tar -czf "${REPO_NAME}.tar.gz" "${REPO_NAME}"
+        else
+            tar -czf "${REPO_NAME}.tar.gz" "${REPO_NAME}" --remove-files
+        fi
         cd $OLDPWD
     fi
 
     # Copy repos into pwd dir (workspace in CI), so each job would work
     # on its own workspace
-    cp -a -f "${SHARE_FOLDER}/${REPO_NAME}" "${WORKSPACE}/${REPO_NAME}"
+    cp -a -f "${SHARE_FOLDER}/${REPO_NAME}.tar.gz" "${WORKSPACE}/${REPO_NAME}.tar.gz"
+    # De-compress the repo
+    tar -xzf "${WORKSPACE}/${REPO_NAME}.tar.gz" -C "${WORKSPACE}"
+    rm "${WORKSPACE}/${REPO_NAME}.tar.gz"
+
+    # Output the info of the current repo
+    cd "${WORKSPACE}/${REPO_NAME}"
+    echo -e "Local Folder ${REPO_NAME} $(git rev-parse --short HEAD)\n"
+    cd $OLDPWD
 }
 
 # Take into consideration non-CI runs where SHARE_FOLDER variable
@@ -122,6 +144,7 @@
 QA_TOOLS_REFSPEC="master"
 QA_TOOLS_NAME="qa-tools"
 
+
 # Array containing "<repo url>;"<repo name>;<refspec>" elements
 dependency_repos=(
     "${TFM_TESTS_PROJECT};${TFM_TESTS_NAME};${TFM_TESTS_REFSPEC}"
@@ -151,4 +174,8 @@
 PSA_ARCH_TESTS_REFSPEC="${PSA_ARCH_TESTS_VERSION:-"$(parse_version ../tf-m-tests/tests_psa_arch/fetch_repo/CMakeLists.txt set\(PSA_ARCH_TESTS_VERSION \" 2)"}"
 PSA_ARCH_TESTS_NAME="psa-arch-tests"
 
+if [-d "${SHARE_FOLDER}/${TFM_TESTS_NAME}" ]; then
+    rm -rf "${SHARE_FOLDER}/${TFM_TESTS_NAME}"
+fi
+
 clone_repo_to_share_folder "${PSA_ARCH_TESTS_PROJECT}" "${PSA_ARCH_TESTS_NAME}" "${PSA_ARCH_TESTS_REFSPEC}"
diff --git a/lava_helper/codecov_helper.py b/lava_helper/codecov_helper.py
index ac60d72..4932c19 100644
--- a/lava_helper/codecov_helper.py
+++ b/lava_helper/codecov_helper.py
@@ -48,6 +48,8 @@
     local_workspace = os.path.join(os.getenv("WORKSPACE"),os.getenv("JOB_NAME"),os.getenv("BUILD_NUMBER"))
     run("mkdir -p %s" % (local_workspace))
     run("cp -ar %s/. %s" % (os.getenv("SHARE_FOLDER"), local_workspace))
+    run("pwd")
+    run("ls -l")
     for job_id, info in jobs.items():
         job_dir = info["job_dir"]
         metadata = info["metadata"]
@@ -66,8 +68,8 @@
             dl_artifact("nspe/bin/tfm_ns.axf")
 
             script_dir = os.path.dirname(__file__)
-            run("python3 $SHARE_FOLDER/qa-tools/coverage-tool/coverage-reporting/intermediate_layer.py --config-json %s/trace2covjson.json --local-workspace %s" % (script_dir, local_workspace), cwd=job_dir)
-            run("python3 $SHARE_FOLDER/qa-tools/coverage-tool/coverage-reporting/generate_info_file.py --workspace %s --json covjson.json" % (local_workspace), cwd=job_dir)
+            run("python3 %s/../../qa-tools/coverage-tool/coverage-reporting/intermediate_layer.py --config-json %s/trace2covjson.json --local-workspace %s" % (script_dir, script_dir, local_workspace), cwd=job_dir)
+            run("python3 %s/../../qa-tools/coverage-tool/coverage-reporting/generate_info_file.py --workspace %s --json covjson.json" % (script_dir, local_workspace), cwd=job_dir)
             # Remove sources, coverage of which we're not interested in (e.g.
             # 3rd party code).
             run(
diff --git a/lava_helper/codecov_merge.sh b/lava_helper/codecov_merge.sh
index 7487624..11c0d8d 100755
--- a/lava_helper/codecov_merge.sh
+++ b/lava_helper/codecov_merge.sh
@@ -12,7 +12,7 @@
 output_coverage_file=merge.info
 output_json_file=merge.json
 
-python3 $SHARE_FOLDER/qa-tools/coverage-tool/coverage-reporting/merge.py \
+python3 $WORKSPACE/qa-tools/coverage-tool/coverage-reporting/merge.py \
       $(find $input_folder -name "*.info" -exec echo "-a {}" \;) \
       -o $output_coverage_file \