[WIP] Estimate system requirement by using Intel perfspect for CI/CD and Documents

ChatQnA/tests/parse_perfspect.py

@@ -0,0 +1,47 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+import sys
+
+
+# jsonfile = "perfspect_2025-02-03_16-28-37" + os.sep + "louie-P14s-Gen-5.json"
+def parse_used_disk_from_report(jsonfile):
+    used_diskspace = 0
+    with open(jsonfile, "r") as file:
+        data = json.load(file)
+        for item in data["Filesystem"]:
+            if item["Mounted on"] == "/":
+                used_diskspace = float(item["Used"].replace("G", ""))
+                # print(item['Used'])
+    return used_diskspace
+
+
+jsonfile = "perfspect_2025-02-03_16-29-32" + os.sep + "louie-P14s-Gen-5_telem.json"
+
+
+def parse_used_memory_from_report(jsonfile):
+    max_memory_usage = 0
+    with open(jsonfile, "r") as file:
+        data = json.load(file)
+        # print(data)
+        for item in data["Memory Stats"]:
+            # if item['Mounted on'] == '/':
+            memory_usage = int(item["used"])
+            if memory_usage > max_memory_usage:
+                max_memory_usage = memory_usage
+    max_memory_usage_mb = float(memory_usage / 1000)
+    # print(max_memory_usage_mb)
+    return max_memory_usage_mb
+
+
+if __name__ == "__main__":
+
+    jsonfile = sys.argv[1]
+    if "telem" in jsonfile:
+        max_memory_usage = parse_used_memory_from_report(jsonfile)
+        print(max_memory_usage)
+    else:
+        disk_usage = parse_used_disk_from_report(jsonfile)
+        print(disk_usage)

ChatQnA/tests/test_compose_on_xeon.sh

@@ -178,13 +178,26 @@ function stop_docker() {
 function main() {
 
     stop_docker
+    # install perfspect
+    wget -qO- https://github.com/intel/PerfSpect/releases/latest/download/perfspect.tgz | tar xvz
+
+    # get used disk
+    list="$(./perfspect/perfspect report --filesystem 2>&1)"
+    list=$( for i in ` echo  $list `; do [[ $i =~ json ]] && echo $i ; done )
+    used_disk_before="$(python parse_perfspect.py $list 2>&1)"
+
     if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
     start_time=$(date +%s)
     start_services
     end_time=$(date +%s)
     duration=$((end_time-start_time))
     echo "Mega service start duration is $duration s" && sleep 1s
 
+    # trace  max used memory
+    list="$(./perfspect/perfspect telemetry --duration 10 2>&1)"
+    list=$( for i in ` echo  $list `; do [[ $i =~ json ]] && echo $i ; done )
+    max_used_mem="$(python parse_perfspect.py $list 2>&1)"
+
     if [ "${mode}" == "perf" ]; then
         python3 $WORKPATH/tests/chatqna_benchmark.py
     elif [ "${mode}" == "" ]; then
@@ -193,6 +206,16 @@ function main() {
         # validate_frontend
     fi
 
+    # get used disk
+    list="$(./perfspect/perfspect report --filesystem 2>&1)"
+    list=$( for i in ` echo  $list `; do [[ $i =~ json ]] && echo $i ; done )
+    used_disk_after="$(python parse_perfspect.py $list 2>&1)"
+
+    used_disk=$used_disk_after - $used_disk_before
+    # used disk and memory for the run
+    echo $used_disk
+    echo $max_used_mem
+
     stop_docker
     echo y | docker system prune