Module: Mesa Branch: main Commit: 2487e18d4e25c56a70630ccf5faf5d2081fbee2f URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=2487e18d4e25c56a70630ccf5faf5d2081fbee2f
Author: Vignesh Raman <[email protected]> Date: Fri Dec 15 17:48:04 2023 +0530 ci: bare-metal: poe: Create strutured logs Use the CustomLogger class and CLI tool to create strutured logs for poe scripts which are used by broadcom and nouveau jobs. Renamed stage lint to code-validation and added python-test job which runs the tests for structured and customer logger to ci. Signed-off-by: Vignesh Raman <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25179> --- .gitlab-ci.yml | 4 +++- .gitlab-ci/bare-metal/poe-powered.sh | 20 ++++++++++++++++++++ .gitlab-ci/bare-metal/poe_run.py | 18 +++++++++++++++--- .gitlab-ci/container/debian/arm_test.sh | 2 ++ .gitlab-ci/image-tags.yml | 8 ++++---- .gitlab-ci/test/gitlab-ci.yml | 25 ++++++++++++++++++++++++- bin/ci/test/requirements.txt | 5 +++++ src/broadcom/ci/gitlab-ci-inc.yml | 1 + src/broadcom/ci/gitlab-ci.yml | 7 +++++++ src/gallium/drivers/nouveau/ci/gitlab-ci.yml | 2 ++ 10 files changed, 83 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f5135c03b13..66afe802c1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,6 +85,8 @@ variables: # Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name` # NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST! ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts + # Python scripts for structured logger + PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install" default: before_script: @@ -127,7 +129,7 @@ stages: - git-archive - build-x86_64 - build-misc - - lint + - code-validation - amd - intel - nouveau diff --git a/.gitlab-ci/bare-metal/poe-powered.sh b/.gitlab-ci/bare-metal/poe-powered.sh index d833a002db4..4d172b1e05b 100755 --- a/.gitlab-ci/bare-metal/poe-powered.sh +++ b/.gitlab-ci/bare-metal/poe-powered.sh @@ -12,6 +12,7 @@ # We're run from the root of the repo, make a helper var for our paths BM=$CI_PROJECT_DIR/install/bare-metal CI_COMMON=$CI_PROJECT_DIR/install/common +CI_INSTALL=$CI_PROJECT_DIR/install # Runner config checks if [ -z "$BM_SERIAL" ]; then @@ -181,8 +182,19 @@ if [ -n "$BM_BOOTCONFIG" ]; then fi set +e +STRUCTURED_LOG_FILE=job_detail.json +python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update dut_job_type "${DEVICE_TYPE}" +python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update farm "${FARM}" ATTEMPTS=3 +first_attempt=True while [ $((ATTEMPTS--)) -gt 0 ]; do + python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --create-dut-job dut_name "${CI_RUNNER_DESCRIPTION}" + # Update subtime time to CI_JOB_STARTED_AT only for the first run + if [ "$first_attempt" = "True" ]; then + python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update-dut-time submit "${CI_JOB_STARTED_AT}" + else + python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update-dut-time submit + fi python3 $BM/poe_run.py \ --dev="$BM_SERIAL" \ --powerup="$BM_POWERUP" \ @@ -192,10 +204,14 @@ while [ $((ATTEMPTS--)) -gt 0 ]; do if [ $ret -eq 2 ]; then echo "Did not detect boot sequence, retrying..." + python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close-dut-job + first_attempt=False else ATTEMPTS=0 fi done +python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close-dut-job +python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close set -e date +'%F %T' @@ -203,6 +219,10 @@ date +'%F %T' # Bring artifacts back from the NFS dir to the build dir where gitlab-runner # will look for them. cp -Rp /nfs/results/. results/ +if [ -f "${STRUCTURED_LOG_FILE}" ]; then + cp -p ${STRUCTURED_LOG_FILE} results/ + echo "Structured log file is available at ${ARTIFACTS_BASE_URL}/results/${STRUCTURED_LOG_FILE}" +fi date +'%F %T' diff --git a/.gitlab-ci/bare-metal/poe_run.py b/.gitlab-ci/bare-metal/poe_run.py index 88948863b30..0220acad545 100755 --- a/.gitlab-ci/bare-metal/poe_run.py +++ b/.gitlab-ci/bare-metal/poe_run.py @@ -24,23 +24,26 @@ import argparse import os import re -from serial_buffer import SerialBuffer import sys import threading +from custom_logger import CustomLogger +from serial_buffer import SerialBuffer class PoERun: - def __init__(self, args, test_timeout): + def __init__(self, args, test_timeout, logger): self.powerup = args.powerup self.powerdown = args.powerdown self.ser = SerialBuffer( args.dev, "results/serial-output.txt", "") self.test_timeout = test_timeout + self.logger = logger def print_error(self, message): RED = '\033[0;31m' NO_COLOR = '\033[0m' print(RED + message + NO_COLOR) + self.logger.update_status_fail(message) def logged_system(self, cmd): print("Running '{}'".format(cmd)) @@ -48,9 +51,11 @@ class PoERun: def run(self): if self.logged_system(self.powerup) != 0: + self.logger.update_status_fail("powerup failed") return 1 boot_detected = False + self.logger.create_job_phase("boot") for line in self.ser.lines(timeout=5 * 60, phase="bootloader"): if re.search("Booting Linux", line): boot_detected = True @@ -61,8 +66,10 @@ class PoERun: "Something wrong; couldn't detect the boot start up sequence") return 1 + self.logger.create_job_phase("test") for line in self.ser.lines(timeout=self.test_timeout, phase="test"): if re.search("---. end Kernel panic", line): + self.logger.update_status_fail("kernel panic") return 1 # Binning memory problems @@ -82,8 +89,10 @@ class PoERun: result = re.search("hwci: mesa: (\S*)", line) if result: if result.group(1) == "pass": + self.logger.update_dut_job("status", "pass") return 0 else: + self.logger.update_status_fail("test fail") return 1 self.print_error( @@ -103,10 +112,13 @@ def main(): '--test-timeout', type=int, help='Test phase timeout (minutes)', required=True) args = parser.parse_args() - poe = PoERun(args, args.test_timeout * 60) + logger = CustomLogger("job_detail.json") + logger.update_dut_time("start", None) + poe = PoERun(args, args.test_timeout * 60, logger) retval = poe.run() poe.logged_system(args.powerdown) + logger.update_dut_time("end", None) sys.exit(retval) diff --git a/.gitlab-ci/container/debian/arm_test.sh b/.gitlab-ci/container/debian/arm_test.sh index 8f3f432c5ac..855c7ea6fc3 100644 --- a/.gitlab-ci/container/debian/arm_test.sh +++ b/.gitlab-ci/container/debian/arm_test.sh @@ -17,6 +17,8 @@ DEPS=( openssh-server procps python3-distutils + python3-filelock + python3-fire python3-minimal python3-serial rsync diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 129f9b2b9b6..c8bff527555 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -18,14 +18,14 @@ variables: DEBIAN_X86_64_TEST_IMAGE_VK_PATH: "debian/x86_64_test-vk" DEBIAN_X86_64_TEST_ANDROID_IMAGE_PATH: "debian/x86_64_test-android" - DEBIAN_X86_64_TEST_ANDROID_TAG: "2023-11-18-deqp-runner" - DEBIAN_X86_64_TEST_GL_TAG: "2023-11-06-deqp-fixes" - DEBIAN_X86_64_TEST_VK_TAG: "2023-12-08-vkd3d" + DEBIAN_X86_64_TEST_ANDROID_TAG: "2023-12-14-structured-logs" + DEBIAN_X86_64_TEST_GL_TAG: "2023-12-14-structured-logs" + DEBIAN_X86_64_TEST_VK_TAG: "2023-12-14-structured-logs" ALPINE_X86_64_BUILD_TAG: "2023-10-30-ci-improv" ALPINE_X86_64_LAVA_SSH_TAG: "2023-06-26-first-version" FEDORA_X86_64_BUILD_TAG: "2023-12-11-meson-bump" - KERNEL_ROOTFS_TAG: "2023-12-08-firmware" + KERNEL_ROOTFS_TAG: "2023-12-14-structured-logs" KERNEL_TAG: "v6.6.4-for-mesa-ci-e4f4c500f7fb" KERNEL_REPO: "gfx-ci/linux" diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index 2b90efd300d..108a6209e02 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -22,7 +22,7 @@ .formatting-check: # Cancel job if a newer commit is pushed to the same branch interruptible: true - stage: lint + stage: code-validation extends: - .use-debian/x86_64_build variables: @@ -57,6 +57,29 @@ clang-format: - clang-format-${LLVM_VERSION} --version - ninja -C build clang-format +.test-check: + # Cancel job if a newer commit is pushed to the same branch + interruptible: true + stage: code-validation + extends: + - .use-debian/x86_64_build + variables: + GIT_STRATEGY: fetch + timeout: 10m + +python-test: + extends: + - .test-check + script: + - cd bin/ci + - pip install --break-system-packages -r test/requirements.txt + - PYTHONPATH=. pytest -v + rules: + - !reference [.disable-farm-mr-rules, rules] + - !reference [.never-post-merge-rules, rules] + - changes: + - bin/ci/**/* + .test-gl: extends: - .test diff --git a/bin/ci/test/requirements.txt b/bin/ci/test/requirements.txt new file mode 100644 index 00000000000..f80621af285 --- /dev/null +++ b/bin/ci/test/requirements.txt @@ -0,0 +1,5 @@ +filelock==3.12.4 +fire==0.5.0 +mock==5.1.0 +polars==0.19.3 +pytest==7.4.2 diff --git a/src/broadcom/ci/gitlab-ci-inc.yml b/src/broadcom/ci/gitlab-ci-inc.yml index 4a7f9926be5..a6ad2aea692 100644 --- a/src/broadcom/ci/gitlab-ci-inc.yml +++ b/src/broadcom/ci/gitlab-ci-inc.yml @@ -107,6 +107,7 @@ variables: HWCI_TEST_SCRIPT: "/install/deqp-runner.sh" FLAKES_CHANNEL: "#videocore-ci" + FARM: igalia timeout: 20m .broadcom-test:arm64: diff --git a/src/broadcom/ci/gitlab-ci.yml b/src/broadcom/ci/gitlab-ci.yml index d942161f8b6..e451daccae2 100644 --- a/src/broadcom/ci/gitlab-ci.yml +++ b/src/broadcom/ci/gitlab-ci.yml @@ -14,6 +14,7 @@ vc4-rpi3-gl:arm32: HWCI_KERNEL_MODULES: vc4 DEQP_SUITE: broadcom-rpi3 HWCI_START_WESTON: 1 + DEVICE_TYPE: rpi3 vc4-rpi3-gl-piglit-full:arm32: extends: @@ -24,6 +25,7 @@ vc4-rpi3-gl-piglit-full:arm32: - igalia-fullrun variables: DEQP_SUITE: broadcom-rpi3-piglit-full + DEVICE_TYPE: rpi3 v3d-rpi4-gl:arm64: @@ -39,6 +41,7 @@ v3d-rpi4-gl:arm64: HWCI_START_WESTON: 1 DEQP_SUITE: broadcom-rpi4 DEQP_FRACTION: 2 + DEVICE_TYPE: rpi4 v3d-rpi4-gl-full:arm64: extends: @@ -52,6 +55,7 @@ v3d-rpi4-gl-full:arm64: variables: TEST_PHASE_TIMEOUT: 40 DEQP_FRACTION: 1 + DEVICE_TYPE: rpi4 v3d-rpi4-traces:arm64: @@ -66,6 +70,7 @@ v3d-rpi4-traces:arm64: PIGLIT_TRACES_FILE: traces-broadcom.yml PIGLIT_REPLAY_DEVICE_NAME: "broadcom-rpi4" PIGLIT_RESULTS: "broadcom-rpi4-replay" + DEVICE_TYPE: rpi4 tags: - igalia-rpi4 @@ -84,6 +89,7 @@ v3dv-rpi4-vk:arm64: GPU_VERSION: broadcom-rpi4 HWCI_KERNEL_MODULES: v3d,vc4 VK_DRIVER: broadcom + DEVICE_TYPE: rpi4 tags: - igalia-rpi4 @@ -99,3 +105,4 @@ v3dv-rpi4-vk-full:arm64: variables: TEST_PHASE_TIMEOUT: 80 DEQP_FRACTION: 1 + DEVICE_TYPE: rpi4 diff --git a/src/gallium/drivers/nouveau/ci/gitlab-ci.yml b/src/gallium/drivers/nouveau/ci/gitlab-ci.yml index 7a5e33b6ce8..90ee2b58d82 100644 --- a/src/gallium/drivers/nouveau/ci/gitlab-ci.yml +++ b/src/gallium/drivers/nouveau/ci/gitlab-ci.yml @@ -10,3 +10,5 @@ gk20a-gles: variables: HWCI_TEST_SCRIPT: "/install/deqp-runner.sh" DEQP_SUITE: nouveau-gk20a + FARM : anholt + DEVICE_TYPE: anholt-jetson
