This is an automated email from the ASF dual-hosted git repository. sjanc pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mynewt-core.git
commit 83e9d10dd5afe0afaa14354db3cf749fcd3b2ab2 Author: Szymon Janc <[email protected]> AuthorDate: Wed Oct 14 17:19:27 2026 +0200 ci: Cache only .git directories of repositories Repositories cache and prepare job artifact contained full working trees, which are bigger after compression than git packs. Store only .git directories and check out working trees (including initialized submodules) after restore. Index is removed before checkout, since it contains file stats from system that created it. This reduces cache and artifact size from 1.65 GB to 1.26 GB, while checkout of all repositories takes about 15 seconds. It also lets each system check out files with its own git settings, so repositories restored on Windows from cache created on Linux no longer appear modified due to symlinks or file modes. Assisted-by: Claude:claude-opus-5-5 --- .github/actions/newt-project-restore/action.yml | 4 +- .github/actions/newt-project/action.yml | 30 +++++++------- .github/scripts/checkout_repos.sh | 54 +++++++++++++++++++++++++ .github/workflows/build.yml | 11 ++++- 4 files changed, 81 insertions(+), 18 deletions(-) diff --git a/.github/actions/newt-project-restore/action.yml b/.github/actions/newt-project-restore/action.yml index 084172c4b..7a1692a13 100644 --- a/.github/actions/newt-project-restore/action.yml +++ b/.github/actions/newt-project-restore/action.yml @@ -21,7 +21,8 @@ name: Restore newt project description: > Restore newt and newt project prepared by prepare job of the same workflow run, so that external repositories are downloaded only once - per run. + per run. Project contains only .git directories of repositories, + working trees are checked out after restore. inputs: path: @@ -46,6 +47,7 @@ runs: rm -rf "$RUNNER_TEMP/newt-project" mv "${{ inputs.path }}/.newt-bin" "$GITHUB_WORKSPACE/.newt-bin" echo "$GITHUB_WORKSPACE/.newt-bin" >> "$GITHUB_PATH" + bash "$GITHUB_WORKSPACE/.github/scripts/checkout_repos.sh" "${{ inputs.path }}" # Project was set up and upgraded by prepare job for tested commit - name: Show project info diff --git a/.github/actions/newt-project/action.yml b/.github/actions/newt-project/action.yml index f4ad58d8b..72f0d1b57 100644 --- a/.github/actions/newt-project/action.yml +++ b/.github/actions/newt-project/action.yml @@ -21,10 +21,12 @@ name: Set up newt project description: > Create newt project for testing current commit. External repositories are restored from cache and then upgraded by newt, so only changes - since cache was created are downloaded. Cache is created on Linux and - shared with other systems. Cache should be saved only by one job and - only from master branch, since only caches from default branch are - available to pull requests. + since cache was created are downloaded. Only .git directories are + cached and working trees are checked out after restore, which keeps + cache small and lets each system check out files with its own git + settings. Cache is created on Linux and shared with other systems. + Cache should be saved only by one job and only from master branch, + since only caches from default branch are available to pull requests. inputs: project: @@ -49,30 +51,28 @@ runs: run: | newt new "${{ inputs.path }}" cp -f "${{ inputs.project }}" "${{ inputs.path }}/project.yml" - echo "cache-prefix=newt-repos-shared-" >> "$GITHUB_OUTPUT" + echo "cache-prefix=newt-repos-git-" >> "$GITHUB_OUTPUT" - name: Restore repositories from cache id: restore uses: actions/cache/restore@v6 with: - path: ${{ inputs.path }}/repos + path: ${{ inputs.path }}/repos/*/.git key: ${{ steps.project.outputs.cache-prefix }}${{ github.sha }} restore-keys: ${{ steps.project.outputs.cache-prefix }} # Cache is created on Linux, Windows needs this to restore it enableCrossOsArchive: true - # Repositories restored on Windows from cache created on Linux may - # appear modified (e.g. due to symlinks or line endings). newt silently - # skips upgrade of modified repositories, so reset them to match local - # git settings and remove those still modified, so that they are - # cloned again. - - name: Clean up restored repositories - if: runner.os == 'Windows' && steps.restore.outputs.cache-matched-key != '' + # newt silently skips upgrade of modified repositories, so remove + # those still appearing modified after checkout (e.g. due to line + # endings on Windows), so that they are cloned again. + - name: Check out restored repositories + if: steps.restore.outputs.cache-matched-key != '' shell: bash working-directory: ${{ inputs.path }} run: | + bash "$GITHUB_WORKSPACE/.github/scripts/checkout_repos.sh" . for d in repos/*/; do - git -C "$d" reset -q --hard if [ -n "$(git -C "$d" status --porcelain --untracked-files=no)" ]; then echo "::warning::$d modified after restore from cache, removing" rm -rf "$d" @@ -113,6 +113,6 @@ runs: steps.restore.outputs.cache-matched-key != steps.key.outputs.key uses: actions/cache/save@v6 with: - path: ${{ inputs.path }}/repos + path: ${{ inputs.path }}/repos/*/.git key: ${{ steps.key.outputs.key }} enableCrossOsArchive: true diff --git a/.github/scripts/checkout_repos.sh b/.github/scripts/checkout_repos.sh new file mode 100755 index 000000000..395e199f6 --- /dev/null +++ b/.github/scripts/checkout_repos.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Recreate working trees of project repositories restored from cache or +# artifact. Only .git directories are stored there, since working trees +# can be checked out from those and are bigger after compression. +# Submodules initialized when repository was cloned are checked out +# from repositories stored in .git/modules. +# +# Usage: checkout_repos.sh <project directory> + +set -e + +cd "$1" + +for d in repos/*/; do + [ -d "$d.git" ] || continue + + # .git may come from other system, use settings of this one + opts=() + if [ "$RUNNER_OS" = "Windows" ]; then + opts=(-c core.symlinks=false -c core.filemode=false) + git -C "$d" config core.symlinks false + git -C "$d" config core.filemode false + fi + + # Index contains file stats from system that created it + rm -f "$d.git/index" + git -C "$d" reset -q --hard + + if [ -f "$d.gitmodules" ]; then + git "${opts[@]}" -C "$d" submodule update -q --recursive + if [ "$RUNNER_OS" = "Windows" ]; then + git -C "$d" submodule foreach -q --recursive \ + "git config core.symlinks false; git config core.filemode false" + fi + fi +done diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 17eff81e9..3dc6fb949 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,12 +70,19 @@ jobs: uses: ./.github/actions/newt-project with: save-cache: true + # Only .git directories of repositories are packed, working trees + # are checked out by jobs restoring project - name: Pack project shell: bash run: | cp -r .newt-bin build/.newt-bin - tar -C build -cf - . | zstd -T0 -3 -o newt-project.tar.zst - ls -lh newt-project.tar.zst + cd build + { + find . -mindepth 1 -maxdepth 1 ! -name repos + find repos -mindepth 1 -maxdepth 1 -name '.*' + find repos -mindepth 2 -maxdepth 2 -name .git + } | tar -cf - -T - | zstd -T0 -3 -o ../newt-project.tar.zst + ls -lh ../newt-project.tar.zst - name: Upload project uses: actions/upload-artifact@v7 with:
