Hello, Many people found the current behavior of git-r3 eclass unfortunate, lightly saying. Most importantly, I underestimated how many packages actually require pretty complete '.git' metadata in the checkout, including complete history. Most of it was collected in bug #489100 [1].
I've finally found time to get around and rewrite git-r3 to properly support all those use cases. I'd like to submit patches implementing that for the mailing list discussion. It is an important change of the eclass behavior. Previously it aimed at downloading minimal amount of data necessary for checking out the requested tree version. The new version mirrors complete upstream repository, including all branches and tags. It also ensures that a proper complete '.git' directory resides in the checkout. Patch #1 removes shallow clone support completely, and uses full mirroring instead. The new code would require even more conditionals to keep the two different behaviors around, and considering the negative reception it had I don't feel like maintaining the extra code is worth the effort. Patch #2 switches the internal git-r3 refs that are used to track changes between updates from 'refs/heads' namespace to dedicated 'refs/git-r3'. Long story short, this prevents 'git fetch --prune' from removing them on next fetch and makes 'git clone' ignore them when cloning the repository. Patch #3 actually makes checkout clone the repository rather than hack it around in order to perform the checkout. It gives all the nice '.git' structure build systems want, and also removes the need to use lock files. Patch #4 introduces some more black magic between fetch & checkout. Long story short, it makes passing refs (rather than commit ids) possible to checkout, and 'git branch' reports the correct branch rather than 'detached ...' state. Attaching the complete eclass for easier reading as well. Please review. [1]:https://bugs.gentoo.org/show_bug.cgi?id=489100 -- Best regards, Michał Górny
From d420815bc096e1eb86f9d2a90c303f0129865d6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgo...@gentoo.org>
Date: Fri, 21 Feb 2014 00:01:06 +0100
Subject: [PATCH 1/4] Remove shallow clone support, always use full clones.
As the community demanded, shallow clones are no longer supported.
The eclass completely removes old ones, so that the repos are re-fetched
from broken git servers with no extra action. Additionally, every branch
and tag is fetched from the remote.
---
eclass/git-r3.eclass | 195 +++++----------------------------------------------
1 file changed, 18 insertions(+), 177 deletions(-)
diff --git a/eclass/git-r3.eclass b/eclass/git-r3.eclass
index 0341972..487b77b 100644
--- a/eclass/git-r3.eclass
+++ b/eclass/git-r3.eclass
@@ -8,8 +8,7 @@
# @BLURB: Eclass for fetching and unpacking git repositories.
# @DESCRIPTION:
# Third generation eclass for easing maitenance of live ebuilds using
-# git as remote repository. The eclass supports lightweight (shallow)
-# clones and bare clones of submodules.
+# git as remote repository.
case "${EAPI:-0}" in
0|1|2|3|4|5)
@@ -83,18 +82,6 @@ fi
#
# EGIT_CHECKOUT_DIR=${WORKDIR}/${P}
-# @ECLASS-VARIABLE: EGIT_NONSHALLOW
-# @DEFAULT_UNSET
-# @DESCRIPTION:
-# Disable performing shallow fetches/clones. Shallow clones have
-# a fair number of limitations. Therefore, if you'd like the eclass to
-# perform complete clones instead, set this to a non-null value.
-#
-# This variable can be set in make.conf and ebuilds. The make.conf
-# value specifies user-specific default, while ebuilds may use it
-# to force deep clones when the server does not support shallow clones
-# (e.g. Google Code).
-
# @FUNCTION: _git-r3_env_setup
# @INTERNAL
# @DESCRIPTION:
@@ -217,14 +204,13 @@ _git-r3_set_gitdir() {
fi
addwrite "${EGIT3_STORE_DIR}"
+ if [[ -e ${GIT_DIR}/shallow ]]; then
+ einfo "${GIT_DIR} was a shallow clone, recreating..."
+ rm -r "${GIT_DIR}" || die
+ fi
if [[ ! -d ${GIT_DIR} ]]; then
mkdir "${GIT_DIR}" || die
git init --bare || die
-
- if [[ ! ${EGIT_NONSHALLOW} ]]; then
- # avoid auto-unshallow :)
- touch "${GIT_DIR}"/shallow || die
- fi
fi
}
@@ -267,93 +253,6 @@ _git-r3_set_submodules() {
done < <(echo "${data}" | git config -f /dev/fd/0 -l || die)
}
-# @FUNCTION: _git-r3_smart_fetch
-# @USAGE: <git-fetch-args>...
-# @DESCRIPTION:
-# Try fetching without '--depth' and switch to '--depth 1' if that
-# will involve less objects fetched.
-_git-r3_smart_fetch() {
- debug-print-function ${FUNCNAME} "$@"
-
- local sed_regexp='.*Counting objects: \([0-9]*\), done\..*'
-
- # start the main fetch
- local cmd=( git fetch --progress "${@}" )
- echo "${cmd[@]}" >&2
-
- # we copy the output to the 'sed' pipe for parsing. whenever sed finds
- # the process count, it quits quickly to avoid delays in writing it.
- # then, we start a dummy 'cat' to keep the pipe alive
-
- "${cmd[@]}" 2>&1 \
- | tee >(
- sed -n -e "/${sed_regexp}/{s/${sed_regexp}/\1/p;q}" \
- > "${T}"/git-r3_main.count
- exec cat >/dev/null
- ) &
- local main_pid=${!}
-
- # start the helper process
- _git-r3_sub_fetch() {
- # wait for main fetch to get object count; if the server doesn't
- # output it, we won't even launch the parallel process
- while [[ ! -s ${T}/git-r3_main.count ]]; do
- sleep 0.25
- done
-
- # ok, let's see if parallel fetch gives us smaller count
- # --dry-run will prevent it from writing to the local clone
- # and sed should terminate git with SIGPIPE
- local sub_count=$(git fetch --progress --dry-run --depth 1 "${@}" 2>&1 \
- | sed -n -e "/${sed_regexp}/{s/${sed_regexp}/\1/p;q}")
- local main_count=$(<"${T}"/git-r3_main.count)
-
- # let's be real sure that '--depth 1' will be good for us.
- # note that we have purely objects counts, and '--depth 1'
- # may involve much bigger objects
- if [[ ${main_count} && ${main_count} -ge $(( sub_count * 3/2 )) ]]
- then
- # signal that we want shallow fetch instead,
- # and terminate the non-shallow fetch process
- touch "${T}"/git-r3_want_shallow || die
- kill ${main_pid} &>/dev/null
- exit 0
- fi
-
- exit 1
- }
- _git-r3_sub_fetch "${@}" &
- local sub_pid=${!}
-
- # wait for main process to terminate, either of its own
- # or by signal from subprocess
- wait ${main_pid}
- local main_ret=${?}
-
- # wait for subprocess to terminate, killing it if necessary.
- # if main fetch finished before it, there's no point in keeping
- # it alive. if main fetch was killed by it, it's done anyway
- kill ${sub_pid} &>/dev/null
- wait ${sub_pid}
-
- # now see if subprocess wanted to tell us something...
- if [[ -f ${T}/git-r3_want_shallow ]]; then
- rm "${T}"/git-r3_want_shallow || die
-
- # if fetch finished already (wasn't killed), ignore it
- [[ ${main_ret} -eq 0 ]] && return 0
-
- # otherwise, restart as shallow fetch
- einfo "Restarting fetch using --depth 1 to save bandwidth ..."
- local cmd=( git fetch --progress --depth 1 "${@}" )
- echo "${cmd[@]}" >&2
- "${cmd[@]}"
- main_ret=${?}
- fi
-
- return ${main_ret}
-}
-
# @FUNCTION: _git-r3_is_local_repo
# @USAGE: <repo-uri>
# @INTERNAL
@@ -425,7 +324,7 @@ git-r3_fetch() {
for r in "${repos[@]}"; do
einfo "Fetching ${remote_ref} from ${r} ..."
- local is_branch lookup_ref
+ local lookup_ref
if [[ ${remote_ref} == refs/heads/* || ${remote_ref} == HEAD ]]
then
is_branch=1
@@ -436,80 +335,22 @@ git-r3_fetch() {
lookup_ref=refs/tags/${remote_ref}
fi
- # first, try ls-remote to see if ${remote_ref} is a real ref
- # and not a commit id. if it succeeds, we can pass ${remote_ref}
- # to 'fetch'. otherwise, we will just fetch everything
-
- # split on whitespace
- local ref=(
- $(git ls-remote "${r}" "${lookup_ref}" || echo __FAIL__)
+ local fetch_command=(
+ git fetch --prune "${r}"
+ # mirror the remote branches as local branches
+ "refs/heads/*:refs/heads/*"
+ # pull tags explicitly in order to prune them properly
+ "refs/tags/*:refs/tags/*"
)
- # normally, ref[0] is a hash, so we can do magic strings here
- [[ ${ref[0]} == __FAIL__ ]] && continue
-
- local nonshallow=${EGIT_NONSHALLOW}
- local ref_param=()
- if [[ ! ${ref[0]} ]]; then
- nonshallow=1
- fi
-
- # trying to do a shallow clone of a local repo makes git try to
- # write to the repo. we don't want that to happen.
- _git-r3_is_local_repo "${r}" && nonshallow=1
-
- # 1. if we need a non-shallow clone and we have a shallow one,
- # we need to unshallow it explicitly.
- # 2. if we want a shallow clone, we just pass '--depth 1'
- # to the first fetch in the repo. passing '--depth'
- # to further requests usually results in more data being
- # downloaded than without it.
- # 3. if we update a shallow clone, we try without '--depth'
- # first since that usually transfers less data. however,
- # we use git-r3_smart_fetch that can switch into '--depth 1'
- # if that looks beneficial.
-
- local fetch_command=( git fetch )
- if [[ ${nonshallow} ]]; then
- if [[ -f ${GIT_DIR}/shallow ]]; then
- ref_param+=( --unshallow )
- fi
- # fetch all branches
- ref_param+=( "refs/heads/*:refs/remotes/origin/*" )
- else
- # 'git show-ref --heads' returns 1 when there are no branches
- if ! git show-ref --heads -q; then
- ref_param+=( --depth 1 )
- else
- fetch_command=( _git-r3_smart_fetch )
- fi
- fi
-
- # now, another important thing. we may only fetch a remote
- # branch directly to a local branch. Otherwise, we need to fetch
- # the commit and re-create the branch on top of it.
-
- if [[ ${ref[0]} ]]; then
- if [[ ${is_branch} ]]; then
- ref_param+=( -f "${remote_ref}:${local_id}/__main__" )
- else
- ref_param+=( "refs/tags/${remote_ref}" )
- fi
- fi
-
- # if ${remote_ref} is branch or tag, ${ref[@]} will contain
- # the respective commit id. otherwise, it will be an empty
- # array, so the following won't evaluate to a parameter.
- set -- "${fetch_command[@]}" --no-tags "${r}" "${ref_param[@]}"
+ set -- "${fetch_command[@]}"
echo "${@}" >&2
if "${@}"; then
- if [[ ! ${is_branch} ]]; then
- set -- git branch -f "${local_id}/__main__" \
- "${ref[0]:-${remote_ref}}"
- echo "${@}" >&2
- if ! "${@}"; then
- die "Creating branch for ${remote_ref} failed (wrong ref?)."
- fi
+ set -- git branch -f "${local_id}/__main__" \
+ "${ref[0]:-${remote_ref}}"
+ echo "${@}" >&2
+ if ! "${@}"; then
+ die "Creating branch for ${remote_ref} failed (wrong ref?)."
fi
success=1
--
1.9.0
From c1e29395fce81f4a1385ed5a820a2c7773ff8a14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgo...@gentoo.org>
Date: Fri, 21 Feb 2014 00:11:04 +0100
Subject: [PATCH 2/4] Use dedicated refs rather than branches for eclass data.
---
eclass/git-r3.eclass | 30 +++++++++---------------------
1 file changed, 9 insertions(+), 21 deletions(-)
diff --git a/eclass/git-r3.eclass b/eclass/git-r3.eclass
index 487b77b..fb75cff 100644
--- a/eclass/git-r3.eclass
+++ b/eclass/git-r3.eclass
@@ -312,7 +312,7 @@ git-r3_fetch() {
local branch=${EGIT_BRANCH:+refs/heads/${EGIT_BRANCH}}
local remote_ref=${2:-${EGIT_COMMIT:-${branch:-HEAD}}}
local local_id=${3:-${CATEGORY}/${PN}/${SLOT%/*}}
- local local_ref=refs/heads/${local_id}/__main__
+ local local_ref=refs/git-r3/${local_id}/__main__
[[ ${repos[@]} ]] || die "No URI provided and EGIT_REPO_URI unset"
@@ -324,17 +324,6 @@ git-r3_fetch() {
for r in "${repos[@]}"; do
einfo "Fetching ${remote_ref} from ${r} ..."
- local lookup_ref
- if [[ ${remote_ref} == refs/heads/* || ${remote_ref} == HEAD ]]
- then
- is_branch=1
- lookup_ref=${remote_ref}
- else
- # ls-remote by commit is going to fail anyway,
- # so we may as well pass refs/tags/ABCDEF...
- lookup_ref=refs/tags/${remote_ref}
- fi
-
local fetch_command=(
git fetch --prune "${r}"
# mirror the remote branches as local branches
@@ -346,11 +335,10 @@ git-r3_fetch() {
set -- "${fetch_command[@]}"
echo "${@}" >&2
if "${@}"; then
- set -- git branch -f "${local_id}/__main__" \
- "${ref[0]:-${remote_ref}}"
+ set -- git update-ref "${local_ref}" "${remote_ref}"
echo "${@}" >&2
if ! "${@}"; then
- die "Creating branch for ${remote_ref} failed (wrong ref?)."
+ die "Referencing ${remote_ref} failed (wrong ref?)."
fi
success=1
@@ -429,7 +417,7 @@ git-r3_checkout() {
einfo "Checking out ${repos[0]} to ${out_dir} ..."
- if ! git cat-file -e refs/heads/"${local_id}"/__main__
+ if ! git cat-file -e refs/git-r3/"${local_id}"/__main__
then
if [[ ${EVCS_OFFLINE} ]]; then
die "No local clone of ${repos[0]}. Unable to work with EVCS_OFFLINE."
@@ -448,7 +436,7 @@ git-r3_checkout() {
done
rm "${lockfile_l}" || die
- set -- git checkout -f "${local_id}"/__main__ .
+ set -- git checkout -f refs/git-r3/"${local_id}"/__main__ .
echo "${@}" >&2
"${@}"
local ret=${?}
@@ -456,12 +444,12 @@ git-r3_checkout() {
# Remove the lock!
rm "${lockfile}" || die
- [[ ${ret} == 0 ]] || die "git checkout ${local_id}/__main__ failed"
+ [[ ${ret} == 0 ]] || die "git checkout refs/git-r3/${local_id}/__main__ failed"
# diff against previous revision (if any)
- local new_commit_id=$(git rev-parse --verify "${local_id}"/__main__)
+ local new_commit_id=$(git rev-parse --verify refs/git-r3/"${local_id}"/__main__)
local old_commit_id=$(
- git rev-parse --verify "${local_id}"/__old__ 2>/dev/null
+ git rev-parse --verify refs/git-r3/"${local_id}"/__old__ 2>/dev/null
)
if [[ ! ${old_commit_id} ]]; then
@@ -482,7 +470,7 @@ git-r3_checkout() {
echo " at the commit: ${new_commit_id}"
fi
fi
- git branch -f "${local_id}"/{__old__,__main__} || die
+ git update-ref refs/git-r3/"${local_id}"/{__old__,__main__} || die
# recursively checkout submodules
if [[ -f ${GIT_WORK_TREE}/.gitmodules ]]; then
--
1.9.0
From 07e8552a3ec20b54ee5537ebaeb05217e4ed4d40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgo...@gentoo.org>
Date: Fri, 21 Feb 2014 00:30:34 +0100
Subject: [PATCH 3/4] Clone the repository into working directory.
This should fix most of the issues weird build systems have. Not all
possible -- e.g. they still can't know what branch is it -- but it's
a good step ahead.
---
eclass/git-r3.eclass | 60 +++++++++++++++++++++-------------------------------
1 file changed, 24 insertions(+), 36 deletions(-)
diff --git a/eclass/git-r3.eclass b/eclass/git-r3.eclass
index fb75cff..fd3610d 100644
--- a/eclass/git-r3.eclass
+++ b/eclass/git-r3.eclass
@@ -410,53 +410,46 @@ git-r3_checkout() {
local out_dir=${2:-${EGIT_CHECKOUT_DIR:-${WORKDIR}/${P}}}
local local_id=${3:-${CATEGORY}/${PN}/${SLOT%/*}}
- local -x GIT_DIR GIT_WORK_TREE
+ local -x GIT_DIR
_git-r3_set_gitdir "${repos[0]}"
- GIT_WORK_TREE=${out_dir}
- mkdir -p "${GIT_WORK_TREE}" || die
einfo "Checking out ${repos[0]} to ${out_dir} ..."
- if ! git cat-file -e refs/git-r3/"${local_id}"/__main__
- then
+ if ! git cat-file -e refs/git-r3/"${local_id}"/__main__; then
if [[ ${EVCS_OFFLINE} ]]; then
die "No local clone of ${repos[0]}. Unable to work with EVCS_OFFLINE."
else
die "Logic error: no local clone of ${repos[0]}. git-r3_fetch not used?"
fi
fi
+ local new_commit_id=$(
+ git rev-parse --verify refs/git-r3/"${local_id}"/__main__
+ )
- # Note: this is a hack to avoid parallel checkout issues.
- # I will try to handle it without locks when I have more time.
- local lockfile=${GIT_DIR}/.git-r3_checkout_lock
- local lockfile_l=${lockfile}.${BASHPID}
- touch "${lockfile_l}" || die
- until ln "${lockfile_l}" "${lockfile}" &>/dev/null; do
- sleep 1
- done
- rm "${lockfile_l}" || die
-
- set -- git checkout -f refs/git-r3/"${local_id}"/__main__ .
+ set -- git clone --shared --no-checkout "${GIT_DIR}" "${out_dir}"/
echo "${@}" >&2
- "${@}"
- local ret=${?}
-
- # Remove the lock!
- rm "${lockfile}" || die
+ "${@}" || die "git clone (for checkout) failed"
- [[ ${ret} == 0 ]] || die "git checkout refs/git-r3/${local_id}/__main__ failed"
+ git-r3_sub_checkout() {
+ local -x GIT_DIR=${out_dir}/.git
+ local -x GIT_WORK_TREE=${out_dir}
- # diff against previous revision (if any)
- local new_commit_id=$(git rev-parse --verify refs/git-r3/"${local_id}"/__main__)
- local old_commit_id=$(
- git rev-parse --verify refs/git-r3/"${local_id}"/__old__ 2>/dev/null
- )
+ set -- git checkout --quiet "${new_commit_id}"
+ echo "${@}" >&2
+ "${@}" || die "git checkout ${new_commit_id} failed"
+ }
+ git-r3_sub_checkout
- if [[ ! ${old_commit_id} ]]; then
+ if ! git cat-file -e refs/git-r3/"${local_id}"/__old__; then
echo "GIT NEW branch -->"
echo " repository: ${repos[0]}"
echo " at the commit: ${new_commit_id}"
else
+ # diff against previous revision
+ local old_commit_id=$(
+ git rev-parse --verify refs/git-r3/"${local_id}"/__old__ 2>/dev/null
+ )
+
echo "GIT update -->"
echo " repository: ${repos[0]}"
# write out message based on the revisions
@@ -473,10 +466,10 @@ git-r3_checkout() {
git update-ref refs/git-r3/"${local_id}"/{__old__,__main__} || die
# recursively checkout submodules
- if [[ -f ${GIT_WORK_TREE}/.gitmodules ]]; then
+ if [[ -f ${out_dir}/.gitmodules ]]; then
local submodules
_git-r3_set_submodules \
- "$(<"${GIT_WORK_TREE}"/.gitmodules)"
+ "$(<"${out_dir}"/.gitmodules)"
while [[ ${submodules[@]} ]]; do
local subname=${submodules[0]}
@@ -487,7 +480,7 @@ git-r3_checkout() {
url=${repos[0]%%/}/${url}
fi
- git-r3_checkout "${url}" "${GIT_WORK_TREE}/${path}" \
+ git-r3_checkout "${url}" "${out_dir}/${path}" \
"${local_id}/${subname}"
submodules=( "${submodules[@]:3}" ) # shift
@@ -497,11 +490,6 @@ git-r3_checkout() {
# keep this *after* submodules
export EGIT_DIR=${GIT_DIR}
export EGIT_VERSION=${new_commit_id}
-
- # create a fake '.git' directory to satisfy 'git rev-parse HEAD'
- GIT_DIR=${GIT_WORK_TREE}/.git
- git init || die
- echo "${EGIT_VERSION}" > "${GIT_WORK_TREE}"/.git/HEAD || die
}
# @FUNCTION: git-r3_peek_remote_ref
--
1.9.0
From 9be0d632d836d596b121eca03ece49b4f2e7fc8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgo...@gentoo.org>
Date: Fri, 21 Feb 2014 17:10:09 +0100
Subject: [PATCH 4/4] Pass branches and tags as symbolic refs.
Now, the checkout phase is fully aware of what we were checking out, and
even 'git branch' shows correct value after the checkout.
---
eclass/git-r3.eclass | 31 ++++++++++++++++++++++++++-----
1 file changed, 26 insertions(+), 5 deletions(-)
diff --git a/eclass/git-r3.eclass b/eclass/git-r3.eclass
index fd3610d..a50890d 100644
--- a/eclass/git-r3.eclass
+++ b/eclass/git-r3.eclass
@@ -322,7 +322,7 @@ git-r3_fetch() {
# try to fetch from the remote
local r success
for r in "${repos[@]}"; do
- einfo "Fetching ${remote_ref} from ${r} ..."
+ einfo "Fetching ${r} ..."
local fetch_command=(
git fetch --prune "${r}"
@@ -335,7 +335,20 @@ git-r3_fetch() {
set -- "${fetch_command[@]}"
echo "${@}" >&2
if "${@}"; then
- set -- git update-ref "${local_ref}" "${remote_ref}"
+ # now let's see what the user wants from us
+ local full_remote_ref=$(
+ git rev-parse --verify --symbolic-full-name "${remote_ref}"
+ )
+
+ if [[ ${full_remote_ref} ]]; then
+ # when we are given a ref, create a symbolic ref
+ # so that we preserve the actual argument
+ set -- git symbolic-ref "${local_ref}" "${full_remote_ref}"
+ else
+ # otherwise, we were likely given a commit id
+ set -- git update-ref --no-deref "${local_ref}" "${remote_ref}"
+ fi
+
echo "${@}" >&2
if ! "${@}"; then
die "Referencing ${remote_ref} failed (wrong ref?)."
@@ -422,6 +435,9 @@ git-r3_checkout() {
die "Logic error: no local clone of ${repos[0]}. git-r3_fetch not used?"
fi
fi
+ local remote_ref=$(
+ git symbolic-ref --quiet refs/git-r3/"${local_id}"/__main__
+ )
local new_commit_id=$(
git rev-parse --verify refs/git-r3/"${local_id}"/__main__
)
@@ -434,9 +450,14 @@ git-r3_checkout() {
local -x GIT_DIR=${out_dir}/.git
local -x GIT_WORK_TREE=${out_dir}
- set -- git checkout --quiet "${new_commit_id}"
+ set -- git checkout --quiet
+ if [[ ${remote_ref} ]]; then
+ set -- "${@}" "${remote_ref#refs/heads/}"
+ else
+ set -- "${@}" "${new_commit_id}"
+ fi
echo "${@}" >&2
- "${@}" || die "git checkout ${new_commit_id} failed"
+ "${@}" || die "git checkout ${remote_ref:-${new_commit_id}} failed"
}
git-r3_sub_checkout
@@ -463,7 +484,7 @@ git-r3_checkout() {
echo " at the commit: ${new_commit_id}"
fi
fi
- git update-ref refs/git-r3/"${local_id}"/{__old__,__main__} || die
+ git update-ref --no-deref refs/git-r3/"${local_id}"/{__old__,__main__} || die
# recursively checkout submodules
if [[ -f ${out_dir}/.gitmodules ]]; then
--
1.9.0
# Copyright 1999-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: /var/cvsroot/gentoo-x86/eclass/git-r3.eclass,v 1.23 2013/11/15
23:03:23 mgorny Exp $
# @ECLASS: git-r3.eclass
# @MAINTAINER:
# MichaŠGórny <mgo...@gentoo.org>
# @BLURB: Eclass for fetching and unpacking git repositories.
# @DESCRIPTION:
# Third generation eclass for easing maitenance of live ebuilds using
# git as remote repository.
case "${EAPI:-0}" in
0|1|2|3|4|5)
;;
*)
die "Unsupported EAPI=${EAPI} (unknown) for ${ECLASS}"
;;
esac
if [[ ! ${_GIT_R3} ]]; then
inherit eutils
fi
EXPORT_FUNCTIONS src_unpack
if [[ ! ${_GIT_R3} ]]; then
if [[ ! ${_INHERITED_BY_GIT_2} ]]; then
DEPEND=">=dev-vcs/git-1.8.2.1"
fi
# @ECLASS-VARIABLE: EGIT3_STORE_DIR
# @DESCRIPTION:
# Storage directory for git sources.
#
# EGIT3_STORE_DIR=${DISTDIR}/git3-src
# @ECLASS-VARIABLE: EGIT_REPO_URI
# @REQUIRED
# @DESCRIPTION:
# URIs to the repository, e.g. git://foo, https://foo. If multiple URIs
# are provided, the eclass will consider them as fallback URIs to try
# if the first URI does not work.
#
# It can be overriden via env using ${PN}_LIVE_REPO variable.
#
# Can be a whitespace-separated list or an array.
#
# Example:
# @CODE
# EGIT_REPO_URI="git://a/b.git https://c/d.git";
# @CODE
# @ECLASS-VARIABLE: EVCS_OFFLINE
# @DEFAULT_UNSET
# @DESCRIPTION:
# If non-empty, this variable prevents any online operations.
# @ECLASS-VARIABLE: EGIT_BRANCH
# @DEFAULT_UNSET
# @DESCRIPTION:
# The branch name to check out. If unset, the upstream default (HEAD)
# will be used.
#
# It can be overriden via env using ${PN}_LIVE_BRANCH variable.
# @ECLASS-VARIABLE: EGIT_COMMIT
# @DEFAULT_UNSET
# @DESCRIPTION:
# The tag name or commit identifier to check out. If unset, newest
# commit from the branch will be used. If set, EGIT_BRANCH will
# be ignored.
#
# It can be overriden via env using ${PN}_LIVE_COMMIT variable.
# @ECLASS-VARIABLE: EGIT_CHECKOUT_DIR
# @DESCRIPTION:
# The directory to check the git sources out to.
#
# EGIT_CHECKOUT_DIR=${WORKDIR}/${P}
# @FUNCTION: _git-r3_env_setup
# @INTERNAL
# @DESCRIPTION:
# Set the eclass variables as necessary for operation. This can involve
# setting EGIT_* to defaults or ${PN}_LIVE_* variables.
_git-r3_env_setup() {
debug-print-function ${FUNCNAME} "$@"
local esc_pn livevar
esc_pn=${PN//[-+]/_}
livevar=${esc_pn}_LIVE_REPO
EGIT_REPO_URI=${!livevar:-${EGIT_REPO_URI}}
[[ ${!livevar} ]] \
&& ewarn "Using ${livevar}, no support will be provided"
livevar=${esc_pn}_LIVE_BRANCH
EGIT_BRANCH=${!livevar:-${EGIT_BRANCH}}
[[ ${!livevar} ]] \
&& ewarn "Using ${livevar}, no support will be provided"
livevar=${esc_pn}_LIVE_COMMIT
EGIT_COMMIT=${!livevar:-${EGIT_COMMIT}}
[[ ${!livevar} ]] \
&& ewarn "Using ${livevar}, no support will be provided"
# Migration helpers. Remove them when git-2 is removed.
if [[ ${EGIT_SOURCEDIR} ]]; then
eerror "EGIT_SOURCEDIR has been replaced by EGIT_CHECKOUT_DIR.
While updating"
eerror "your ebuild, please check whether the variable is
necessary at all"
eerror "since the default has been changed from \${S} to
\${WORKDIR}/\${P}."
eerror "Therefore, proper setting of S may be sufficient."
die "EGIT_SOURCEDIR has been replaced by EGIT_CHECKOUT_DIR."
fi
if [[ ${EGIT_MASTER} ]]; then
eerror "EGIT_MASTER has been removed. Instead, the upstream
default (HEAD)"
eerror "is used by the eclass. Please remove the assignment or
use EGIT_BRANCH"
eerror "as necessary."
die "EGIT_MASTER has been removed."
fi
if [[ ${EGIT_HAS_SUBMODULES} ]]; then
eerror "EGIT_HAS_SUBMODULES has been removed. The eclass no
longer needs"
eerror "to switch the clone type in order to support submodules
and therefore"
eerror "submodules are detected and fetched automatically."
die "EGIT_HAS_SUBMODULES is no longer necessary."
fi
if [[ ${EGIT_PROJECT} ]]; then
eerror "EGIT_PROJECT has been removed. Instead, the eclass
determines"
eerror "the local clone path using path in canonical
EGIT_REPO_URI."
eerror "If the current algorithm causes issues for you, please
report a bug."
die "EGIT_PROJECT is no longer necessary."
fi
if [[ ${EGIT_BOOTSTRAP} ]]; then
eerror "EGIT_BOOTSTRAP has been removed. Please create proper
src_prepare()"
eerror "instead."
die "EGIT_BOOTSTRAP has been removed."
fi
if [[ ${EGIT_NOUNPACK} ]]; then
eerror "EGIT_NOUNPACK has been removed. The eclass no longer
calls default"
eerror "unpack function. If necessary, please declare proper
src_unpack()."
die "EGIT_NOUNPACK has been removed."
fi
}
# @FUNCTION: _git-r3_set_gitdir
# @USAGE: <repo-uri>
# @INTERNAL
# @DESCRIPTION:
# Obtain the local repository path and set it as GIT_DIR. Creates
# a new repository if necessary.
#
# <repo-uri> may be used to compose the path. It should therefore be
# a canonical URI to the repository.
_git-r3_set_gitdir() {
debug-print-function ${FUNCNAME} "$@"
local repo_name=${1#*://*/}
# strip the trailing slash
repo_name=${repo_name%/}
# strip common prefixes to make paths more likely to match
# e.g. git://X/Y.git vs https://X/git/Y.git
# (but just one of the prefixes)
case "${repo_name}" in
# gnome.org... who else?
browse/*) repo_name=${repo_name#browse/};;
# cgit can proxy requests to git
cgit/*) repo_name=${repo_name#cgit/};;
# pretty common
git/*) repo_name=${repo_name#git/};;
# gentoo.org
gitroot/*) repo_name=${repo_name#gitroot/};;
# google code, sourceforge
p/*) repo_name=${repo_name#p/};;
# kernel.org
pub/scm/*) repo_name=${repo_name#pub/scm/};;
esac
# ensure a .git suffix, same reason
repo_name=${repo_name%.git}.git
# now replace all the slashes
repo_name=${repo_name//\//_}
local distdir=${PORTAGE_ACTUAL_DISTDIR:-${DISTDIR}}
: ${EGIT3_STORE_DIR:=${distdir}/git3-src}
GIT_DIR=${EGIT3_STORE_DIR}/${repo_name}
if [[ ! -d ${EGIT3_STORE_DIR} ]]; then
(
addwrite /
mkdir -m0755 -p "${EGIT3_STORE_DIR}" || die
) || die "Unable to create ${EGIT3_STORE_DIR}"
fi
addwrite "${EGIT3_STORE_DIR}"
if [[ -e ${GIT_DIR}/shallow ]]; then
einfo "${GIT_DIR} was a shallow clone, recreating..."
rm -r "${GIT_DIR}" || die
fi
if [[ ! -d ${GIT_DIR} ]]; then
mkdir "${GIT_DIR}" || die
git init --bare || die
fi
}
# @FUNCTION: _git-r3_set_submodules
# @USAGE: <file-contents>
# @INTERNAL
# @DESCRIPTION:
# Parse .gitmodules contents passed as <file-contents>
# as in "$(cat .gitmodules)"). Composes a 'submodules' array that
# contains in order (name, URL, path) for each submodule.
_git-r3_set_submodules() {
debug-print-function ${FUNCNAME} "$@"
local data=${1}
# ( name url path ... )
submodules=()
local l
while read l; do
# submodule.<path>.path=<path>
# submodule.<path>.url=<url>
[[ ${l} == submodule.*.url=* ]] || continue
l=${l#submodule.}
local subname=${l%%.url=*}
# skip modules that have 'update = none', bug #487262.
local upd=$(echo "${data}" | git config -f /dev/fd/0 \
submodule."${subname}".update)
[[ ${upd} == none ]] && continue
submodules+=(
"${subname}"
"$(echo "${data}" | git config -f /dev/fd/0 \
submodule."${subname}".url || die)"
"$(echo "${data}" | git config -f /dev/fd/0 \
submodule."${subname}".path || die)"
)
done < <(echo "${data}" | git config -f /dev/fd/0 -l || die)
}
# @FUNCTION: _git-r3_is_local_repo
# @USAGE: <repo-uri>
# @INTERNAL
# @DESCRIPTION:
# Determine whether the given URI specifies a local (on-disk)
# repository.
_git-r3_is_local_repo() {
debug-print-function ${FUNCNAME} "$@"
local uri=${1}
[[ ${uri} == file://* || ${uri} == /* ]]
}
# @FUNCTION: git-r3_fetch
# @USAGE: [<repo-uri> [<remote-ref> [<local-id>]]]
# @DESCRIPTION:
# Fetch new commits to the local clone of repository.
#
# <repo-uri> specifies the repository URIs to fetch from, as a space-
# -separated list. The first URI will be used as repository group
# identifier and therefore must be used consistently. When not
# specified, defaults to ${EGIT_REPO_URI}.
#
# <remote-ref> specifies the remote ref or commit id to fetch.
# It is preferred to use 'refs/heads/<branch-name>' for branches
# and 'refs/tags/<tag-name>' for tags. Other options are 'HEAD'
# for upstream default branch and hexadecimal commit SHA1. Defaults
# to the first of EGIT_COMMIT, EGIT_BRANCH or literal 'HEAD' that
# is set to a non-null value.
#
# <local-id> specifies the local branch identifier that will be used to
# locally store the fetch result. It should be unique to multiple
# fetches within the repository that can be performed at the same time
# (including parallel merges). It defaults to ${CATEGORY}/${PN}/${SLOT%/*}.
# This default should be fine unless you are fetching multiple trees
# from the same repository in the same ebuild.
#
# The fetch operation will affect the EGIT_STORE only. It will not touch
# the working copy, nor export any environment variables.
# If the repository contains submodules, they will be fetched
# recursively.
git-r3_fetch() {
debug-print-function ${FUNCNAME} "$@"
[[ ${EVCS_OFFLINE} ]] && return
local repos
if [[ ${1} ]]; then
repos=( ${1} )
elif [[ $(declare -p EGIT_REPO_URI) == "declare -a"* ]]; then
repos=( "${EGIT_REPO_URI[@]}" )
else
repos=( ${EGIT_REPO_URI} )
fi
local branch=${EGIT_BRANCH:+refs/heads/${EGIT_BRANCH}}
local remote_ref=${2:-${EGIT_COMMIT:-${branch:-HEAD}}}
local local_id=${3:-${CATEGORY}/${PN}/${SLOT%/*}}
local local_ref=refs/git-r3/${local_id}/__main__
[[ ${repos[@]} ]] || die "No URI provided and EGIT_REPO_URI unset"
local -x GIT_DIR
_git-r3_set_gitdir "${repos[0]}"
# try to fetch from the remote
local r success
for r in "${repos[@]}"; do
einfo "Fetching ${r} ..."
local fetch_command=(
git fetch --prune "${r}"
# mirror the remote branches as local branches
"refs/heads/*:refs/heads/*"
# pull tags explicitly in order to prune them properly
"refs/tags/*:refs/tags/*"
)
set -- "${fetch_command[@]}"
echo "${@}" >&2
if "${@}"; then
# now let's see what the user wants from us
local full_remote_ref=$(
git rev-parse --verify --symbolic-full-name
"${remote_ref}"
)
if [[ ${full_remote_ref} ]]; then
# when we are given a ref, create a symbolic ref
# so that we preserve the actual argument
set -- git symbolic-ref "${local_ref}"
"${full_remote_ref}"
else
# otherwise, we were likely given a commit id
set -- git update-ref --no-deref "${local_ref}"
"${remote_ref}"
fi
echo "${@}" >&2
if ! "${@}"; then
die "Referencing ${remote_ref} failed (wrong
ref?)."
fi
success=1
break
fi
done
[[ ${success} ]] || die "Unable to fetch from any of EGIT_REPO_URI"
# recursively fetch submodules
if git cat-file -e "${local_ref}":.gitmodules &>/dev/null; then
local submodules
_git-r3_set_submodules \
"$(git cat-file -p "${local_ref}":.gitmodules || die)"
while [[ ${submodules[@]} ]]; do
local subname=${submodules[0]}
local url=${submodules[1]}
local path=${submodules[2]}
local commit=$(git rev-parse "${local_ref}:${path}")
if [[ ! ${commit} ]]; then
die "Unable to get commit id for submodule
${subname}"
fi
if [[ ${url} == ./* || ${url} == ../* ]]; then
local subrepos=( "${repos[@]/%//${url}}" )
else
local subrepos=( "${url}" )
fi
git-r3_fetch "${subrepos[*]}" "${commit}"
"${local_id}/${subname}"
submodules=( "${submodules[@]:3}" ) # shift
done
fi
}
# @FUNCTION: git-r3_checkout
# @USAGE: [<repo-uri> [<checkout-path> [<local-id>]]]
# @DESCRIPTION:
# Check the previously fetched tree to the working copy.
#
# <repo-uri> specifies the repository URIs, as a space-separated list.
# The first URI will be used as repository group identifier
# and therefore must be used consistently with git-r3_fetch.
# The remaining URIs are not used and therefore may be omitted.
# When not specified, defaults to ${EGIT_REPO_URI}.
#
# <checkout-path> specifies the path to place the checkout. It defaults
# to ${EGIT_CHECKOUT_DIR} if set, otherwise to ${WORKDIR}/${P}.
#
# <local-id> needs to specify the local identifier that was used
# for respective git-r3_fetch.
#
# The checkout operation will write to the working copy, and export
# the repository state into the environment. If the repository contains
# submodules, they will be checked out recursively.
git-r3_checkout() {
debug-print-function ${FUNCNAME} "$@"
local repos
if [[ ${1} ]]; then
repos=( ${1} )
elif [[ $(declare -p EGIT_REPO_URI) == "declare -a"* ]]; then
repos=( "${EGIT_REPO_URI[@]}" )
else
repos=( ${EGIT_REPO_URI} )
fi
local out_dir=${2:-${EGIT_CHECKOUT_DIR:-${WORKDIR}/${P}}}
local local_id=${3:-${CATEGORY}/${PN}/${SLOT%/*}}
local -x GIT_DIR
_git-r3_set_gitdir "${repos[0]}"
einfo "Checking out ${repos[0]} to ${out_dir} ..."
if ! git cat-file -e refs/git-r3/"${local_id}"/__main__; then
if [[ ${EVCS_OFFLINE} ]]; then
die "No local clone of ${repos[0]}. Unable to work with
EVCS_OFFLINE."
else
die "Logic error: no local clone of ${repos[0]}.
git-r3_fetch not used?"
fi
fi
local remote_ref=$(
git symbolic-ref --quiet refs/git-r3/"${local_id}"/__main__
)
local new_commit_id=$(
git rev-parse --verify refs/git-r3/"${local_id}"/__main__
)
set -- git clone --shared --no-checkout "${GIT_DIR}" "${out_dir}"/
echo "${@}" >&2
"${@}" || die "git clone (for checkout) failed"
git-r3_sub_checkout() {
local -x GIT_DIR=${out_dir}/.git
local -x GIT_WORK_TREE=${out_dir}
set -- git checkout --quiet
if [[ ${remote_ref} ]]; then
set -- "${@}" "${remote_ref#refs/heads/}"
else
set -- "${@}" "${new_commit_id}"
fi
echo "${@}" >&2
"${@}" || die "git checkout ${remote_ref:-${new_commit_id}}
failed"
}
git-r3_sub_checkout
if ! git cat-file -e refs/git-r3/"${local_id}"/__old__; then
echo "GIT NEW branch -->"
echo " repository: ${repos[0]}"
echo " at the commit: ${new_commit_id}"
else
# diff against previous revision
local old_commit_id=$(
git rev-parse --verify
refs/git-r3/"${local_id}"/__old__ 2>/dev/null
)
echo "GIT update -->"
echo " repository: ${repos[0]}"
# write out message based on the revisions
if [[ "${old_commit_id}" != "${new_commit_id}" ]]; then
echo " updating from commit: ${old_commit_id}"
echo " to commit: ${new_commit_id}"
git --no-pager diff --stat \
${old_commit_id}..${new_commit_id}
else
echo " at the commit: ${new_commit_id}"
fi
fi
git update-ref --no-deref refs/git-r3/"${local_id}"/{__old__,__main__}
|| die
# recursively checkout submodules
if [[ -f ${out_dir}/.gitmodules ]]; then
local submodules
_git-r3_set_submodules \
"$(<"${out_dir}"/.gitmodules)"
while [[ ${submodules[@]} ]]; do
local subname=${submodules[0]}
local url=${submodules[1]}
local path=${submodules[2]}
if [[ ${url} == ./* || ${url} == ../* ]]; then
url=${repos[0]%%/}/${url}
fi
git-r3_checkout "${url}" "${out_dir}/${path}" \
"${local_id}/${subname}"
submodules=( "${submodules[@]:3}" ) # shift
done
fi
# keep this *after* submodules
export EGIT_DIR=${GIT_DIR}
export EGIT_VERSION=${new_commit_id}
}
# @FUNCTION: git-r3_peek_remote_ref
# @USAGE: [<repo-uri> [<remote-ref>]]
# @DESCRIPTION:
# Peek the reference in the remote repository and print the matching
# (newest) commit SHA1.
#
# <repo-uri> specifies the repository URIs to fetch from, as a space-
# -separated list. When not specified, defaults to ${EGIT_REPO_URI}.
#
# <remote-ref> specifies the remote ref to peek. It is preferred to use
# 'refs/heads/<branch-name>' for branches and 'refs/tags/<tag-name>'
# for tags. Alternatively, 'HEAD' may be used for upstream default
# branch. Defaults to the first of EGIT_COMMIT, EGIT_BRANCH or literal
# 'HEAD' that is set to a non-null value.
#
# The operation will be done purely on the remote, without using local
# storage. If commit SHA1 is provided as <remote-ref>, the function will
# fail due to limitations of git protocol.
#
# On success, the function returns 0 and writes hexadecimal commit SHA1
# to stdout. On failure, the function returns 1.
git-r3_peek_remote_ref() {
debug-print-function ${FUNCNAME} "$@"
local repos
if [[ ${1} ]]; then
repos=( ${1} )
elif [[ $(declare -p EGIT_REPO_URI) == "declare -a"* ]]; then
repos=( "${EGIT_REPO_URI[@]}" )
else
repos=( ${EGIT_REPO_URI} )
fi
local branch=${EGIT_BRANCH:+refs/heads/${EGIT_BRANCH}}
local remote_ref=${2:-${EGIT_COMMIT:-${branch:-HEAD}}}
[[ ${repos[@]} ]] || die "No URI provided and EGIT_REPO_URI unset"
local r success
for r in "${repos[@]}"; do
einfo "Peeking ${remote_ref} on ${r} ..." >&2
local is_branch lookup_ref
if [[ ${remote_ref} == refs/heads/* || ${remote_ref} == HEAD ]]
then
is_branch=1
lookup_ref=${remote_ref}
else
# ls-remote by commit is going to fail anyway,
# so we may as well pass refs/tags/ABCDEF...
lookup_ref=refs/tags/${remote_ref}
fi
# split on whitespace
local ref=(
$(git ls-remote "${r}" "${lookup_ref}")
)
if [[ ${ref[0]} ]]; then
echo "${ref[0]}"
return 0
fi
done
return 1
}
git-r3_src_fetch() {
debug-print-function ${FUNCNAME} "$@"
if [[ ! ${EGIT3_STORE_DIR} && ${EGIT_STORE_DIR} ]]; then
ewarn "You have set EGIT_STORE_DIR but not EGIT3_STORE_DIR.
Please consider"
ewarn "setting EGIT3_STORE_DIR for git-r3.eclass. It is
recommended to use"
ewarn "a different directory than EGIT_STORE_DIR to ease
removing old clones"
ewarn "when git-2 eclass becomes deprecated."
fi
_git-r3_env_setup
git-r3_fetch
}
git-r3_src_unpack() {
debug-print-function ${FUNCNAME} "$@"
_git-r3_env_setup
git-r3_src_fetch
git-r3_checkout
}
# https://bugs.gentoo.org/show_bug.cgi?id=482666
git-r3_pkg_outofdate() {
debug-print-function ${FUNCNAME} "$@"
local new_commit_id=$(git-r3_peek_remote_ref)
ewarn "old: ${EGIT_VERSION}"
ewarn "new: ${new_commit_id}"
[[ ${new_commit_id} && ${old_commit_id} ]] || return 2
[[ ${EGIT_VERSION} != ${new_commit_id} ]]
}
_GIT_R3=1
fi
signature.asc
Description: PGP signature
