Hi, Quoting Johannes Schauer (2020-11-10 09:49:59) > On Fri, 30 Oct 2020 09:28:22 +0100 Johannes Schauer <jo...@debian.org> wrote: > > The offending commit is: > > > commit 2a53909e732b19ccbaf6d1534e932dff74fa757e (refs/bisect/bad) > > > Author: Vasyl Gello <vasek.ge...@gmail.com> > > > Date: Fri Oct 2 03:48:45 2020 +0000 > > > > > > Use fixed-width members in fake_msg > > > > > > * Also usevthe same padding on all architectures > > > * Fixes cross-architecture SysV IPC connection failures > > > (i.e using system-wide fakeroot in chroot-ed jails) > > > > If I revert that commit on the current master branch, then everything works > > fine again, thus I'm tagging this bug with "patch". > > > > Please either fix this issue or revert the commit until you have a solution > > that doesn't break this use-case. It currently makes the mmdebstrap > > autopkgtest fail and thus will block it from migration to testing (thus > > raising severity). > > since there has been no activity on this RC bug for 10 days, I made a NMU with > a delay of seven days. Please chime in if you'd like me to cancel the upload. > I > attached the debdiff of my NMU to this message.
additionally, I prepared an autopkgtest (attached) which is able to test the functionality that broke. Unfortunately, neither salsaci nor debci support binfmt_misc, which is necessary for qemu-user mode. That's why the test is run inside a qemu virtual machine. Since neither salsaci nor debci support kvm, the test is way too slow to run on either. Consider this script an option for once binfmt_misk or kvm is available on our infrastructure. But even without it, you can run the script yourself on your own machine. Since it's using qemu, you don't need superuser privileges either. To still regularly test for this bug, I created a jenkins job running a script doing nearly the same as the attached script. The main difference is, that it only takes the fakeroot packages from the archive and does not support crosscompiling the foreign architecture binaries. As you can see, the test correctly reports "failed architectures: armel armhf mipsel s390x". https://jenkins.debian.net/job/fakeroot-foreign-worker/20/consoleFull Once this bug is fixed, the job should succeed. Thanks! cheers, josch
#!/bin/sh
# Copyright 2019 Johannes 'josch' Schauer <jo...@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# This script tests, whether fakeroot can be used to create foreign
# architecture chroots. This is to find bugs like #973405 early.
#
# Currently, there is no machine level isolation for autopkgtest jobs as run by
# salsaci and debci. Foreign architecture fakeroot relies on qemu binfmt
# support which is not provided by either. We thus first build a virtual
# machine and then run the tests inside of it.
#
# If the package is not yet released, foreign architecture versions of the
# libfakeroot binary will be cross-compiled on-the-fly.
#
# If the package is released and this test is run on unstable, then M-A version
# skews are ignored
#
# If the package is released and this test is run anywhere else, then
# uninstallable foreign architecture libraries will be an error.
set -exu
if [ "$(dpkg --print-architecture)" != "amd64" ]; then
# creating the bootable qemu machine is only supported for amd64
echo "W: Architecture isn't amd64, skipping test." >&2
exit 77
fi
# the archive with the highest priority where the base-files package comes from
# determines whether we are on stable, testing or unstable
DEFAULT_DIST=$(cat << END | python3 -
import apt_pkg, sys
apt_pkg.init()
c = apt_pkg.Cache(None)
d = apt_pkg.DepCache(c)
s = apt_pkg.SourceList()
s.read_main_list()
highest_prio = -1
highest_archive = None
for pkgfile, _ in d.get_candidate_ver(c["base-files"]).file_list:
print("processing: %s"%pkgfile, file=sys.stderr)
index = s.find_index(pkgfile)
if index is None:
print("index is none -- skipping", file=sys.stderr)
continue
if not index.is_trusted:
print("index is not trusted -- skipping", file=sys.stderr)
continue
archive = pkgfile.archive
if archive not in ["stable", "testing", "unstable"]:
print("index archive %s is %s -- skipping"%(index, archive), file=sys.stderr)
continue
prio = d.policy.get_priority(pkgfile)
if prio > highest_prio:
highest_prio = prio
highest_archive = archive
if highest_archive is None:
print("highest priority apt archive is neither stable, testing or unstable", file=sys.stderr)
for f in c.file_list:
print('========================', file=sys.stderr)
for a in ['architecture', 'archive', 'codename', 'component', 'filename', 'id', 'index_type', 'label', 'not_automatic', 'not_source', 'origin', 'site', 'size', 'version']:
print("%s: %s"%(a, getattr(f, a, None)), file=sys.stderr)
print("priority: ", d.policy.get_priority(f), file=sys.stderr)
exit(1)
print("highest archive priority: %s"%highest_archive, file=sys.stderr)
print(highest_archive)
END
)
IGNORE_MA_SAME_SKEWS=no
if [ "$DEFAULT_DIST" = unstable ]; then
IGNORE_MA_SAME_SKEWS=yes
fi
if [ -z ${AUTOPKGTEST_TMP+x} ]; then
# if AUTOPKGTEST_TMP is not set, then this script is probably not
# executed under autopkgtest
TMPDIR=$(mktemp --directory --tmpdir fakeroot_autopkgtest_foreign.XXXXXXXXXX)
else
# since AUTOPKGTEST_TMP is set, we assume that this script is executed
# under autopkgtest --> switch to the temporary directory
TMPDIR="$AUTOPKGTEST_TMP"
mkdir -p "$TMPDIR"
fi
# current directory is the unpacked sources
SRCDIR="$(pwd)"
cd "$TMPDIR"
# if this autopkgtest is run on a not-yet-released package version (for
# example on salsaci), then we need to build the foreign architecture
# binaries ourselves
if [ "$(dpkg-parsechangelog --show-field Distribution --file "$SRCDIR/debian/changelog")" = "UNRELEASED" ]; then
tar --directory "$SRCDIR" -czf source.tar.gz .
fi
# We would like to run mmdebstrap without superuser privileges but we cannot
# use fakechroot mode because of #944929, proot mode produces wrong permissions
# and unshare mode only works if kernel.unprivileged_userns_clone is set to 1
if [ "$(cat /proc/sys/kernel/unprivileged_userns_clone)" = "1" ]; then
MODE="unshare"
# the temporary directory might not have read permissions for the
# unshared user
chmod a+rx "$TMPDIR"
else
MODE="root"
fi
if [ ! -e "$TMPDIR/id_rsa" ]; then
ssh-keygen -q -t rsa -f "$TMPDIR/id_rsa" -N ""
fi
cat << SCRIPT > "$TMPDIR/customize.sh"
#!/bin/sh
set -exu
rootfs="\$1"
# setup various files in /etc
echo host > "\$rootfs/etc/hostname"
echo "127.0.0.1 localhost host" > "\$rootfs/etc/hosts"
echo "/dev/vda1 / auto errors=remount-ro 0 1" > "\$rootfs/etc/fstab"
cat /etc/resolv.conf > "\$rootfs/etc/resolv.conf"
# give a trivial password to the root user for easy debugging in case something fails
echo root:abcdef | chroot "\$rootfs" /usr/sbin/chpasswd
# extlinux config to boot from /dev/vda1 with predictable network interface
# naming and a serial console for logging
cat << END > "\$rootfs/extlinux.conf"
default linux
timeout 0
label linux
kernel /vmlinuz
append initrd=/initrd.img root=/dev/vda1 net.ifnames=0 console=ttyS0
END
# network interface config
# we can use eth0 because we boot with net.ifnames=0 for predictable interface
# names
cat << END > "\$rootfs/etc/network/interfaces"
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet dhcp
END
# copy in the public key
mkdir "\$rootfs/root/.ssh"
cp "$TMPDIR/id_rsa.pub" "\$rootfs/root/.ssh/authorized_keys"
chroot "\$rootfs" chown 0:0 /root/.ssh/authorized_keys
# copy in the sources if required
if [ -e source.tar.gz ]; then
chroot "\$rootfs" mkdir /src
cat source.tar.gz | chroot "\$rootfs" tar --directory /src -xz
chroot "\$rootfs" env --chdir=/src autoreconf -fi
fi
chroot "\$rootfs" sh -c "apt-get install --yes --no-install-recommends /debpkg/*.deb"
chroot "\$rootfs" sh -c "rm /debpkg/*.deb && rmdir /debpkg"
SCRIPT
chmod +x "$TMPDIR/customize.sh"
include=python3-apt,mmdebstrap,time,arch-test,debootstrap,qemu-user,qemu-user-static,binfmt-support,curl,openssh-server,systemd-sysv,linux-image-amd64,ifupdown,netbase,isc-dhcp-client,udev,policykit-1,fakechroot,dpkg-dev
# we also need to install build dependencies
if [ "$(dpkg-parsechangelog --show-field Distribution --file "$SRCDIR/debian/changelog")" = "UNRELEASED" ]; then
include="$include,autoconf,automake,libtool,build-essential,libacl1-dev,libcap-dev"
fi
# apt sources.list names must not contain spaces, so we can use a string to
# store the list
srclist=""
for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
if [ ! -e "$f" ]; then
continue
fi
bn=$(basename "$f")
# remove all file:// entries as those cannot be accessed from within
# the chroot and would break an apt-get update
grep -v "file://" "$f" > "$TMPDIR/$bn" || continue
srclist="$srclist $bn"
done
test -n "$srclist"
mkdir debpkg
env --chdir=debpkg apt-get download libfakeroot fakeroot
mmdebstrap --mode=$MODE \
--variant=apt \
--include=$include \
--customize-hook="copy-in debpkg /" \
--customize-hook="$TMPDIR/customize.sh" \
dummy "$TMPDIR/debian.tar" $srclist
for f in $srclist; do
rm "$TMPDIR/$f"
done
rm debpkg/*.deb
rmdir debpkg
# use guestfish to prepare the host system
#
# - create a single 2G partition and unpack the rootfs tarball into it
# - put a syslinux MBR into the first 440 bytes of the drive
# - install extlinux and make partition bootable
#
# useful stuff to debug any errors:
# LIBGUESTFS_BACKEND_SETTINGS=force_tcg
# libguestfs-test-tool || true
# export LIBGUESTFS_DEBUG=1 LIBGUESTFS_TRACE=1
guestfish -N "$TMPDIR/debian.img"=disk:4G -- \
part-disk /dev/sda mbr : \
mkfs ext2 /dev/sda1 : \
mount /dev/sda1 / : \
tar-in "$TMPDIR/debian.tar" / : \
upload /usr/lib/SYSLINUX/mbr.bin /mbr.bin : \
copy-file-to-device /mbr.bin /dev/sda size:440 : \
rm /mbr.bin : \
extlinux / : \
sync : \
umount / : \
part-set-bootable /dev/sda 1 true : \
shutdown
# start the host system
# prefer using kvm but fall back to tcg if not available
# avoid entropy starvation by feeding the crypt system with random bits from /dev/urandom
# the default memory size of 128 MiB is not enough for Debian, so we go with 1G
# use a virtio network card instead of emulating a real network device
# we don't need any graphics
# this also multiplexes the console and the monitor to stdio
# creates a multiplexed stdio backend connected to the serial port and the qemu
# monitor
# redirect tcp connections on port 10022 localhost to the host system port 22
# redirect all output to a file
# run in the background
qemu-system-x86_64 \
-M accel=kvm:tcg \
-no-user-config \
-object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
-m 1G \
-net nic,model=virtio \
-nographic \
-serial mon:stdio \
-net user,hostfwd=tcp:127.0.0.1:10022-:22 \
-drive file="$TMPDIR/debian.img",format=raw,if=virtio \
> "$TMPDIR/qemu.log" </dev/null 2>&1 &
# store the pid
QEMUPID=$!
onerror() {
cat --show-nonprinting $TMPDIR/qemu.log
# attempt poweroff
$ssh -o ConnectTimeout=$TIMEOUT root@localhost systemctl poweroff
# give a few seconds for poweroff
sleep 10
kill $QEMUPID || true
# turn off verbose output
set +x
echo "script failed -- temporary files are stored in $TMPDIR:"
echo
ls -lha "$TMPDIR"
echo
echo "to test yourself, run qemu with:"
echo
echo " $ qemu-system-x86_64 -no-user-config -m 1G -net nic,model=virtio -nographic -serial mon:stdio -net user,hostfwd=tcp:127.0.0.1:10022-:22 -drive file=\"$TMPDIR/debian.img\",format=raw,if=virtio"
echo
echo "and log in using:"
echo
echo " user: root"
echo " pass: abcdef"
echo
echo "or connect to it via ssh:"
echo
echo " $ $ssh root@localhost"
echo
echo "when you are done, cleanup temporary files with:"
echo
echo " $ rm -r \"$TMPDIR\""
}
# show the log and kill qemu in case the script exits first
trap onerror EXIT
# the default ssh command does not store known hosts and even ignores host keys
# it identifies itself with the rsa key generated above
# pseudo terminal allocation is disabled or otherwise, programs executed via
# ssh might wait for input on stdin of the ssh process
ssh="ssh -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no -i "$TMPDIR/id_rsa" -T -p 10022"
# we use sleepenh to make sure that we wait the right number of seconds
# independent on how long the command took beforehand
TIMESTAMP=$(sleepenh 0 || [ $? -eq 1 ])
# the timeout in seconds
TIMEOUT=5
# the maximum number of tries
NUM_TRIES=20
i=0
while true; do
rv=0
$ssh -o ConnectTimeout=$TIMEOUT root@localhost echo success || rv=1
# with an exit code of zero, the ssh connection was successful
# and we break out of the loop
[ $rv -eq 0 ] && break
# if the command before took less than $TIMEOUT seconds, wait the remaining time
TIMESTAMP=$(sleepenh $TIMESTAMP $TIMEOUT || [ $? -eq 1 ]);
# increment the counter and break out of the loop if we tried
# too often
i=$((i+1))
if [ $i -ge $NUM_TRIES ]; then
break
fi
done
# if all tries were exhausted, the process failed
if [ $i -eq $NUM_TRIES ]; then
echo "timeout reached: unable to connect to qemu via ssh"
exit 1
fi
$ssh root@localhost sh << 'END'
set -exu
curl --output Release http://deb.debian.org/debian/dists/unstable/Release
curl --output Release.gpg http://deb.debian.org/debian/dists/unstable/Release.gpg
gpgv --quiet --keyring=/usr/share/keyrings/debian-archive-keyring.gpg Release.gpg Release
failed_arches=""
for arch in $(sed -ne 's/^Architectures: \(.*\)/\1/p' Release); do
if [ "$arch" = amd64 ]; then
# only test this for foreign architectures
continue
fi
dpkg --add-architecture "$arch"
apt-get update
d_h_m=$(dpkg-architecture --host-arch="$arch" -qDEB_HOST_MULTIARCH)
ret=0
if [ -d /src ]; then
apt-get --yes install --no-install-recommends \
"crossbuild-essential-$arch" \
libcap-dev:amd64 libcap-dev:"$arch" \
libacl1-dev:amd64 libacl1-dev:"$arch" \
libfakechroot:amd64 libfakechroot:"$arch" \
|| ret=$?
else
apt-get --yes install --no-install-recommends \
libfakeroot:amd64 libfakeroot:"$arch" \
libfakechroot:amd64 libfakechroot:"$arch" \
|| ret=$?
fi
if [ "$ret" -ne 0 ]; then
echo "failed installing fakeroot/fakechroot on $arch" >&2
if [ "$IGNORE_MA_SAME_SKEWS" = yes ]; then
echo "ignoring M-A version skew" >&2
if [ -d /src ]; then
apt-get remove --purge --yes \
"crossbuild-essential-$arch" \
libcap-dev:"$arch" \
libacl1-dev:"$arch" \
libfakechroot:"$arch"
apt-get autoremove --yes --purge
fi
apt-get remove --purge --yes --allow-remove-essential $(dpkg-query -f '${binary:Package}\n' --show '*:'"$arch")
dpkg --remove-architecture "$arch"
continue
else
exit 1
fi
fi
if [ -d /src ]; then
# perform an out-of-tree cross-build of fakeroot
mkdir -p /tmp/build
env --chdir=/tmp/build /src/configure --prefix=/usr --mandir=/usr/share/man --libdir=/usr/lib/$d_h_m/libfakeroot --program-suffix=-sysv --host $d_h_m
env --chdir=/tmp/build make
mkdir -p /usr/lib/$d_h_m/libfakeroot/
cp /tmp/build/.libs/libfakeroot-0.so /usr/lib/$d_h_m/libfakeroot/libfakeroot-sysv.so
fi
ret=0
# we could also run debootstrap first with --foreign and then with --second-stage
# but on salsaci and debci there is no support for kvm acceleration so
# the second stage would be doubly emulated, taking way too much time
#fakechroot fakeroot -s fakeroot.environ /usr/sbin/debootstrap --arch="$arch" --foreign --variant=fakechroot sid debian-sid
#timeout 10m fakechroot fakeroot -i fakeroot.environ sh -c 'env QEMU_LD_PREFIX="$(pwd)/debian-sid" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/'"$d_h_m"'/fakechroot:/usr/lib/'"$d_h_m"'/libfakeroot" PATH=/usr/sbin:/usr/bin:/sbin:/bin /usr/sbin/chroot debian-sid /debootstrap/debootstrap --second-stage' || ret=$?
#timeout 20m mmdebstrap --arch="$arch" --variant=apt --mode=fakechroot sid debian-sid
# for the quickest possible test of foreign architecture chroot, we
# set up the most minimal possible chroot using busybox
#timeout 20m mmdebstrap \
# --arch="$arch" \
# --variant=custom \
# --mode=fakechroot \
# --include=base-files,base-passwd,busybox,debianutils,dpkg,libc-bin,mawk,tar \
# --setup-hook='echo root:x:0:0:root:/root:/bin/sh > "$1/etc/passwd"' \
# --setup-hook='printf "root:x:0:\nmail:x:8:\nutmp:x:43:\n" > "$1/etc/group"' \
# --extract-hook='chroot "$1" /bin/busybox --install -s' \
# sid debian-sid || ret=$?
timeout 20m time mmdebstrap \
--arch="$arch" \
--variant=custom \
--mode=fakechroot \
--include=base-files,base-passwd,coreutils,dash,diffutils,dpkg,grep,libc-bin,sed,tar \
sid debian-sid || ret=$?
if [ "$ret" -ne 0 ]; then
failed_arches="$failed_arches $arch"
fi
rm -rf debian-sid fakeroot.environ
if [ -d /src ]; then
apt-get remove --purge --yes "crossbuild-essential-$arch"
apt-get autoremove --yes --purge
rm -r /tmp/build
rm /usr/lib/$d_h_m/libfakeroot/libfakeroot-sysv.so
rmdir /usr/lib/$d_h_m/libfakeroot/
fi
# juliank: instead of --allow-remove-essential, a future apt version
# will add support for '--allow-remove=?architecture(foreign)'
# https://salsa.debian.org/apt-team/apt/-/merge_requests/135
apt-get remove --purge --yes --allow-remove-essential $(dpkg-query -f '${binary:Package}\n' --show '*:'"$arch")
dpkg --remove-architecture "$arch"
done
echo "failed architectures: $failed_arches"
test -z "$failed_arches"
END
# shut the system off
trap - EXIT
$ssh root@localhost systemctl poweroff || true
wait $QEMUPID
# cleanup
for f in debian.tar id_rsa id_rsa.pub \
qemu.log debian.img customize.sh; do
rm "$TMPDIR/$f"
done
if [ -e "$TMPDIR/source.tar.gz" ]; then
rm "$TMPDIR/source.tar.gz"
fi
if [ -z ${AUTOPKGTEST_TMP+x} ]; then
rmdir "$TMPDIR"
fi
signature.asc
Description: signature
