On Thu, Oct 13, 2022 at 09:26:21PM -0400, Daniel Kahn Gillmor wrote: > What's the best approach? Should someone who wanted to add an nspawn > isolation container try to copy ./virt/autopkgtest-virt-lxc and > ./tools/autopkgtest-build-lxc (and their associated manpages) and tweak > them to use nspawn instead of lxc?
I couldn't think of anything better, so attached to this mail is a start. This is based on https://github.com/Truelite/nspawn-runner/blob/main/nspawn-runner and on https://github.com/ARPA-SIMC/moncic-ci/blob/main/moncic/container.py#L463 This is what currently happens with it: autopkgtest-virt-nspawn: DBG: executing open autopkgtest-virt-nspawn: DBG: will start a container (with isolation-container capability) autopkgtest-virt-nspawn: DBG: using container name 8a087ee0-a8c1-4c87-a2f1-e43709e11184 autopkgtest-virt-nspawn: DBG: execute-timeout: sudo systemd-run --quiet --property=KillMode=mixed --property=Type=notify --property=RestartForceExitStatus=133 --property=SuccessExitStatus=133 --property=Slice=machine.slice --property=Delegate=yes --property=TasksMax=16384 --property=WatchdogSec=3min systemd-nspawn --quiet --directory=/var/lib/machines/sid-build --machine=8a087ee0-a8c1-4c87-a2f1-e43709e11184 --boot --notify-ready=yes --resolv-conf=replace-host --volatile=overlay --read-only --suppress-sync=yes systemd.hostname=8a087ee0-a8c1-4c87-a2f1-e43709e11184 autopkgtest-virt-nspawn: DBG: container started autopkgtest-virt-nspawn: DBG: execute-timeout: sudo systemd-run --quiet --machine=8a087ee0-a8c1-4c87-a2f1-e43709e11184 -- sh -c getent passwd | sort -t: -nk3 | awk -F: '{if ($3 >= 1000 && $3 <= 59999) { print $1; exit } }' autopkgtest-virt-nspawn: DBG: determine_normal_user: no uid in [1000,59999] available autopkgtest-virt-nspawn: DBG: auxverb = ['sudo', 'systemd-run', '--quiet', '--machine=8a087ee0-a8c1-4c87-a2f1-e43709e11184', '--', 'env', '-i', 'bash', '-c', 'set -a; [ -r /etc/environment ] && . /etc/environment 2>/dev/null || true; [ -r /etc/default/locale ] && . /etc/default/locale 2>/dev/null || true; [ -r /etc/profile ] && . /etc/profile 2>/dev/null || true; set +a;"$@"; RC=$?; [ $RC != 255 ] || RC=253; set -e;myout=$(readlink /proc/$$/fd/1);myerr=$(readlink /proc/$$/fd/2);myout="${myout/[/\\\\[}"; myout="${myout/]/\\\\]}";myerr="${myerr/[/\\\\[}"; myerr="${myerr/]/\\\\]}";PS=$(ls -l /proc/[0-9]*/fd/* 2>/dev/null | sed -nr \'\\#(\'"$myout"\'|\'"$myerr"\')# { s#^.*/proc/([0-9]+)/.*$#\\1#; p}\'|sort -u);KILL="";for pid in $PS; do [ $pid -ne $$ ] && [ $pid -ne $PPID ] || continue; KILL="$KILL $pid";done;[ -z "$KILL" ] || kill -9 $KILL >/dev/null 2>&1 || true;exit $RC', '--'], downtmp = None autopkgtest-virt-nspawn: DBG: execute-timeout: sudo systemd-run --quiet --machine=8a087ee0-a8c1-4c87-a2f1-e43709e11184 -- env -i bash -c set -a; [ -r /etc/environment ] && . /etc/environment 2>/dev/null || true; [ -r /etc/default/locale ] && . /etc/default/locale 2>/dev/null || true; [ -r /etc/profile ] && . /etc/profile 2>/dev/null || true; set +a;"$@"; RC=$?; [ $RC != 255 ] || RC=253; set -e;myout=$(readlink /proc/$$/fd/1);myerr=$(readlink /proc/$$/fd/2);myout="${myout/[/\\[}"; myout="${myout/]/\\]}";myerr="${myerr/[/\\[}"; myerr="${myerr/]/\\]}";PS=$(ls -l /proc/[0-9]*/fd/* 2>/dev/null | sed -nr '\#('"$myout"'|'"$myerr"')# { s#^.*/proc/([0-9]+)/.*$#\1#; p}'|sort -u);KILL="";for pid in $PS; do [ $pid -ne $$ ] && [ $pid -ne $PPID ] || continue; KILL="$KILL $pid";done;[ -z "$KILL" ] || kill -9 $KILL >/dev/null 2>&1 || true;exit $RC -- mktemp --directory --tmpdir autopkgtest.XXXXXX autopkgtest-virt-nspawn: DBG: execute-timeout: sudo systemd-run --quiet --machine=8a087ee0-a8c1-4c87-a2f1-e43709e11184 -- env -i bash -c set -a; [ -r /etc/environment ] && . /etc/environment 2>/dev/null || true; [ -r /etc/default/locale ] && . /etc/default/locale 2>/dev/null || true; [ -r /etc/profile ] && . /etc/profile 2>/dev/null || true; set +a;"$@"; RC=$?; [ $RC != 255 ] || RC=253; set -e;myout=$(readlink /proc/$$/fd/1);myerr=$(readlink /proc/$$/fd/2);myout="${myout/[/\\[}"; myout="${myout/]/\\]}";myerr="${myerr/[/\\[}"; myerr="${myerr/]/\\]}";PS=$(ls -l /proc/[0-9]*/fd/* 2>/dev/null | sed -nr '\#('"$myout"'|'"$myerr"')# { s#^.*/proc/([0-9]+)/.*$#\1#; p}'|sort -u);KILL="";for pid in $PS; do [ $pid -ne $$ ] && [ $pid -ne $PPID ] || continue; KILL="$KILL $pid";done;[ -z "$KILL" ] || kill -9 $KILL >/dev/null 2>&1 || true;exit $RC -- chmod 1777 autopkgtest: DBG: got reply from testbed: ok autopkgtest: DBG: TestbedFailure sent `open', got `ok ' (0 result parameters), expected 1 result parameters autopkgtest: DBG: testbed stop autopkgtest: DBG: testbed close, scratch=None autopkgtest: DBG: sending command to testbed: quit autopkgtest-virt-nspawn: DBG: executing quit autopkgtest-virt-nspawn: DBG: cleanup... autopkgtest [13:56:31]: ERROR: testbed failure: sent `open', got `ok ' (0 result parameters), expected 1 result parameters autopkgtest: DBG: testbed stop I know a thing or three about using nspawn containers, but I don't know much about autopkgtest, and I'd love to team up with someone with the opposite kind of knowledge to make this work. Enrico -- GPG key: 4096R/634F4BD1E7AD5568 2009-05-08 Enrico Zini <enr...@enricozini.org>
#!/usr/bin/python3
#
# autopkgtest-virt-nspawn is part of autopkgtest
# autopkgtest is a tool for testing Debian binary packages
#
# autopkgtest is Copyright (C) 2006-2015 Canonical Ltd.
#
# Author: Enrico Zini <enr...@enricozini.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# See the file CREDITS for a full list of credits information (often
# installed as /usr/share/doc/autopkgtest/CREDITS).
import sys
import os
import subprocess
import time
import argparse
import uuid
import signal
import errno
import shlex
from typing import Dict
sys.path.insert(0, '/usr/share/autopkgtest/lib')
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
os.path.abspath(__file__))), 'lib'))
import VirtSubproc
import adtlog
capabilities = [
'revert',
'revert-full-system',
'root-on-testbed',
]
args = None
container_name = None
normal_user = None
container_properties: Dict[str, str] = {}
def parse_args():
global args
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--debug', action='store_true',
help='Enable debugging output')
parser.add_argument('-r', '--gain-root', metavar='COMMAND',
help='can become root by prefixing commands with
COMMAND')
parser.add_argument('image', help='nspawn ostree name from under
/var/lib/machines/')
parser.add_argument('nspawnargs', nargs=argparse.REMAINDER,
help='Additional arguments to pass to systemd-nspawn')
args = parser.parse_args()
if args.debug:
adtlog.verbosity = 2
def get_available_container_name():
'''Return a container name that isn't already taken'''
return str(uuid.uuid4())
def determine_normal_user():
'''Check for a normal user to run tests as.'''
global capabilities, normal_user
# get the first UID in the Debian Policy ยง9.2.2 "dynamically allocated
# user account" range
cmd = []
if args.gain_root:
cmd += shlex.split(args.gain_root)
cmd += ['systemd-run', '--quiet', f'--machine={container_name}', '--',
'sh', '-c',
'getent passwd | sort -t: -nk3 | '
"awk -F: '{if ($3 >= 1000 && $3 <= 59999) { print $1; exit } }'"]
out = VirtSubproc.execute_timeout(None, 10, cmd,
stdout=subprocess.PIPE)[1].strip()
if out:
normal_user = out
capabilities.append('suggested-normal-user=' + normal_user)
adtlog.debug('determine_normal_user: got user "%s"' % normal_user)
else:
adtlog.debug('determine_normal_user: no uid in [1000,59999] available')
def stop_container():
"""
Stop the running container
"""
cmd = []
if args.gain_root:
cmd += shlex.split(args.gain_root)
cmd += ["machinectl", "terminate", container_name]
VirtSubproc.execute_timeout(None, 300, cmd)
# We cannot run this code because we are likely not root ourselves.
# Consider execing ourselves with the gain_root command instead of
# prepending it to all systemd-run commands
#
# # See https://github.com/systemd/systemd/issues/6458
# leader_pid = int(container_properties["Leader"])
# os.kill(leader_pid, signal.SIGRTMIN + 4)
# while True:
# try:
# os.kill(leader_pid, 0)
# except OSError as e:
# if e.errno == errno.ESRCH:
# break
# raise
# time.sleep(0.1)
def hook_open():
global args, container_name, capabilities
unit_config = [
'KillMode=mixed',
'Type=notify',
'RestartForceExitStatus=133',
'SuccessExitStatus=133',
'Slice=machine.slice',
'Delegate=yes',
'TasksMax=16384',
'WatchdogSec=3min',
]
systemd_run_cmd = []
if args.gain_root:
systemd_run_cmd += shlex.split(args.gain_root)
systemd_run_cmd += ["systemd-run", "--quiet"]
for c in unit_config:
systemd_run_cmd.append(f"--property={c}")
extra_nspawnargs = args.nspawnargs
adtlog.debug('will start a container (with isolation-container capability)')
capabilities.append('isolation-container')
container_name = get_available_container_name()
adtlog.debug('using container name %s' % container_name)
ostree = os.path.join("/var/lib/machines", args.image)
systemd_run_cmd += [
"systemd-nspawn",
"--quiet",
f"--directory={ostree}",
f"--machine={container_name}",
"--boot",
"--notify-ready=yes",
"--resolv-conf=replace-host",
]
if False: # TODO: check if /var/lib/machines is on BTRFS
systemd_run_cmd.append("--ephemeral")
else:
systemd_run_cmd.append("--volatile=overlay")
systemd_run_cmd.append("--read-only")
# If systemd_version >= 250:
systemd_run_cmd.append("--suppress-sync=yes")
systemd_run_cmd.append(f"systemd.hostname={container_name}")
systemd_run_cmd += extra_nspawnargs
# This will only return after the container is fully started
VirtSubproc.check_exec(
systemd_run_cmd,
outp=True,
timeout=600
)
# Read machine properties
res = subprocess.run(
["machinectl", "show", container_name],
capture_output=True, text=True, check=True)
container_properties = {}
for line in res.stdout.splitlines():
key, value = line.split('=', 1)
container_properties[key] = value
try:
adtlog.debug('container started')
determine_normal_user()
# provide a minimal and clean environment in the container
# We also want to avoid exiting with 255 as that's auxverb's exit code
# if the auxverb itself failed; so we translate that to 253.
# Tests or builds sometimes leak background processes which might still
# be connected to lxc exec's stdout/err; we need to kill these after the
# main program (build or test script) finishes, otherwise we get
# eternal hangs.
VirtSubproc.auxverb = []
if args.gain_root:
VirtSubproc.auxverb += shlex.split(args.gain_root)
VirtSubproc.auxverb += [
'systemd-run', "--quiet", f'--machine={container_name}', '--',
'env', '-i', 'bash', '-c',
'set -a; '
'[ -r /etc/environment ] && . /etc/environment 2>/dev/null || true;
'
'[ -r /etc/default/locale ] && . /etc/default/locale 2>/dev/null ||
true; '
'[ -r /etc/profile ] && . /etc/profile 2>/dev/null || true; '
'set +a;'
'"$@"; RC=$?; [ $RC != 255 ] || RC=253; '
'set -e;'
'myout=$(readlink /proc/$$/fd/1);'
'myerr=$(readlink /proc/$$/fd/2);'
'myout="${myout/[/\\\\[}"; myout="${myout/]/\\\\]}";'
'myerr="${myerr/[/\\\\[}"; myerr="${myerr/]/\\\\]}";'
'PS=$(ls -l /proc/[0-9]*/fd/* 2>/dev/null | sed -nr
\'\\#(\'"$myout"\'|\'"$myerr"\')# { s#^.*/proc/([0-9]+)/.*$#\\1#; p}\'|sort
-u);'
'KILL="";'
'for pid in $PS; do'
' [ $pid -ne $$ ] && [ $pid -ne $PPID ] || continue;'
' KILL="$KILL $pid";'
'done;'
'[ -z "$KILL" ] || kill -9 $KILL >/dev/null 2>&1 || true;'
'exit $RC', '--'
]
except Exception:
# Clean up on failure
stop_container()
raise
def hook_downtmp(path):
return VirtSubproc.downtmp_mktemp(path)
def hook_revert():
hook_cleanup()
hook_open()
def hook_cleanup():
VirtSubproc.downtmp_remove()
stop_container()
def hook_capabilities():
return capabilities
parse_args()
VirtSubproc.main()
signature.asc
Description: PGP signature
