Source: python-xarray
Version: 2023.08.0-1
Severity: serious
Tags: patch
Justification: autopkgtest failures
Dear Maintainer,
The autopkgtest for your package is failing:
230s =================================== FAILURES
===================================
230s ____________ test_open_mfdataset_manyfiles[netcdf4-20-True-None-5]
_____________
230s
230s self = CachingFileManager(<class 'netCDF4._netCDF4.Dataset'>,
'/tmp/tmp4hr6i68_/temp-1120.nc', mode='r', kwargs={'clobber': True, 'diskless':
False, 'persist': False, 'format': 'NETCDF4'},
manager_id='abdbaa71-0b5f-4544-982d-afa923d39953')
230s needs_lock = True
230s
230s def _acquire_with_cache_info(self, needs_lock=True):
230s """Acquire a file, returning the file and whether it was
cached."""
230s with self._optional_lock(needs_lock):
230s try:
230s > file = self._cache[self._key]
230s
230s /usr/lib/python3/dist-packages/xarray/backends/file_manager.py:211:
230s _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _ _
230s
230s self = <xarray.backends.lru_cache.LRUCache object at 0x7fe48b508b00>
230s key = [<class 'netCDF4._netCDF4.Dataset'>,
('/tmp/tmp4hr6i68_/temp-1120.nc',), 'r', (('clobber', True), ('diskless',
False), ('format', 'NETCDF4'), ('persist', False)),
'abdbaa71-0b5f-4544-982d-afa923d39953']
230s
230s def __getitem__(self, key: K) -> V:
230s # record recent use of the key by moving it to the front of the
list
230s with self._lock:
230s > value = self._cache[key]
230s E KeyError: [<class 'netCDF4._netCDF4.Dataset'>,
('/tmp/tmp4hr6i68_/temp-1120.nc',), 'r', (('clobber', True), ('diskless',
False), ('format', 'NETCDF4'), ('persist', False)),
'abdbaa71-0b5f-4544-982d-afa923d39953']
230s
230s /usr/lib/python3/dist-packages/xarray/backends/lru_cache.py:56: KeyError
230s
230s During handling of the above exception, another exception occurred:
230s
230s readengine = 'netcdf4', nfiles = 20, parallel = True, chunks = None
230s file_cache_maxsize = 5
230s
230s @requires_dask
230s @pytest.mark.filterwarnings("ignore:use make_scale(name) instead")
230s def test_open_mfdataset_manyfiles(
230s readengine, nfiles, parallel, chunks, file_cache_maxsize
230s ):
230s # skip certain combinations
230s skip_if_not_engine(readengine)
230s
230s if ON_WINDOWS:
230s pytest.skip("Skipping on Windows")
230s
230s randdata = np.random.randn(nfiles)
230s original = Dataset({"foo": ("x", randdata)})
230s # test standard open_mfdataset approach with too many files
230s with create_tmp_files(nfiles) as tmpfiles:
230s writeengine = readengine if readengine != "pynio" else
"netcdf4"
230s # split into multiple sets of temp files
230s for ii in original.x.values:
230s subds = original.isel(x=slice(ii, ii + 1))
230s if writeengine != "zarr":
230s subds.to_netcdf(tmpfiles[ii], engine=writeengine)
230s else: # if writeengine == "zarr":
230s subds.to_zarr(store=tmpfiles[ii])
230s
230s # check that calculation on opened datasets works properly
230s > with open_mfdataset(
230s tmpfiles,
230s combine="nested",
230s concat_dim="x",
230s engine=readengine,
230s parallel=parallel,
230s chunks=chunks if (not chunks and readengine != "zarr")
else "auto",
230s ) as actual:
230s
230s /usr/lib/python3/dist-packages/xarray/tests/test_backends.py:3407:
230s _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _ _
230s /usr/lib/python3/dist-packages/xarray/backends/api.py:1020: in
open_mfdataset
230s datasets, closers = dask.compute(datasets, closers)
230s /usr/lib/python3/dist-packages/dask/base.py:600: in compute
230s results = schedule(dsk, keys, **kwargs)
230s /usr/lib/python3/dist-packages/dask/threaded.py:89: in get
230s results = get_async(
230s /usr/lib/python3/dist-packages/dask/local.py:511: in get_async
230s raise_exception(exc, tb)
230s /usr/lib/python3/dist-packages/dask/local.py:319: in reraise
230s raise exc
230s /usr/lib/python3/dist-packages/dask/local.py:224: in execute_task
230s result = _execute_task(task, data)
230s /usr/lib/python3/dist-packages/dask/core.py:119: in _execute_task
230s return func(*(_execute_task(a, cache) for a in args))
230s /usr/lib/python3/dist-packages/dask/utils.py:71: in apply
230s return func(*args, **kwargs)
230s /usr/lib/python3/dist-packages/xarray/backends/api.py:570: in open_dataset
230s backend_ds = backend.open_dataset(
230s /usr/lib/python3/dist-packages/xarray/backends/netCDF4_.py:602: in
open_dataset
230s store = NetCDF4DataStore.open(
230s /usr/lib/python3/dist-packages/xarray/backends/netCDF4_.py:400: in open
230s return cls(manager, group=group, mode=mode, lock=lock,
autoclose=autoclose)
230s /usr/lib/python3/dist-packages/xarray/backends/netCDF4_.py:347: in
__init__
230s self.format = self.ds.data_model
230s /usr/lib/python3/dist-packages/xarray/backends/netCDF4_.py:409: in ds
230s return self._acquire()
230s /usr/lib/python3/dist-packages/xarray/backends/netCDF4_.py:403: in
_acquire
230s with self._manager.acquire_context(needs_lock) as root:
230s /usr/lib/python3.11/contextlib.py:137: in __enter__
230s return next(self.gen)
230s /usr/lib/python3/dist-packages/xarray/backends/file_manager.py:199: in
acquire_context
230s file, cached = self._acquire_with_cache_info(needs_lock)
230s /usr/lib/python3/dist-packages/xarray/backends/file_manager.py:217: in
_acquire_with_cache_info
230s file = self._opener(*self._args, **kwargs)
230s src/netCDF4/_netCDF4.pyx:2490: in netCDF4._netCDF4.Dataset.__init__
230s ???
230s src/netCDF4/_netCDF4.pyx:1879: in netCDF4._netCDF4._get_grps
230s ???
230s _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_ _ _
230s
230s > ???
230s E RuntimeError: NetCDF: Not a valid ID
230s
230s src/netCDF4/_netCDF4.pyx:2029: RuntimeError
https://ci.debian.net/data/autopkgtest/testing/amd64/p/python-xarray/37237465/log.gz
You would have caught this if you hadn't disabled the tests during the build.
The attached patch resolves the issue skipping the failing test.
Kind Regards,
Bas
diff -Nru python-xarray-2023.08.0/debian/changelog
python-xarray-2023.08.0/debian/changelog
--- python-xarray-2023.08.0/debian/changelog 2023-08-28 17:28:23.000000000
+0200
+++ python-xarray-2023.08.0/debian/changelog 2023-08-30 10:40:43.000000000
+0200
@@ -1,3 +1,12 @@
+python-xarray (2023.08.0-1.1) UNRELEASED; urgency=medium
+
+ * Non-maintainer upload.
+ * Don't override dh_auto_test.
+ * Use autopkgtest-pkg-pybuild testsuite.
+ * Skip test_open_mfdataset_manyfiles, causes FTBFS.
+
+ -- Bas Couwenberg <sebas...@debian.org> Wed, 30 Aug 2023 10:40:43 +0200
+
python-xarray (2023.08.0-1) unstable; urgency=medium
* New upstream release
diff -Nru python-xarray-2023.08.0/debian/control
python-xarray-2023.08.0/debian/control
--- python-xarray-2023.08.0/debian/control 2023-08-28 17:28:23.000000000
+0200
+++ python-xarray-2023.08.0/debian/control 2023-08-30 10:40:43.000000000
+0200
@@ -50,6 +50,7 @@
Vcs-Browser: https://salsa.debian.org/science-team/python-xarray
Vcs-Git: https://salsa.debian.org/science-team/python-xarray.git
Homepage: http://xarray.pydata.org/
+Testsuite: autopkgtest-pkg-pybuild
Package: python3-xarray
Architecture: all
diff -Nru python-xarray-2023.08.0/debian/rules
python-xarray-2023.08.0/debian/rules
--- python-xarray-2023.08.0/debian/rules 2023-08-28 17:28:23.000000000
+0200
+++ python-xarray-2023.08.0/debian/rules 2023-08-30 10:40:43.000000000
+0200
@@ -4,10 +4,10 @@
export DH_VERBOSE=1
export PYBUILD_NAME=xarray
-PY3VERS:= $(shell py3versions -s)
+export PYBUILD_TEST_ARGS=-vv -k 'not test_open_mfdataset_manyfiles'
-# Disable checking for this release:
-DEB_BUILD_OPTIONS += nocheck nodoc
+# Disable docs for this release:
+DEB_BUILD_OPTIONS += nodoc
export PYDEVD_DISABLE_FILE_VALIDATION=1
@@ -33,14 +33,6 @@
PYTHONPATH=$(CURDIR) $(MAKE) -C doc html
endif
-override_dh_auto_test:
-ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
- for p in $(PY3VERS); do \
- PY3VERNUM=`echo $$p | sed -e 's/python//' `; \
- pybuild --test --test-pytest -i $$p -p $$PY3VERNUM ; \
- done
-endif
-
ifeq (,$(filter nodoc,$(DEB_BUILD_OPTIONS)))
override_dh_sphinxdoc:
dh_sphinxdoc --exclude=MathJax.js
diff -Nru python-xarray-2023.08.0/debian/tests/control
python-xarray-2023.08.0/debian/tests/control
--- python-xarray-2023.08.0/debian/tests/control 2023-08-28
17:28:23.000000000 +0200
+++ python-xarray-2023.08.0/debian/tests/control 1970-01-01
01:00:00.000000000 +0100
@@ -1,18 +0,0 @@
-Test-Command: set -e
- ; for py in $(py3versions -r 2>/dev/null)
- ; do cd "$AUTOPKGTEST_TMP"
- ; echo "Testing with $py:"
- ; $py -m pytest -v --pyargs xarray
- ; done
-Depends: python3-all,
- python3-pytest (>= 2.7.1),
- python3-xarray,
- python3-bottleneck,
- python3-dask (>= 2.8.1+dfsg),
- python3-netcdf4,
- python3-scipy,
- python3-matplotlib,
- python3-cftime,
- python3-rasterio
-
-
