--- Begin Message ---
Source: prometheus
Version: 2.33.5+ds1-2
Severity: serious
User: debian...@lists.debian.org
Usertags: flaky
Dear maintainer(s),
I looked at the results of the autopkgtest of your package (because it
blocked glibc). I noticed that it regularly fails. At least on 32 bits
architectures, I think the test (and maybe the general use of the
package?) is nearing memory limititations? Maybe the failures on 64 bit
architectures were due to non-available test dependencies (I didn't
check all logs).
Because the unstable-to-testing migration software now blocks on
regressions in testing, flaky tests, i.e. tests that flip between
passing and failing without changes to the list of installed packages,
are causing people unrelated to your package to spend time on these
tests.
Don't hesitate to reach out if you need help and some more information
from our infrastructure.
Paul
https://ci.debian.net/data/autopkgtest/testing/armhf/p/prometheus/28799146/log.gz
FAIL
FAIL github.com/prometheus/prometheus/promql 9.743s
--- FAIL: TestQueryHistogramFromBlocksWithCompaction (11.14s)
--- PASS:
TestQueryHistogramFromBlocksWithCompaction/serial_blocks_with_only_histograms
(1.23s)
--- PASS:
TestQueryHistogramFromBlocksWithCompaction/serial_blocks_with_either_histograms_or_floats_in_a_block_and_not_both
(2.57s)
--- PASS:
TestQueryHistogramFromBlocksWithCompaction/serial_blocks_with_mix_of_histograms_and_float64
(2.59s)
--- PASS:
TestQueryHistogramFromBlocksWithCompaction/overlapping_blocks_with_only_histograms
(1.68s)
--- PASS:
TestQueryHistogramFromBlocksWithCompaction/overlapping_blocks_with_only_histograms_and_only_float_in_a_series
(2.63s)
--- FAIL:
TestQueryHistogramFromBlocksWithCompaction/overlapping_blocks_with_mix_of_histograms_and_float64
(0.44s)
panic: mmap, size 134217728: cannot allocate memory [recovered]
panic: mmap, size 134217728: cannot allocate memory
goroutine 11328 [running]:
testing.tRunner.func1.2({0x88a108, 0x4c04030})
/usr/lib/go-1.19/src/testing/testing.go:1396 +0x27c
testing.tRunner.func1()
/usr/lib/go-1.19/src/testing/testing.go:1399 +0x3f4
panic({0x88a108, 0x4c04030})
/usr/lib/go-1.19/src/runtime/panic.go:884 +0x23c
github.com/prometheus/prometheus/tsdb.handleChunkWriteError({0xbe5844,
0x4c04030})
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1225
+0x88
github.com/prometheus/prometheus/tsdb/chunks.(*ChunkDiskMapper).WriteChunk(0x4bba000,
0x1, 0x0, 0xafc80, {0xbe9cd8, 0x4b9a6b0}, 0x93e6b8)
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go:418
+0x130
github.com/prometheus/prometheus/tsdb.(*memSeries).mmapCurrentHeadChunk(0x4baa120,
0x4bba000)
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1214
+0xa0
github.com/prometheus/prometheus/tsdb.(*memSeries).cutNewHeadChunk(0x4baa120,
0xdbba0, 0x1, 0x4bba000, 0x6ddd00)
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1149
+0x24
github.com/prometheus/prometheus/tsdb.(*memSeries).appendPreprocessor(0x4baa120,
0xdbba0, 0x1, 0x4bba000, 0x6ddd00)
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:1101
+0x19c
github.com/prometheus/prometheus/tsdb.(*memSeries).append(0x4baa120,
0xdbba0, 0x3fe1b0583e7c7c6a, 0x2, 0x4bba000, 0x6ddd00)
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:983
+0x4c
github.com/prometheus/prometheus/tsdb.(*headAppender).Commit(0x4be6140)
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/head_append.go:876
+0x1410
github.com/prometheus/prometheus/tsdb.CreateBlock({0x3eea780, 0xa, 0xa},
{0x3d12000, 0x72}, 0x0, {0xbe57e0, 0x105b3f8})
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/tsdbblockutil.go:63
+0x570
github.com/prometheus/prometheus/tsdb.createBlock({0xbeb950, 0x35e6960},
{0x3d12000, 0x72}, {0x3eea780, 0xa, 0xa})
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/block_test.go:474
+0x5c
github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction.func2(0x35e6960,
{0x6142000, 0x4, 0x4})
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6035
+0x95c
github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction.func8(0x35e6960)
/tmp/autopkgtest-lxc.mtbe33dm/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6133
+0x43c
testing.tRunner(0x35e6960, 0x300efb0)
/usr/lib/go-1.19/src/testing/testing.go:1446 +0x118
created by testing.(*T).Run
/usr/lib/go-1.19/src/testing/testing.go:1493 +0x3a0
FAIL github.com/prometheus/prometheus/tsdb 74.037s
https://ci.debian.net/data/autopkgtest/testing/i386/p/prometheus/28799709/log.gz
runtime: out of memory: cannot allocate 58720256-byte block (3870720000
in use)
fatal error: out of memory
goroutine 6356 [running]:
runtime.throw({0x89263f7, 0xd})
/usr/lib/go-1.19/src/runtime/panic.go:1047 +0x4d fp=0xacf2540
sp=0xacf252c pc=0x8087f1d
runtime.(*mcache).allocLarge(0xf7d13088, 0x3679508, 0x0)
/usr/lib/go-1.19/src/runtime/mcache.go:235 +0x208 fp=0xacf2568
sp=0xacf2540 pc=0x805e9b8
runtime.mallocgc(0x3679508, 0x88c3c00, 0x1)
/usr/lib/go-1.19/src/runtime/malloc.go:1029 +0x458 fp=0xacf25b0
sp=0xacf2568 pc=0x8055828
runtime.makeslice(0x88c3c00, 0x0, 0x6cf2a1)
/usr/lib/go-1.19/src/runtime/slice.go:103 +0x4f fp=0xacf25c4
sp=0xacf25b0 pc=0x80a00ff
runtime.makeslice64(0x88c3c00, 0x0, 0x6cf2a1)
/usr/lib/go-1.19/src/runtime/slice.go:117 +0x6f fp=0xacf25d8
sp=0xacf25c4 pc=0x80a01df
github.com/prometheus/prometheus/tsdb.genSeriesFromSampleGenerator(0xa,
0x5, 0xea60, 0x6ddd00, 0x2bf20, 0x8968524)
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/block_test.go:671
+0x1e0 fp=0xacf26fc sp=0xacf25d8 pc=0x871c7b0
github.com/prometheus/prometheus/tsdb.genHistogramSeries(0xa, 0x5,
0xea60, 0x6ddd00, 0x2bf20)
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/block_test.go:607
+0x63 fp=0xacf2730 sp=0xacf26fc pc=0x871c223
github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction.func6(0xb7f2f00)
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6117
+0x15f fp=0xacf279c sp=0xacf2730 pc=0x878d6cf
testing.tRunner(0xb7f2f00, 0xc59ad5b0)
/usr/lib/go-1.19/src/testing/testing.go:1446 +0x113 fp=0xacf27e4
sp=0xacf279c pc=0x816b5b3
testing.(*T).Run.func1()
/usr/lib/go-1.19/src/testing/testing.go:1493 +0x2e fp=0xacf27f0
sp=0xacf27e4 pc=0x816c3ae
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xacf27f4
sp=0xacf27f0 pc=0x80bd3c1
created by testing.(*T).Run
/usr/lib/go-1.19/src/testing/testing.go:1493 +0x374
goroutine 1 [chan receive]:
runtime.gopark(0x8969034, 0xac2b930, 0xe, 0x17, 0x2)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xec8fc18
sp=0xec8fc04 pc=0x808b228
runtime.chanrecv(0xac2b900, 0xec8fc97, 0x1)
/usr/lib/go-1.19/src/runtime/chan.go:583 +0x3f2 fp=0xec8fc60
sp=0xec8fc18 pc=0x804f3f2
runtime.chanrecv1(0xac2b900, 0xec8fc97)
/usr/lib/go-1.19/src/runtime/chan.go:442 +0x1c fp=0xec8fc74
sp=0xec8fc60 pc=0x804efcc
testing.(*T).Run(0xac84690, {0x89492bd, 0x2a}, 0x896842c)
/usr/lib/go-1.19/src/testing/testing.go:1494 +0x393 fp=0xec8fcc0
sp=0xec8fc74 pc=0x816c333
testing.runTests.func1(0xac84690)
/usr/lib/go-1.19/src/testing/testing.go:1846 +0x5e fp=0xec8fce4
sp=0xec8fcc0 pc=0x816e18e
testing.tRunner(0xac84690, 0xad9fd7c)
/usr/lib/go-1.19/src/testing/testing.go:1446 +0x113 fp=0xec8fd2c
sp=0xec8fce4 pc=0x816b5b3
testing.runTests(0xaed6000, {0x909a640, 0xc6, 0xc6},
{0xc0d9d2f513675009, 0x11765f6e595, 0x909d480})
/usr/lib/go-1.19/src/testing/testing.go:1844 +0x3a6 fp=0xec8fd90
sp=0xec8fd2c pc=0x816e076
testing.(*M).Run(0xacb3310)
/usr/lib/go-1.19/src/testing/testing.go:1726 +0x680 fp=0xec8fecc
sp=0xec8fd90 pc=0x816cc10
go.uber.org/goleak.VerifyTestMain({0x8c0e8b8, 0xacb3310}, {0xec8ff4c,
0x2, 0x2})
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/go.uber.org/goleak/testmain.go:53
+0x37 fp=0xec8ff20 sp=0xec8fecc pc=0x853ebe7
github.com/prometheus/prometheus/tsdb.TestMain(0xacb3310)
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:66
+0x205 fp=0xec8ff60 sp=0xec8ff20 pc=0x8736bf5
main.main()
_testmain.go:487 +0x189 fp=0xec8ffc4 sp=0xec8ff60 pc=0x881c629
runtime.main()
/usr/lib/go-1.19/src/runtime/proc.go:250 +0x24d fp=0xec8fff0
sp=0xec8ffc4 pc=0x808ae2d
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xec8fff4
sp=0xec8fff0 pc=0x80bd3c1
goroutine 2 [force gc (idle), 1 minutes]:
runtime.gopark(0x89691a8, 0x909c860, 0x11, 0x14, 0x1)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3efdc
sp=0xac3efc8 pc=0x808b228
runtime.goparkunlock(...)
/usr/lib/go-1.19/src/runtime/proc.go:369
runtime.forcegchelper()
/usr/lib/go-1.19/src/runtime/proc.go:302 +0xc3 fp=0xac3eff0
sp=0xac3efdc pc=0x808b0a3
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3eff4
sp=0xac3eff0 pc=0x80bd3c1
created by runtime.init.5
/usr/lib/go-1.19/src/runtime/proc.go:290 +0x23
goroutine 3 [GC sweep wait]:
runtime.gopark(0x89691a8, 0x909d290, 0xc, 0x14, 0x1)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3f7d4
sp=0xac3f7c0 pc=0x808b228
runtime.goparkunlock(...)
/usr/lib/go-1.19/src/runtime/proc.go:369
runtime.bgsweep(0xac2a0c0)
/usr/lib/go-1.19/src/runtime/mgcsweep.go:297 +0xf3 fp=0xac3f7e8
sp=0xac3f7d4 pc=0x8073743
runtime.gcenable.func1()
/usr/lib/go-1.19/src/runtime/mgc.go:178 +0x27 fp=0xac3f7f0 sp=0xac3f7e8
pc=0x8064827
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3f7f4
sp=0xac3f7f0 pc=0x80bd3c1
created by runtime.gcenable
/usr/lib/go-1.19/src/runtime/mgc.go:178 +0x7c
goroutine 4 [GC scavenge wait]:
runtime.gopark(0x89691a8, 0x909d640, 0xd, 0x14, 0x2)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3ffb8
sp=0xac3ffa4 pc=0x808b228
runtime.goparkunlock(...)
/usr/lib/go-1.19/src/runtime/proc.go:369
runtime.(*scavengerState).park(0x909d640)
/usr/lib/go-1.19/src/runtime/mgcscavenge.go:389 +0x5f fp=0xac3ffcc
sp=0xac3ffb8 pc=0x807108f
runtime.bgscavenge(0xac2a0c0)
/usr/lib/go-1.19/src/runtime/mgcscavenge.go:622 +0x77 fp=0xac3ffe8
sp=0xac3ffcc pc=0x80718a7
runtime.gcenable.func2()
/usr/lib/go-1.19/src/runtime/mgc.go:179 +0x27 fp=0xac3fff0 sp=0xac3ffe8
pc=0x80647e7
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3fff4
sp=0xac3fff0 pc=0x80bd3c1
created by runtime.gcenable
/usr/lib/go-1.19/src/runtime/mgc.go:179 +0xc1
goroutine 18 [finalizer wait, 1 minutes]:
runtime.gopark(0x89691a8, 0x90b1888, 0x10, 0x14, 0x1)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3e790
sp=0xac3e77c pc=0x808b228
runtime.goparkunlock(...)
/usr/lib/go-1.19/src/runtime/proc.go:369
runtime.runfinq()
/usr/lib/go-1.19/src/runtime/mfinal.go:180 +0xfb fp=0xac3e7f0
sp=0xac3e790 pc=0x8063a7b
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3e7f4
sp=0xac3e7f0 pc=0x80bd3c1
created by runtime.createfing
/usr/lib/go-1.19/src/runtime/mfinal.go:157 +0x54
goroutine 6305 [chan receive]:
runtime.gopark(0x8969034, 0xc3be9470, 0xe, 0x17, 0x2)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3bedc
sp=0xac3bec8 pc=0x808b228
runtime.chanrecv(0xc3be9440, 0xac3bf5b, 0x1)
/usr/lib/go-1.19/src/runtime/chan.go:583 +0x3f2 fp=0xac3bf24
sp=0xac3bedc pc=0x804f3f2
runtime.chanrecv1(0xc3be9440, 0xac3bf5b)
/usr/lib/go-1.19/src/runtime/chan.go:442 +0x1c fp=0xac3bf38
sp=0xac3bf24 pc=0x804efcc
testing.(*T).Run(0xb7f2e10, {0x8945a1a, 0x27}, 0xc59ad5b0)
/usr/lib/go-1.19/src/testing/testing.go:1494 +0x393 fp=0xac3bf84
sp=0xac3bf38 pc=0x816c333
github.com/prometheus/prometheus/tsdb.TestQueryHistogramFromBlocksWithCompaction(0xb7f2e10)
/tmp/autopkgtest-lxc.cdoskdiv/downtmp/autopkgtest_tmp/.build/src/github.com/prometheus/prometheus/tsdb/db_test.go:6114
+0x151 fp=0xac3bf9c sp=0xac3bf84 pc=0x878cde1
testing.tRunner(0xb7f2e10, 0x896842c)
/usr/lib/go-1.19/src/testing/testing.go:1446 +0x113 fp=0xac3bfe4
sp=0xac3bf9c pc=0x816b5b3
testing.(*T).Run.func1()
/usr/lib/go-1.19/src/testing/testing.go:1493 +0x2e fp=0xac3bff0
sp=0xac3bfe4 pc=0x816c3ae
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3bff4
sp=0xac3bff0 pc=0x80bd3c1
created by testing.(*T).Run
/usr/lib/go-1.19/src/testing/testing.go:1493 +0x374
goroutine 35 [GC worker (idle)]:
runtime.gopark(0x8969068, 0xace0690, 0x18, 0x14, 0x0)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3a794
sp=0xac3a780 pc=0x808b228
runtime.gcBgMarkWorker()
/usr/lib/go-1.19/src/runtime/mgc.go:1235 +0x103 fp=0xac3a7f0
sp=0xac3a794 pc=0x8067073
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3a7f4
sp=0xac3a7f0 pc=0x80bd3c1
created by runtime.gcBgMarkStartWorkers
/usr/lib/go-1.19/src/runtime/mgc.go:1159 +0x25
goroutine 36 [GC worker (idle)]:
runtime.gopark(0x8969068, 0xace06a8, 0x18, 0x14, 0x0)
/usr/lib/go-1.19/src/runtime/proc.go:363 +0x108 fp=0xac3af94
sp=0xac3af80 pc=0x808b228
runtime.gcBgMarkWorker()
/usr/lib/go-1.19/src/runtime/mgc.go:1235 +0x103 fp=0xac3aff0
sp=0xac3af94 pc=0x8067073
runtime.goexit()
/usr/lib/go-1.19/src/runtime/asm_386.s:1326 +0x1 fp=0xac3aff4
sp=0xac3aff0 pc=0x80bd3c1
created by runtime.gcBgMarkStartWorkers
/usr/lib/go-1.19/src/runtime/mgc.go:1159 +0x25
FAIL github.com/prometheus/prometheus/tsdb 97.396s
OpenPGP_signature
Description: OpenPGP digital signature
--- End Message ---
