On Wed, Mar 24, 2021 at 05:40:21PM +0900, YASUOKA Masahiko wrote:
> Hi,
>
> I hit a problem which is caused by going back of monotonic time. It
> happens on hosts on VMware ESXi.
>
> I wrote the program which repeats the problem.
>
> % cc -o monotime monotime.c -lpthread
> % ./monotime
> 194964 Starting
> 562210 Starting
> 483046 Starting
> 148865 Starting
> 148865 Back 991.808048665 => 991.007447931
> 562210 Back 991.808048885 => 991.007448224
> 483046 Back 991.808049115 => 991.007449172
> 148865 Stopped
> 562210 Stopped
> 483046 Stopped
> 194964 Stopped
> % uname -a
> OpenBSD yasuoka-ob-c.tokyo.iiji.jp 6.8 GENERIC.MP#5 amd64
> % sysctl kern.version
> kern.version=OpenBSD 6.8 (GENERIC.MP) #5: Mon Feb 22 04:36:10 MST 2021
>
> r...@syspatch-68-amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC.MP
> %
>
> monotime.c
> ----
> #include <sys/types.h>
> #include <sys/time.h>
> #include <stdio.h>
> #include <time.h>
> #include <unistd.h>
> #include <pthread.h>
> #include <stdlib.h>
>
> #define NTHREAD 4
> #define NTRY 50000
>
> void *
> start(void *dummy)
> {
> int i;
> struct timespec ts0, ts1;
>
> printf("%d Starting\n", (int)getthrid());
> clock_gettime(CLOCK_MONOTONIC, &ts0);
>
> for (i = 0; i < NTRY; i++) {
> clock_gettime(CLOCK_MONOTONIC, &ts1);
> if (timespeccmp(&ts0, &ts1, <=)) {
> ts0 = ts1;
> continue;
> }
> printf("%d Back %lld.%09lu => %lld.%09lu\n",
> (int)getthrid(), ts0.tv_sec, ts0.tv_nsec, ts1.tv_sec,
> ts1.tv_nsec);
> break;
> }
> printf("%d Stopped\n", (int)getthrid());
>
> return (NULL);
> }
>
> int
> main(int argc, char *argv[])
> {
> int i, n = NTHREAD;
> pthread_t *threads;
>
> threads = calloc(n, sizeof(pthread_t));
>
> for (i = 0; i < n; i++)
> pthread_create(&threads[i], NULL, start, NULL);
> for (i = 0; i < n; i++)
> pthread_join(threads[i], NULL);
>
> }
> ----
>
> The machine has 4 vCPUs and showing the following message on boot.
>
> cpu1: disabling user TSC (skew=-5310)
> cpu2: disabling user TSC (skew=-5335)
> cpu3: disabling user TSC (skew=-7386)
>
> This means "user TSC" is disabled because of TSC of cpu{1,2,3} is much
> delayed against cpu0.
>
> Simply ignoring the skews by the following diff seems to workaround
> this problem.
>
> diff --git a/sys/arch/amd64/amd64/tsc.c b/sys/arch/amd64/amd64/tsc.c
> index 238a5a068e1..3b951a8b5a3 100644
> --- a/sys/arch/amd64/amd64/tsc.c
> +++ b/sys/arch/amd64/amd64/tsc.c
> @@ -212,7 +212,8 @@ cpu_recalibrate_tsc(struct timecounter *tc)
> u_int
> tsc_get_timecount(struct timecounter *tc)
> {
> - return rdtsc_lfence() + curcpu()->ci_tsc_skew;
> + //return rdtsc_lfence() + curcpu()->ci_tsc_skew;
> + return rdtsc_lfence();
> }
>
> void
>
> So I supposed the skews are not calculated properly. Also I found
> NetBSD changed the skew calculating so that it checks 1000 times and
> take the minimum value.
>
>
> https://github.com/NetBSD/src/commit/1dec05c1ae197b4acfc7038e49dfddabcbed0dff
>
> https://github.com/NetBSD/src/commit/66d76b89792bac1c71cd5507ba62b08ad02129ef
>
>
> I checked skews on the machine by the following debug code.
>
> diff --git a/sys/arch/amd64/amd64/tsc.c b/sys/arch/amd64/amd64/tsc.c
> index 238a5a068e1..83e835e4f82 100644
> --- a/sys/arch/amd64/amd64/tsc.c
> +++ b/sys/arch/amd64/amd64/tsc.c
> @@ -302,16 +302,42 @@ tsc_read_bp(struct cpu_info *ci, uint64_t *bptscp,
> uint64_t *aptscp)
> *aptscp = tsc_sync_val;
> }
>
> +#define TSC_SYNC_NTIMES 1000
> +
> +static int tsc_difs[MAXCPUS][TSC_SYNC_NTIMES];
> +
> +void
> +tsc_debug(void)
> +{
> + int i, cpuid = curcpu()->ci_cpuid;
> +
> + for (i = 0; i < TSC_SYNC_NTIMES; i++) {
> + if (i % 10 == 0)
> + printf("%5d", tsc_difs[cpuid][i]);
> + else
> + printf(" %5d", tsc_difs[cpuid][i]);
> + if (i % 10 == 9)
> + printf("\n");
> + }
> + printf("\n");
> +}
> +
> void
> tsc_sync_bp(struct cpu_info *ci)
> {
> + int i, mindif = INT_MAX, dif;
> uint64_t bptsc, aptsc;
>
> - tsc_read_bp(ci, &bptsc, &aptsc); /* discarded - cache effects */
> - tsc_read_bp(ci, &bptsc, &aptsc);
> + for (i = 0; i < TSC_SYNC_NTIMES; i++) {
> + tsc_read_bp(ci, &bptsc, &aptsc);
> + dif = bptsc - aptsc;
> + if (abs(dif) < abs(mindif))
> + mindif = dif;
> + tsc_difs[ci->ci_cpuid][i] = dif;
> + }
>
> /* Compute final value to adjust for skew. */
> - ci->ci_tsc_skew = bptsc - aptsc;
> + ci->ci_tsc_skew = mindif;
> }
>
> /*
> @@ -342,8 +368,10 @@ tsc_post_ap(struct cpu_info *ci)
> void
> tsc_sync_ap(struct cpu_info *ci)
> {
> - tsc_post_ap(ci);
> - tsc_post_ap(ci);
> + int i;
> +
> + for (i = 0; i < TSC_SYNC_NTIMES; i++)
> + tsc_post_ap(ci);
> }
>
> void
>
> ----
> Stopped at db_enter+0x10: popq %rbp
> ddb{0}> machine ddbcpu 1
> Stopped at x86_ipi_db+0x12: leave
> ddb{1}> call tsc_debug
> -8445 -6643 -52183 0 -3 -4 -7 -11 -5 0
> -11 -9 -5 -3 -4 -3 -7 8 -5 -6
> -5 -9 -3 -9 -7 -1 -5 -5 -9 -2
> -6 -4 -6 -4 -11 -8 -3 -4 -8 -1
> -9 -1 -8 1 -8 6 -5 -4 2 -2
> -8 -3 -1 -5 -2 -2 1 2 -2 -9
> -12 0 -9 -2 -2 -5 -2 1 2 0
> -1 2 -2 6 -5 -1 -2 -4 2 -2
> 0 -9 -9 -6 -2 2 3 -6 -1 3
> 8 4 -2 2 -8 7 1 2 -2 1
> -2 -6 -2 5 0 0 -4 -9 6 -2
> -3 -6 -2 -12 1 -9 -2 -3 -10 10
> 2 -1 -3 -2 3 1 1 -5 3 -3
> -5 1 -6 -2 -3 0 0 9 1 6
> 8 -6 5 4 -12 -1 4 2 -1 -1
> -1 2 2 0 -5 1 2 -8 3 9
> 0 6 -3 4 6 0 8 6 -14 -1
> -1 0 -6 -7 6 -10 7 -6 -5 -4
> 6 -12 4 3 -5 5 1 -6 3 0
> -2 0 6 -9 -2 -1 1 -1 4 0
> 4 10 -13 1 -8 -2 -8 -3 -5 -3
> -5 -5 1 -9 -9 0 -3 -1 2 6
> -2 2 -3 -9 -9 -11 -7 -6 -4 -9
> -4 -9 -3 -4 0 -5 0 -9 -12 -7
> -6 -9 1 -5 -4 -12 7 -3 -12 -4
> -5 -5 -6 -9 -7 -1 0 0 -1 -2
> -6 -8 0 1 -8 -5 -2 -4 0 -1
> -3 -10 -15 -3 -8 -11 -9 -9 2 0
> -2 -4 -2 -3 -13 -9 -9 -1 -10 -6
> 0 0 2 -2 -4 1 -6 0 0 -5
> -2 -7 -5 -2 -2 1 -2 -6 -1 -7
> -6 -1 -9 -3 -2 -1 -4 -6 -3 -4
> -4 -3 -4 -11 1 -9 0 -3 2 -9
> -8 2 -1 -7 -5 -5 -9 2 -3 -5
> 0 5 -12 0 -5 -3 -6 -1 -13 -10
> -9 0 0 -5 -7 -4 -3 -3 -3 -2
> -2 3 -5 -3 -5 -1 -7 -4 -10 0
> -3 0 2 1 -4 -1 -5 -3 -5 -6
> -4 -8 -3 0 -1 -2 -13 -10 -9 -5
> -11 -11 -4 -3 0 5 -2 -3 -6 0
> -6 -9 -1 -4 -1 2 -2 -7 -9 0
> -8 -4 -6 -5 -12 -9 -5 -11 -5 -8
> -8 -6 -2 -3 -9 -5 -9 -11 -10 1
> -3 -6 -1 -1 -6 0 0 -8 -4 0
> -3 -10 -4 -2 -3 -2 -1 -9 -11 -12
> -4 2 -2 -5012 5 2 -17 0 7 -5
> 0 -4 -3 6 -7 -1 -1 4 -6 3
> 0 -4 -9 -7 -11 -11 8 -7 -15 -10
> 3 -4 1 -17 -4 3 -17 0 4 3
> -2 0 -3 -10 -2 1 3 -5 -12 -19
> 1 2 5 1 -9 4 -2 -3 -4 0
> -1 -11 -3 -1 -9 -5 0 -8 7 -2
> -6 -7 4 -5 -2 -1 -5 0 -5 -5
> -14 -2 -8 0 -11 9 -10 2 -6 -17
> -3 -5 -6 2 0 9 -14 0 -4 -7
> 6 2 2 -9 -9 5 5 0 -6 3
> -12 5 -2 -13 -10 -5 -7 2 -11 -3
> -6 -2 -13 1 8 -5 -14 2 4 -3
> -13 -5 -11 -9 -10 4 -3 -1 9 -17
> -11 2 -13 -2 -1 -9 -10 0 -5 -4
> 0 10 -8 -5 -8 -3 -14 -6 3 -15
> 1 -5 1 176 -8 -7 -7 -4 -1 -1
> -8 -7 -4 1 -6 -9 1 -2 -9 -4
> -4 -1 -7 0 -8 -3 -4 -3 -1 -2
> -5 -6 -9 2 -6 0 -8 -5 0 -9
> -10 0 -4 4 -6 -11 -3 2 -12 1
> -2 -6 -6 -3 -7 -7 0 -9 -1 -9
> -1 -8 -4 -3 -11 1 0 1 -2 -4
> -11 -1 -9 -9 -10 -1 -1 -9 -8 -6
> -3 -4 -2 -4 1 0 -5 -2 -1 4
> -9 -1 -4 1 -8 -11 0 -10 -4 -9
> -5 -2 -2 4 0 -7 -4 1 -2 1
> -4 -1 -5 -9 -9 -5 -10 -4 -12 -8
> -4 -9 -7 -5 -3 3 -5 -12 -3 0
> -8 -4 -9 -5 -6 0 0 -1 -2 -6
> -8 -12 -3 1 2 -6 -1 -7 -10 -9
> -6 -8 0 -2 -3 -7 -3 -2 6 -3
> -12 0 0 -7 -9 -6 -1 -5 -2 -9
> -6 1 0 -3 -1 -1 -2 2 -2 -3
> -7 -9 -1 -8 -4 -2 5 -5 -3 -10
> 2 6 -3 0 -6 -8 -9 -1 -1 -7
> -8 -1 -1 -4 -4 7 -2 -10 -11 -6
> 2 2 -4 3 -2 -1 -3 0 0 -7
> -1 -3 -4 -9 -5 -2 -5 -7 -5 -3
> 0 1 -3 5 -3 -4 -1 -6 -9 -4
> -6 0 -9 -6 0 -2 4 -2 -4 -10
> -9 -4 -3 -9 -3 -6 -9 -8 -4 1
> -5 -6 -5 1 0 -2 -3 -6 -5 -9
> -4 1 -5 -4 -2 -4 -8 -3 -4 0
> 2 -5 -3 -7 -1 -2 -1 -9 -6 -15
> -10 -6 -2 -7 -1 -3 3 -6 -6 -9
> -10 -8 -9 -2 -3 0 -6 3 -4 4
> 3 3 8 -2 -2 -4 0 -3 -9 -3
> -6 -4 3 2 1 1 -3 -7 -15 -1
> -4 -6 -1 -2 -1 -12 2 -1 -4 1
> 2 3 -5 3 -3 -7 -6 -5 0 1
> 5 -13 -8 0 -5 2 0 -5 -3 6
> -4 -9 -2 -8 -1 -9 -10 -1 -10 -6
> -10 -4 -10 -9 -2 1 0 -4 -3 0
> 1 -3 -1 -4 -7 -10 -13 -8 -1 -1
>
> 0x1
> ddb{1}> machine ddbcpu 2
> Stopped at x86_ipi_db+0x12: leave
> ddb{2}> call tsc_debug
> -8242 -6496 -50265 -1 -2 -2 1 109 -2 -3
> 3 3 -8 -3 -4 4 0 -8 -7 -5
> -5 3 -7 -5 -4 -9 -7 -3 0 2
> -5 2 1 3 -2 3 8 -6 -11 8
> 8 -5 1 5 0 -8 2 0 6 3
> -14 7 -2 -1 -3 1 -5 -6 0 5
> 1 -1 0 -2 -5 2 -3 0 -3 -1
> -5 -12 -4 -4 -9 4 0 -2 -2 -8
> 2 5 7 -2 0 -6 110 -8 -8 -4
> 0 5 -7 -3 -5 -4 9 -2 -2 3
> -8 -2 -5 4 -3 217 0 -7 -7 -6
> 7 -10 -9 -3 3 -14 3 -5 5 -12
> 5 -8 -17 -5 286 -6 0 -3 -4 -2
> 1 -5 -5 -9 -6 -7 -3 -5 6 0
> 1 -1 -4 -2 2 -2 -2 -2 -5 2
> 12 3 -18 -8 6 -4 -3 6 2 -3
> -7 -3 4 -5 -23 9 6 6 -6 -11
> 9 -1 -10 50505 -1 2 6 -11 2 -2
> -4 -6 201 1 3 4 -9 6 0 1
> -4 0 -1 3 4 1 6 -7 -5 4
> -14 -3 -1 -8 5 6 -5 3 -7 -9
> -7 1 -2 5 -2 0 -2 -9 4 -3
> 98 -5 7 -7 3 0 -5 0 9 2
> -7 -5 -3 -12 -11 -11 6 -5 -7 -6
> 210 5 -3 -5 -4 -11 -6 0 -5 -9
> 3 0 -9 5 1 0 0 -7 -5 210
> 1 -6 -17 -8 0 1 -2 -8 1 -7
> 10 -8 -8 -9 4 -2 -4 -3 204 5
> -9 -15 3 -1 -5 0 -12 -1 0 1
> -1 -6 -5 -9 -1 4 -1 -1 0 1
> 4 -8 -13 1 0 -5 -6 0 -4 0
> 6 -2 -4 -8 -7 -12 -2 -2 -6 -8
> -5 0 -8 -7 -11 0 6 -1 -8 -3
> 2 6 -6 0 -10 285 -1 -2 -8 -6
> -6 -1 -5 -6 0 -5 -8 -5 1 -8
> -1 1 4 -3 -4 188 -3 -3 -10 5
> 6 0 -7 4 1 2 0 -2 -2 2
> -3 -1 -9 -12 201 -1 -7 -1 8 0
> 0 0 -5 0 7 -18 5 1 -2 15
> 5 -6 4 -10 272 0 -4 -3 2 -10
> -7 7 3 4 1 6 -9 -8 -12 -2
> -2 -2 -6 8 3 97 -1 -7 5 1
> -7 4 -6 -9 -7 -2 0 -6 8 -13
> -1 3 -9 -4 233 -2 0 0 -5 -5
> -2 0 0 5 -7 6 -14 4 -6 5
> 4 3 3 -3 103 -2 -6 -11 2 -2
> -3 -8 0 -1 1 0 -6 1 1 -10
> -1 0 -1 0 0 -1 -6 -4 -4 -3
> 3 0 -5 0 2 -2 6 0 -13 3
> 1 -16 -2 -12 206 -4 -6 -5 -3 2
> -8 -2 -9 -2 9 7 -1 1 3 -5
> 6 6 10 0 99 -2 -1 -2 0 -6
> -8 -9 0 3 0 -11 -6 1 -4 9
> -11 -11 2 -1 276 -7 2 4 -4 -3
> 4 2 -12 8 1 -4 0 -1 -1 -1
> 5 -8 0 6 232 -4 2 -2 -11 2
> -6 0 4 6 -7 8 -6 3 -3 -9
> 0 1 -7 2 -3 -5 -3 -2 -8 -4
> -14 -2 0 4 -4 -3 -4 9 3 -1
> -5 -8 1 0 210 0 -6 -4 2 2
> 0 -3 -4 0 4 -12 10 -2 -3 -1
> 3 7 -3 6 -9 190 4 6 -2 11
> -10 -2 -9 3 2 4 -4 -7 -9 -2
> -10 12 1 -7 3 -1 0 -1 -11 6
> 6 0 -8 1 4 2 -2 -5 0 0
> 5 -5 -3 -5 205 -1 -3 -1 0 -4
> -3 -7 3 0 -7 -5 -4 -3 7 1
> -14 -2 0 -14 2 0 7 11 -2 -7
> 6 -6 -3 -4 0 4 -3 -4 -3 -7
> -6 3 -4 0 4 -3 -6 3 4 -4
> -6 7 -11 1 -10 -10 -12 -4 -3 6
> -5 -6 5 212 -8 1 -4 3 -3 -8
> 0 -10 0 2 6 -3 -1 -13 1 6
> -6 5 -4 0 -7 -4 -5 -6 1 -2
> 1 -10 -12 -3 -8 9 -11 0 1 -3
> -1 1 186 2 -12 3 -4 0 0 -1
> -5 -4 1 -1 4 5 -10 0 -4 1
> -1 -6 270 0 -17 5 0 0 -4 -1
> 4 6 1 0 -6 6 5 -3 3 -2
> -6 -7 18 -16 0 9 4 3 -1 6
> 4 1 -5 7 -11 -1 -6 -2 6 -2
> -8 256 -5 -2 1 -1 -5 -3 -9 0
> -1 3 -2 -11 4 3 -7 -3 3 -1
> -5 -5 0 -4 -1 -3 -3 6 -3 3
> 3 -2 -11 -2 -3 11 -8 1 5 -7
> 5 -12 -3 8 -9 -5 7 5 -3 2
> -7 -3 -6 2 -2 -1 -2 -9 -8 99
> 6 -2 9 -1 -4 -2 0 -2 -7 -5
> 0 1 -4 -8 -1 -2 2 -8 2 205
> -3 -10 -1 -1 -2 -5 2 2 5 -2
> -5 -6 -4 6 6 -6 4 1 -5 0
> -7 3 1 0 -11 -7 -3 5 -5 5
> 1 -1 3 -5 -8 0 -1 0 183 -5
> 0 4 -1 -6 -11 -10 1 -18 3 -1
> -5 -9 -2 2 2 -2 0 99 -7 -8
> -1 -3 -5 -1 12 -3 -1 2 1 4
> 7 3 -14 2 -4 8 -9 -3 -8 -5
> 6 -6 5 -12 6 -1 -9 -4 -4 1
> -6 0 0 -2 -3 -5 -9 -2 -9 -3
> 3 -16 -2 -1 0 9 -4 5 -6 5
>
> 0x1
> ddb{2}> machine ddbcpu 3
> Stopped at x86_ipi_db+0x12: leave
> ddb{3}> call tsc_debug
> -8336 -6457 -45527 0 -1 0 -2 4 0 5
> -9 -6 -4 -4 6 4 0 -3 -4 5
> 3 6 -12 -1 1 1 -3 6 -4 -2
> -2 2 4 -3 0 -1 3 0 1 3
> -3 3 -1 4 -5 -2 -2 -9 6 -9
> 0 1 -2 -6 8 -4 -2 -2 6 1
> -1 1 -6 -6 4 -5 -1 6 -1 -1
> 3 5 0 -6 5 -4 -2 -6 -3 -4
> -5 2 -3 -3 -5 -3 -5 -5720 0 5
> -1 -3 3 -2 4 -6 8 -16 -6 -3
> -2 4 8 -3 3 -1 2 -8 -3 -19
> -8 3 -7 -9 -6 -3 1 -3 -6 7
> 4 1 3 2 -4 7 -3 2 2 -4
> -10 -8 -14 -2 2 -3 3 -2 3 -3
> 5 5 -6 9 -3 -12 -6 1 -10 7
> -6 5 -4 1 6 3 2 2 -6 -1
> 6 -8 -5 7 3 -3 -4 -6 1 -5
> 4 -6 2 6 -6 3 -8 -5 -6 0
> -5 -2 -13 -8 3 0 -17 -7 -9 1
> 6 -12 3 6 3 -3 4 -1 -7 0
> 2 0 -7 -10 -6 3 1 2 -19 4
> 1 -18 1 -3 -6 -14 4 6 3 -4
> -7 -11 -1 -1 3 0 6 6 -8 -14
> 3 -2 6 -5 0 1 0 -7 0 4
> -3 -16 -2 -4 -12 -4 6 0 -8 -4
> -4 -4 -3 0 -6 -13 -10 -15 -6 2
> 0 -3 0 -8 4 -1 -1 5 -4 -1
> -7 2 1 1 3 -3 -1 -18 6 8
> -2 3 -6 0 -6 -2 2 -2 -7 1
> -6 -13 -4 -2 -1 -6 6 -5 -9 -14
> 4 5 -4 -2 -9 -2 -2 -13 1 -18
> -1 2 -5 6 -6 -7 -9 -6 1 6
> -13 -4 3 0 -8 -6 -6 -10 -2 -9
> 5 2 4 1 6 -5 -8 -6 2 -4
> -3 -5 -2 -6 -10 0 5 -2 8 -3
> 8 -11 -11 -7 -5 -13 -19 -5 -14 -3
> 3 2 1 3 6 -1 -9 -16 3 -7
> 1 -3 -5 0 -7 6 4 -2 -2 4
> 4 -6 -6 8 -6 -7 -5 -2 0 4
> -1 3 1 3 -6 -2 -4 -9 1 0
> -2 3 2 -16 4 -15 -11 -3 8 0
> -6 -3 -18 -7 -8 8 -8 6 -7 -4
> -8 -7 -9 0 -2 3 7 -2 1 -6
> 2 -6 8 -1 -12 -8 -4 3 -13 -2
> -11 1 -2 -7 -3 0 -16 -8 4 -9
> -15 -8 -9 8 5 7 -9 5 -10 10
> 1 6 -6 3 -6 -4 5 0 -3 -7
> -1 -4 -10 -2 0 -11 8 -8 -3 -11
> 4 -6 1 -8 -1 -6 3 -1 -12 -7
> 11 -1 0 -4 -13 0 -10 7 -2 0
> -6 4 0 -12 -9 6 -2 -5 -5 -7
> -1 -9 -5 3 3 -7 3 -16 -2 -10
> 0 -2 6 4 4 -6 -9 -3 -6 4
> -5 -1 7 -11 -21 -9 -3 -1 4 -13
> -7 0 -3 0 -10 -7 8 -9 7 6
> -9 -14 5 -9 -7 -8 11 -6 1 -3
> -9 7 5 0 -12 -3 -4 -18 0 4
> -1 8 -8 9 4 0 5 -10 -10 -8
> 1 8 -1 4 1 -3 -6 -6 -5 -1
> -12 -6 -8 -14 -10 2 -1 9 0 -9
> 0 -8 -6 -5 1 -14 -6 5 1 -7
> 3 -9 6 -4 3 0 -21 8 7 5
> 0 0 -2 2 1 -7 -11 1 -7 2
> -3 -2 9 1 3 -1 -8 0 -18 -7
> -3 6 1 1 -8 -5 -1 2 -4 -6
> -10 -5 -20 -18 5 -7 2 5 -3 -3
> -6 -3 -3 4 -7 -2 -4 2 -6 -11
> -4 -15 -6 -5 2 -1 -10 -4 -6 -3
> -1 0 -6 -10 5 -5 2 2 -8 6
> -1 5 -5 -9 12 0 -6 -17 2 -4
> 4 -11 -1 -8 0 -3 -11 -8 0 -6
> 2 -1 -4 -9 -12 3 6 -10 -2 -6
> -13 1 -10 1 -13 5 -4 3 -11 4
> -8 4 4 -17 6 -12 3 -13 0 -3
> -7 4 -11 -1 1 3 5 -8 -3 0
> 4 -8 -1 4 -13 2 -1 2 5 -6
> -6 4 -4 -5 1 -4 4 5 -2 -10
> -9 0 -14 -2 -7 1 -1 -14 -15 8
> -3 9 -3 -4 -8 0 2 5 -11 -13
> -5 -13 -3 -7 9 -3 -7 -6 -9 -7
> -8 -10 -1 6 2 2 3 -13 -6 -5
> 3 -13 2 -6 4 1 0 -9 4 -11
> -15 0 1 -12 -1 2 -11 2 -1 -13
> -13 -5 3 9 -11 -8 2 -12 -2 -5
> 0 -3 -2 1 -6 -2 -1 0 -3 2
> 0 -9 -4 1 -2 3 -6 0 -4 -6
> 2 1 -14 -4 -7 -23 -1 -4 -8 -8
> 2 -4 1 6 -10 5 -9 -6 6 -1
> 4 -2 -9 0 -5 -1 8 0 3 4
> -4 -7 -4 2 -15 -6 2 1 -6 -4
> -3 -3 3 -7 -2 -9 9 -2 -9 -4
> 3 -8 -10 -3 -4 1 -4 -10 -1 3
> -1 -15 5 0 2 -6 4 6 -2 0
> 2 5 3 -12 -5 -5 -2 -4 -4 5
> -8 -4 -6 -16 -1 4 -4 -1 -8 0
> 6 -10 5 0 1 -3 -9 8 0 -11
> -3 -13 -1 3 3 -6 -6 -2 3 5
> 4 -6 12 -17 1 -5 -5 -2 1 -3
> -7 5 -4 8 -5 -1 -5 8 -12 4
> -6 2 -5 -9 2 -2 5 3 -5 -8
>
> 0x1
> ddb{3}>
> ----
>
> I seems that we can fix the problem by comparing TSC more times and
> choosing the minimum value.
>
> The diff also includes a fix for the problem (use the minimum value of
> 1000 samplings). With the diff, I tried "monotime" test program over
> 100 times, it didn't find any clock going back.
>
>
> dmesg
> ----
> OpenBSD 6.8 (GENERIC.MP) #5: Mon Feb 22 04:36:10 MST 2021
>
> r...@syspatch-68-amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC.MP
> real mem = 6425083904 (6127MB)
> avail mem = 6215311360 (5927MB)
> random: good seed from bootblocks
> mpath0 at root
> scsibus0 at mpath0: 256 targets
> mainbus0 at root
> bios0 at mainbus0: SMBIOS rev. 2.7 @ 0xbfbb401f (98 entries)
> bios0: vendor VMware, Inc. version "VMW71.00V.0.B64.1506250318" date
> 06/25/2015
> bios0: VMware, Inc. VMware7,1
> acpi0 at bios0: ACPI 4.0
> acpi0: sleep states S0 S1 S4 S5
> acpi0: tables DSDT SRAT FACP APIC MCFG HPET WAET
> acpi0: wakeup devices PCI0(S3) USB_(S1) P2P0(S3) S1F0(S3) S2F0(S3) S3F0(S3)
> S4F0(S3) S5F0(S3) S6F0(S3) S7F0(S3) S8F0(S3) S9F0(S3) S10F(S3) S11F(S3)
> S12F(S3) S13F(S3) [...]
> acpitimer0 at acpi0: 3579545 Hz, 24 bits
> acpimadt0 at acpi0 addr 0xfee00000: PC-AT compat
> cpu0 at mainbus0: apid 0 (boot processor)
> cpu0: Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz, 2397.42 MHz, 06-3f-02
> cpu0:
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,MMX,FXSR,SSE,SSE2,SS,HTT,SSE3,PCLMUL,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,HV,NXE,RDTSCP,LONG,LAHF,PERF,ITSC,SENSOR,ARAT,MELTDOWN
> cpu0: 256KB 64b/line 8-way L2 cache
> cpu0: smt 0, core 0, package 0
> mtrr: Pentium Pro MTRR support, 8 var ranges, 88 fixed ranges
> cpu0: apic clock running at 65MHz
> cpu1 at mainbus0: apid 1 (application processor)
> cpu1: Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz, 2397.29 MHz, 06-3f-02
> cpu1:
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,MMX,FXSR,SSE,SSE2,SS,HTT,SSE3,PCLMUL,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,HV,NXE,RDTSCP,LONG,LAHF,PERF,ITSC,SENSOR,ARAT,MELTDOWN
> cpu1: 256KB 64b/line 8-way L2 cache
> cpu1: disabling user TSC (skew=-5310)
> cpu1: smt 0, core 1, package 0
> cpu2 at mainbus0: apid 2 (application processor)
> cpu2: Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz, 2397.28 MHz, 06-3f-02
> cpu2:
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,MMX,FXSR,SSE,SSE2,SS,HTT,SSE3,PCLMUL,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,HV,NXE,RDTSCP,LONG,LAHF,PERF,ITSC,SENSOR,ARAT,MELTDOWN
> cpu2: 256KB 64b/line 8-way L2 cache
> cpu2: disabling user TSC (skew=-5335)
> cpu2: smt 0, core 2, package 0
> cpu3 at mainbus0: apid 3 (application processor)
> cpu3: Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz, 2397.29 MHz, 06-3f-02
> cpu3:
> FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,DS,MMX,FXSR,SSE,SSE2,SS,HTT,SSE3,PCLMUL,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,HV,NXE,RDTSCP,LONG,LAHF,PERF,ITSC,SENSOR,ARAT,MELTDOWN
> cpu3: 256KB 64b/line 8-way L2 cache
> cpu3: disabling user TSC (skew=-7386)
> cpu3: smt 0, core 3, package 0
> ioapic0 at mainbus0: apid 4 pa 0xfec00000, version 11, 24 pins, remapped
> acpimcfg0 at acpi0
> acpimcfg0: addr 0xe0000000, bus 0-127
> acpihpet0 at acpi0: 14318179 Hz
> acpiprt0 at acpi0: bus 0 (PCI0)
> acpipci0 at acpi0 PCI0: 0x00000000 0x00000011 0x00000001
> acpicmos0 at acpi0
> "PNP0A05" at acpi0 not configured
> acpiac0 at acpi0: AC unit online
> acpicpu0 at acpi0: C1(@1 halt!)
> acpicpu1 at acpi0: C1(@1 halt!)
> acpicpu2 at acpi0: C1(@1 halt!)
> acpicpu3 at acpi0: C1(@1 halt!)
> cpu0: using Broadwell MDS workaround
> pvbus0 at mainbus0: VMware
> vmt0 at pvbus0
> pci0 at mainbus0 bus 0
> 0:16:0: rom address conflict 0xffffc000/0x4000
> pchb0 at pci0 dev 0 function 0 "Intel 82443BX AGP" rev 0x01
> ppb0 at pci0 dev 1 function 0 "Intel 82443BX AGP" rev 0x01
> pci1 at ppb0 bus 1
> pcib0 at pci0 dev 7 function 0 "Intel 82371AB PIIX4 ISA" rev 0x08
> pciide0 at pci0 dev 7 function 1 "Intel 82371AB IDE" rev 0x01: DMA, channel 0
> configured to compatibility, channel 1 configured to compatibility
> pciide0: channel 0 disabled (no drives)
> atapiscsi0 at pciide0 channel 1 drive 0
> scsibus1 at atapiscsi0: 2 targets
> cd0 at scsibus1 targ 0 lun 0: <NECVMWar, VMware IDE CDR10, 1.00> removable
> cd0(pciide0:1:0): using PIO mode 4, Ultra-DMA mode 2
> piixpm0 at pci0 dev 7 function 3 "Intel 82371AB Power" rev 0x08: SMBus
> disabled
> "VMware VMCI" rev 0x10 at pci0 dev 7 function 7 not configured
> "VMware SVGA II" rev 0x00 at pci0 dev 15 function 0 not configured
> mpi0 at pci0 dev 16 function 0 "Symbios Logic 53c1030" rev 0x01: apic 4 int 17
> mpi0: 0, firmware 1.3.41.32
> scsibus2 at mpi0: 16 targets, initiator 7
> sd0 at scsibus2 targ 0 lun 0: <VMware, Virtual disk, 1.0>
> sd0: 16384MB, 512 bytes/sector, 33554432 sectors
> sd1 at scsibus2 targ 1 lun 0: <VMware, Virtual disk, 1.0>
> sd1: 81920MB, 512 bytes/sector, 167772160 sectors
> mpi0: target 0 Sync at 160MHz width 16bit offset 127 QAS 1 DT 1 IU 1
> mpi0: target 1 Sync at 160MHz width 16bit offset 127 QAS 1 DT 1 IU 1
> ppb1 at pci0 dev 17 function 0 "VMware PCI" rev 0x02
> pci2 at ppb1 bus 2
> ppb2 at pci0 dev 21 function 0 "VMware PCIE" rev 0x01: msi
> pci3 at ppb2 bus 3
> vmx0 at pci3 dev 0 function 0 "VMware VMXNET3" rev 0x01: msix, 4 queues,
> address 00:0c:29:7d:49:9c
> ppb3 at pci0 dev 21 function 1 "VMware PCIE" rev 0x01: msi
> pci4 at ppb3 bus 4
> ppb4 at pci0 dev 21 function 2 "VMware PCIE" rev 0x01: msi
> pci5 at ppb4 bus 5
> ppb5 at pci0 dev 21 function 3 "VMware PCIE" rev 0x01: msi
> pci6 at ppb5 bus 6
> ppb6 at pci0 dev 21 function 4 "VMware PCIE" rev 0x01: msi
> pci7 at ppb6 bus 7
> ppb7 at pci0 dev 21 function 5 "VMware PCIE" rev 0x01: msi
> pci8 at ppb7 bus 8
> ppb8 at pci0 dev 21 function 6 "VMware PCIE" rev 0x01: msi
> pci9 at ppb8 bus 9
> ppb9 at pci0 dev 21 function 7 "VMware PCIE" rev 0x01: msi
> pci10 at ppb9 bus 10
> ppb10 at pci0 dev 22 function 0 "VMware PCIE" rev 0x01: msi
> pci11 at ppb10 bus 11
> ppb11 at pci0 dev 22 function 1 "VMware PCIE" rev 0x01: msi
> pci12 at ppb11 bus 12
> ppb12 at pci0 dev 22 function 2 "VMware PCIE" rev 0x01: msi
> pci13 at ppb12 bus 13
> ppb13 at pci0 dev 22 function 3 "VMware PCIE" rev 0x01: msi
> pci14 at ppb13 bus 14
> ppb14 at pci0 dev 22 function 4 "VMware PCIE" rev 0x01: msi
> pci15 at ppb14 bus 15
> ppb15 at pci0 dev 22 function 5 "VMware PCIE" rev 0x01: msi
> pci16 at ppb15 bus 16
> ppb16 at pci0 dev 22 function 6 "VMware PCIE" rev 0x01: msi
> pci17 at ppb16 bus 17
> ppb17 at pci0 dev 22 function 7 "VMware PCIE" rev 0x01: msi
> pci18 at ppb17 bus 18
> ppb18 at pci0 dev 23 function 0 "VMware PCIE" rev 0x01: msi
> pci19 at ppb18 bus 19
> ppb19 at pci0 dev 23 function 1 "VMware PCIE" rev 0x01: msi
> pci20 at ppb19 bus 20
> ppb20 at pci0 dev 23 function 2 "VMware PCIE" rev 0x01: msi
> pci21 at ppb20 bus 21
> ppb21 at pci0 dev 23 function 3 "VMware PCIE" rev 0x01: msi
> pci22 at ppb21 bus 22
> ppb22 at pci0 dev 23 function 4 "VMware PCIE" rev 0x01: msi
> pci23 at ppb22 bus 23
> ppb23 at pci0 dev 23 function 5 "VMware PCIE" rev 0x01: msi
> pci24 at ppb23 bus 24
> ppb24 at pci0 dev 23 function 6 "VMware PCIE" rev 0x01: msi
> pci25 at ppb24 bus 25
> ppb25 at pci0 dev 23 function 7 "VMware PCIE" rev 0x01: msi
> pci26 at ppb25 bus 26
> ppb26 at pci0 dev 24 function 0 "VMware PCIE" rev 0x01: msi
> pci27 at ppb26 bus 27
> ppb27 at pci0 dev 24 function 1 "VMware PCIE" rev 0x01: msi
> pci28 at ppb27 bus 28
> ppb28 at pci0 dev 24 function 2 "VMware PCIE" rev 0x01: msi
> pci29 at ppb28 bus 29
> ppb29 at pci0 dev 24 function 3 "VMware PCIE" rev 0x01: msi
> pci30 at ppb29 bus 30
> ppb30 at pci0 dev 24 function 4 "VMware PCIE" rev 0x01: msi
> pci31 at ppb30 bus 31
> ppb31 at pci0 dev 24 function 5 "VMware PCIE" rev 0x01: msi
> pci32 at ppb31 bus 32
> ppb32 at pci0 dev 24 function 6 "VMware PCIE" rev 0x01: msi
> pci33 at ppb32 bus 33
> ppb33 at pci0 dev 24 function 7 "VMware PCIE" rev 0x01: msi
> pci34 at ppb33 bus 34
> isa0 at pcib0
> isadma0 at isa0
> fdc0 at isa0 port 0x3f0/6 irq 6 drq 2
> com0 at isa0 port 0x3f8/8 irq 4: ns16550a, 16 byte fifo
> com0: console
> com1 at isa0 port 0x2f8/8 irq 3: ns16550a, 16 byte fifo
> pckbc0 at isa0 port 0x60/5 irq 1 irq 12
> pckbd0 at pckbc0 (kbd slot)
> wskbd0 at pckbd0 mux 1
> pms0 at pckbc0 (aux slot)
> wsmouse0 at pms0 mux 0
> pcppi0 at isa0 port 0x61
> spkr0 at pcppi0
> efifb0 at mainbus0: 1152x864, 32bpp
> wsdisplay0 at efifb0 mux 1
> wskbd0: connecting to wsdisplay0
> wsdisplay0: screen 0-5 added (std, vt100 emulation)
> vscsi0 at root
> scsibus3 at vscsi0: 256 targets
> softraid0 at root
> scsibus4 at softraid0: 256 targets
> root on sd0a (a5bd0d220920df21.a) swap on sd0b dump on sd0b
> ----
>
> This happens on 2 ESXi hosts at least
>
> - 1
> - ESXi 6.0.0, 3029758
> - Xeon E5-2620 x 2
> - 2
> - ESXi 6.7.0 Update 3 (Build 14320388)
> - Xeon Silver 4208 x 2
>
This broke the monotonic clock on my Ryzen 5 2500U.
Note: userland TSC has never worked on this device.
$ ./monotime
320678 Starting
351995 Starting
387215 Starting
505501 Starting
387215 Stopped
505501 Back 213.019672364 => 212.599096325
505501 Stopped
351995 Stopped
320678 Stopped
OpenBSD 6.9-beta (GENERIC.MP) #10: Wed Mar 24 09:03:54 EDT 2021
jr...@drifter.i.zettaport.com:/usr/src/sys/arch/amd64/compile/GENERIC.MP
real mem = 16762552320 (15986MB)
avail mem = 16238886912 (15486MB)
random: good seed from bootblocks
mpath0 at root
scsibus0 at mpath0: 256 targets
mainbus0 at root
bios0 at mainbus0: SMBIOS rev. 3.1 @ 0x986eb000 (62 entries)
bios0: vendor LENOVO version "R0UET78W (1.58 )" date 11/17/2020
bios0: LENOVO 20KUCTO1WW
acpi0 at bios0: ACPI 5.0
acpi0: sleep states S0 S3 S4 S5
acpi0: tables DSDT FACP SSDT SSDT CRAT CDIT SSDT TPM2 UEFI MSDM BATB HPET APIC
MCFG SBST WSMT VFCT IVRS FPDT SSDT SSDT SSDT BGRT UEFI SSDT
acpi0: wakeup devices GPP0(S3) GPP1(S3) GPP2(S3) GPP3(S3) GPP4(S3) GPP5(S3)
GPP6(S3) GP17(S3) XHC0(S3) XHC1(S3) GP18(S3) LID_(S3) SLPB(S3)
acpitimer0 at acpi0: 3579545 Hz, 32 bits
acpihpet0 at acpi0: 14318180 Hz
acpimadt0 at acpi0 addr 0xfee00000: PC-AT compat
cpu0 at mainbus0: apid 0 (boot processor)
cpu0: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.57 MHz, 17-11-00
cpu0:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu0: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu0: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu0: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu0: smt 0, core 0, package 0
mtrr: Pentium Pro MTRR support, 8 var ranges, 88 fixed ranges
cpu0: apic clock running at 24MHz
cpu0: mwait min=64, max=64, C-substates=1.1, IBE
cpu1 at mainbus0: apid 1 (application processor)
cpu1: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.26 MHz, 17-11-00
cpu1:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu1: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu1: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu1: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu1: disabling user TSC (skew=1459148152)
cpu1: smt 1, core 0, package 0
cpu2 at mainbus0: apid 2 (application processor)
cpu2: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.26 MHz, 17-11-00
cpu2:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu2: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu2: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu2: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu2: disabling user TSC (skew=1459148192)
cpu2: smt 0, core 1, package 0
cpu3 at mainbus0: apid 3 (application processor)
cpu3: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.26 MHz, 17-11-00
cpu3:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu3: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu3: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu3: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu3: disabling user TSC (skew=1459148152)
cpu3: smt 1, core 1, package 0
cpu4 at mainbus0: apid 4 (application processor)
cpu4: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.26 MHz, 17-11-00
cpu4:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu4: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu4: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu4: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu4: disabling user TSC (skew=1459148182)
cpu4: smt 0, core 2, package 0
cpu5 at mainbus0: apid 5 (application processor)
cpu5: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.26 MHz, 17-11-00
cpu5:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu5: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu5: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu5: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu5: disabling user TSC (skew=1459148152)
cpu5: smt 1, core 2, package 0
cpu6 at mainbus0: apid 6 (application processor)
cpu6: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.26 MHz, 17-11-00
cpu6:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu6: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu6: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu6: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu6: disabling user TSC (skew=1459148192)
cpu6: smt 0, core 3, package 0
cpu7 at mainbus0: apid 7 (application processor)
cpu7: AMD Ryzen 5 2500U with Radeon Vega Mobile Gfx, 1996.26 MHz, 17-11-00
cpu7:
FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu7: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line
8-way L2 cache
cpu7: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu7: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu7: disabling user TSC (skew=1459148152)
cpu7: smt 1, core 3, package 0
ioapic0 at mainbus0: apid 32 pa 0xfec00000, version 21, 24 pins, can't remap
ioapic1 at mainbus0: apid 33 pa 0xfec01000, version 21, 32 pins, can't remap
acpimcfg0 at acpi0
acpimcfg0: addr 0xf8000000, bus 0-63
acpiprt0 at acpi0: bus 0 (PCI0)
acpiprt1 at acpi0: bus 1 (GPP0)
acpiprt2 at acpi0: bus 2 (GPP1)
acpiprt3 at acpi0: bus 3 (GPP2)
acpiprt4 at acpi0: bus -1 (GPP3)
acpiprt5 at acpi0: bus -1 (GPP4)
acpiprt6 at acpi0: bus 4 (GPP5)
acpiprt7 at acpi0: bus -1 (GPP6)
acpiprt8 at acpi0: bus 5 (GP17)
acpiprt9 at acpi0: bus 6 (GP18)
acpiec0 at acpi0
acpibtn0 at acpi0: PWRB
acpipci0 at acpi0 PCI0: 0x00000010 0x00000011 0x00000000
acpicmos0 at acpi0
acpibat0 at acpi0: BAT0 model "01AV448" serial 2798 type LiP oem "Celxpert"
acpiac0 at acpi0: AC unit online
acpithinkpad0 at acpi0: version 2.0
"SMB0001" at acpi0 not configured
acpibtn1 at acpi0: LID_
acpibtn2 at acpi0: SLPB
"PNP0C14" at acpi0 not configured
"PNP0C14" at acpi0 not configured
"PNP0C14" at acpi0 not configured
"STM7304" at acpi0 not configured
"USBC000" at acpi0 not configured
acpicpu0 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu1 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu2 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu3 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu4 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu5 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu6 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu7 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpipwrres0 at acpi0: P0ST, resource for SATA
acpipwrres1 at acpi0: P3ST, resource for SATA
acpivideo0 at acpi0: VGA_
acpivout0 at acpivideo0: LCD_
cpu0: 1996 MHz: speeds: 2000 1700 1600 MHz
pci0 at mainbus0 bus 0
ksmn0 at pci0 dev 0 function 0 "AMD 17h/1xh Root Complex" rev 0x00
"AMD 17h/1xh IOMMU" rev 0x00 at pci0 dev 0 function 2 not configured
pchb0 at pci0 dev 1 function 0 "AMD 17h PCIE" rev 0x00
ppb0 at pci0 dev 1 function 1 "AMD 17h/1xh PCIE" rev 0x00: msi
pci1 at ppb0 bus 1
ppb1 at pci0 dev 1 function 2 "AMD 17h/1xh PCIE" rev 0x00: msi
pci2 at ppb1 bus 2
re0 at pci2 dev 0 function 0 "Realtek 8168" rev 0x10: RTL8168GU/8111GU
(0x5080), msi, address e8:6a:64:f0:f8:14
rgephy0 at re0 phy 7: RTL8251 PHY, rev. 0
ppb2 at pci0 dev 1 function 3 "AMD 17h/1xh PCIE" rev 0x00: msi
pci3 at ppb2 bus 3
sdhc0 at pci3 dev 0 function 0 "O2 Micro 0Z8621 SD/MMC" rev 0x01: apic 33 int 8
sdhc0: SDHC 4.0, 50 MHz base clock
sdmmc0 at sdhc0: 4-bit, sd high-speed, mmc high-speed, ddr52, dma
ppb3 at pci0 dev 1 function 6 "AMD 17h/1xh PCIE" rev 0x00: msi
pci4 at ppb3 bus 4
iwm0 at pci4 dev 0 function 0 "Intel Dual Band Wireless-AC 8265" rev 0x78, msi
pchb1 at pci0 dev 8 function 0 "AMD 17h PCIE" rev 0x00
ppb4 at pci0 dev 8 function 1 "AMD 17h/1xh PCIE" rev 0x00
pci5 at ppb4 bus 5
amdgpu0 at pci5 dev 0 function 0 "ATI Radeon Vega" rev 0xc4
drm0 at amdgpu0
amdgpu0: msi
azalia0 at pci5 dev 0 function 1 "ATI Radeon Vega HD Audio" rev 0x00: msi
azalia0: no supported codecs
ccp0 at pci5 dev 0 function 2 "AMD 17h/1xh Crypto" rev 0x00
xhci0 at pci5 dev 0 function 3 "AMD 17h/1xh xHCI" rev 0x00: msi, xHCI 1.10
usb0 at xhci0: USB revision 3.0
uhub0 at usb0 configuration 1 interface 0 "AMD xHCI root hub" rev 3.00/1.00
addr 1
xhci1 at pci5 dev 0 function 4 "AMD 17h/1xh xHCI" rev 0x00: msi, xHCI 1.10
usb1 at xhci1: USB revision 3.0
uhub1 at usb1 configuration 1 interface 0 "AMD xHCI root hub" rev 3.00/1.00
addr 1
azalia1 at pci5 dev 0 function 6 "AMD 17h/1xh HD Audio" rev 0x00: apic 33 int 30
azalia1: codecs: Conexant/0x5111
audio0 at azalia1
ppb5 at pci0 dev 8 function 2 "AMD 17h/1xh PCIE" rev 0x00
pci6 at ppb5 bus 6
ahci0 at pci6 dev 0 function 0 "AMD FCH AHCI" rev 0x61: msi, AHCI 1.3.1
ahci0: port 0: 6.0Gb/s
scsibus1 at ahci0: 32 targets
sd0 at scsibus1 targ 0 lun 0: <ATA, CT500MX500SSD1, M3CR> naa.500a0751e13ebe2f
sd0: 476940MB, 512 bytes/sector, 976773168 sectors, thin
piixpm0 at pci0 dev 20 function 0 "AMD FCH SMBus" rev 0x61: SMI
iic0 at piixpm0
spdmem0 at iic0 addr 0x50: 8GB DDR4 SDRAM PC4-19200 SO-DIMM
spdmem1 at iic0 addr 0x52: 8GB DDR4 SDRAM PC4-19200 SO-DIMM
iic1 at piixpm0
pcib0 at pci0 dev 20 function 3 "AMD FCH LPC" rev 0x51
pchb2 at pci0 dev 24 function 0 "AMD 17h/1xh Data Fabric" rev 0x00
pchb3 at pci0 dev 24 function 1 "AMD 17h/1xh Data Fabric" rev 0x00
pchb4 at pci0 dev 24 function 2 "AMD 17h/1xh Data Fabric" rev 0x00
pchb5 at pci0 dev 24 function 3 "AMD 17h/1xh Data Fabric" rev 0x00
pchb6 at pci0 dev 24 function 4 "AMD 17h/1xh Data Fabric" rev 0x00
pchb7 at pci0 dev 24 function 5 "AMD 17h/1xh Data Fabric" rev 0x00
pchb8 at pci0 dev 24 function 6 "AMD 17h/1xh Data Fabric" rev 0x00
pchb9 at pci0 dev 24 function 7 "AMD 17h/1xh Data Fabric" rev 0x00
isa0 at pcib0
isadma0 at isa0
pckbc0 at isa0 port 0x60/5 irq 1 irq 12
pckbd0 at pckbc0 (kbd slot)
wskbd0 at pckbd0: console keyboard
pms0 at pckbc0 (aux slot)
wsmouse0 at pms0 mux 0
wsmouse1 at pms0 mux 0
pms0: Synaptics clickpad, firmware 8.16, 0x1e2b1 0x940300 0x373740 0xf020a3
0x12e800
pcppi0 at isa0 port 0x61
spkr0 at pcppi0
vmm0 at mainbus0: SVM/RVI
efifb at mainbus0 not configured
uaudio0 at uhub0 port 3 configuration 1 interface 1 "Plantronics Plantronics
Headset" rev 1.10/0.04 addr 2
uaudio0: class v1, full-speed, sync, channels: 2 play, 2 rec, 9 ctls
audio1 at uaudio0
uhidev0 at uhub0 port 3 configuration 1 interface 3 "Plantronics Plantronics
Headset" rev 1.10/0.04 addr 2
uhidev0: iclass 3/0
uhid0 at uhidev0: input=2, output=0, feature=0
uvideo0 at uhub1 port 2 configuration 1 interface 0 "Azurewave Integrated
Camera" rev 2.01/17.11 addr 2
video0 at uvideo0
vscsi0 at root
scsibus2 at vscsi0: 256 targets
softraid0 at root
scsibus3 at softraid0: 256 targets
root on sd0a (3a85b6926424009b.a) swap on sd0b dump on sd0b
iwm0: hw rev 0x230, fw ver 34.0.1, address a0:51:0b:ed:56:de
amdgpu0: RAVEN 8 CU rev 0x01
amdgpu0: 1920x1080, 32bpp
wsdisplay0 at amdgpu0 mux 1: console (std, vt100 emulation), using wskbd0
wsdisplay0: screen 1-5 added (std, vt100 emulation)
