[Bug target/116086] RISC-V: Hash mismatch with vectorized 557.xz_r at zvl128b and LMUL=m2

[rdapp at gcc dot gnu.org via Gcc-bugs]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116086

--- Comment #1 from Robin Dapp <rdapp at gcc dot gnu.org> ---
The following reproduces the problem for me, though not very minimal yet:

typedef unsigned int uint32_t;
typedef unsigned long long uint64_t;

typedef struct
{
    uint64_t length;
    uint64_t state[8];
    uint32_t curlen;
    unsigned char buf[128];
} sha512_state;

typedef uint32_t u32;
typedef uint64_t u64;

static u64 load64(const unsigned char* y)
{
    u64 res = 0;
    for(int i = 0; i != 8; ++i)
        res |= (u64)(y[i]) << ((7-i) * 8);
    return res;
}

static u64 Ch(u64 x, u64 y, u64 z)  { return z ^ (x & (y ^ z)); }
static u64 Maj(u64 x, u64 y, u64 z) { return ((x | y) & z) | (x & y); }
static u64 Rot(u64 x, u64 n)        { return (x >> (n & 63)) | (x << (64 - (n &
63))); }
static u64 Sh(u64 x, u64 n)         { return x >> n; }
static u64 Sigma0(u64 x)            { return Rot(x, 28) ^ Rot(x, 34) ^ Rot(x,
39); }
static u64 Sigma1(u64 x)            { return Rot(x, 14) ^ Rot(x, 18) ^ Rot(x,
41); }
static u64 Gamma0(u64 x)            { return Rot(x, 1) ^ Rot(x, 8) ^ Sh(x, 7);
}
static u64 Gamma1(u64 x)            { return Rot(x, 19) ^ Rot(x, 61) ^ Sh(x,
6); }

static const u64 K[80] =
{
    0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL,
0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL,
    0x59f111f1b605d019ULL, 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL,
0xd807aa98a3030242ULL, 0x12835b0145706fbeULL,
    0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, 0x72be5d74f27b896fULL,
0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL,
    0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL,
0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL,
    0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL,
0x76f988da831153b5ULL, 0x983e5152ee66dfabULL,
    0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL,
0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL,
    0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL,
0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL,
    0x53380d139d95b3dfULL, 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL,
0x81c2c92e47edaee6ULL, 0x92722c851482353bULL,
    0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, 0xc24b8b70d0f89791ULL,
0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL,
    0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL,
0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL,
    0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL,
0x4ed8aa4ae3418acbULL, 0x5b9cca4f7763e373ULL,
    0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL,
0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL,
    0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL,
0xc67178f2e372532bULL, 0xca273eceea26619cULL,
    0xd186b8c721c0c207ULL, 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL,
0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL,
    0x113f9804bef90daeULL, 0x1b710b35131c471bULL, 0x28db77f523047d84ULL,
0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL,
    0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL
};

// Compress
#define RND(a,b,c,d,e,f,g,h,i)                    \
{                                                 \
  t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \
  t1 = Sigma0(a) + Maj(a, b, c);                  \
  d += t0;                                        \
  h  = t0 + t1;                                   \
}

__attribute__ ((noipa))
static void sha_compress(sha512_state *md, const unsigned char *buf)
{
    u64 S[8], W[80], t0, t1;

    // Copy state into S
    for(int i = 0; i < 8; i++)
        S[i] = md->state[i];

    // Copy the state into 1024-bits into W[0..15]
    for(int i = 0; i < 16; i++)
        W[i] = load64(buf + (8*i));

    // Fill W[16..79]
    for(int i = 16; i < 80; i++)
        W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16];

    for(int i = 0; i < 80; i += 8)
    {
        RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i+0);
        RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],i+1);
        RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],i+2);
        RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],i+3);
        RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],i+4);
        RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],i+5);
        RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],i+6);
        RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],i+7);
    }

     // Feedback
     for(int i = 0; i < 8; i++)
         md->state[i] = md->state[i] + S[i];
}

int main ()
{
  sha512_state md;
  md.curlen = 0;
  md.length = 0;
  md.state[0] = 0x6a09e667f3bcc908ULL;
  md.state[1] = 0xbb67ae8584caa73bULL;
  md.state[2] = 0x3c6ef372fe94f82bULL;
  md.state[3] = 0xa54ff53a5f1d36f1ULL;
  md.state[4] = 0x510e527fade682d1ULL;
  md.state[5] = 0x9b05688c2b3e6c1fULL;
  md.state[6] = 0x1f83d9abfb41bd6bULL;
  md.state[7] = 0x5be0cd19137e2179ULL;

  for (int i = 0; i < 128; i++)
    md.buf[i] = 0;

  md.buf[md.curlen++] = (unsigned char)0x80;

  sha_compress (&md, md.buf);

  sha512_state ref;
  ref.state[0] = 0xcf83e1357eefb8bdULL;
  ref.state[1] = 0xf1542850d66d8007ULL;
  ref.state[2] = 0xd620e4050b5715dcULL;
  ref.state[3] = 0x83f4a921d36ce9ceULL;
  ref.state[4] = 0x47d0d13c5d85f2b0ULL;
  ref.state[5] = 0xff8318d2877eec2fULL;
  ref.state[6] = 0x63b931bd47417a81ULL;
  ref.state[7] = 0xa538327af927da3eULL;

  for(int i = 0; i < 8; i++)
    if (md.state[i] != ref.state[i])
      __builtin_abort ();
}