Bug#552454: libgcrypt11: HMAC-SHA512 computation with 128-bytes key yields wrong result

[Stefan Westerfeld]

Package: libgcrypt11
Version: 1.4.1-1
Severity: normal

The HMAC-512 computation with a long key (128 bytes) gives a wrong result. This
is a known problem, a mail from the upstream developer can be found here:

http://www.nabble.com/Important-fix-for-HMAC-SHA-384-512-to20348035.html

I wrote a test program which computes:
 - HMAC-SHA512 with 128 bytes key via Debian/stable grcypt
 - HMAC-SHA512 implemented manually
 - HMAC-SHA512 implemented using python

Results:

* Debian/stable package:
ste...@quadcorn:/big/home/stefan/src/sandbox/fmc$ hmac512 abc
Key:  
KeyKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK
Data: abc
GCrypt: 
7933494498a4dadbe4e5d7a84d4aea568ea2e7da23f9c159ba609e19fa60e256a079f27c4cd2a7980701aac72243550900e66367af06110adf9dbcc93ec43fb4
MyHMAC: 
2c5b16e1d82fff0c6ef5754b97bcfad5324a5bc9515031b5f1ab1a25751db9bca44c28833a51ab108bc67b63803281fece57d596001941a717550082c6f9522c
Python: 
2c5b16e1d82fff0c6ef5754b97bcfad5324a5bc9515031b5f1ab1a25751db9bca44c28833a51ab108bc67b63803281fece57d596001941a717550082c6f9522c

* manually compiled gcrypt 1.4.4
ste...@quadcorn:/big/home/stefan/src/sandbox/fmc$ 
LD_LIBRARY_PATH=/usr/local/gcrypt/lib hmac512 abc
Key:  
KeyKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK
Data: abc
GCrypt: 
2c5b16e1d82fff0c6ef5754b97bcfad5324a5bc9515031b5f1ab1a25751db9bca44c28833a51ab108bc67b63803281fece57d596001941a717550082c6f9522c
MyHMAC: 
2c5b16e1d82fff0c6ef5754b97bcfad5324a5bc9515031b5f1ab1a25751db9bca44c28833a51ab108bc67b63803281fece57d596001941a717550082c6f9522c
Python: 
2c5b16e1d82fff0c6ef5754b97bcfad5324a5bc9515031b5f1ab1a25751db9bca44c28833a51ab108bc67b63803281fece57d596001941a717550082c6f9522c

As you see, the newer gcrypt 1.4.4 gives the correct result (the same result as
my routine and python), whereas the Debian/stable gcrypt HMAC-SHA512 is wrong.

-- System Information:
Debian Release: 5.0.3
  APT prefers stable
  APT policy: (500, 'stable')
Architecture: i386 (i686)

Kernel: Linux 2.6.26-2-686 (SMP w/4 CPU cores)
Locale: LANG=de_DE, LC_CTYPE=de_DE (charmap=ISO-8859-1)
Shell: /bin/sh linked to /bin/bash

Versions of packages libgcrypt11 depends on:
ii  libc6                         2.7-18     GNU C Library: Shared libraries
ii  libgpg-error0                 1.4-2      library for common error values an

libgcrypt11 recommends no packages.

Versions of packages libgcrypt11 suggests:
pn  rng-tools                     <none>     (no description available)

-- no debconf information

#include <gcrypt.h>
#include <stdio.h>
#include <string>

using std::string;

void
hmac_sha512_k128_my (const char *data, const char *key128, unsigned char *hmac)
{
  char ipad[128], opad[128];
  /* compute ipad */
  for (int i = 0; i < 128; i++)
    {
      ipad[i] = 0x36 ^ key128[i];
      opad[i] = 0x5c ^ key128[i];
    }

  gcry_control (GCRYCTL_DISABLE_SECMEM);
  gcry_md_hd_t ihd, ohd;

  /* inner sha512 hash */
  gcry_error_t error = gcry_md_open (&ihd, GCRY_MD_SHA512, 0);
  gcry_md_write (ihd, ipad, 128);
  gcry_md_write (ihd, data, strlen (data));
  unsigned char *ihash = gcry_md_read (ihd, 0);

  /* outer sha512 hash */
  gcry_error_t error1 = gcry_md_open (&ohd, GCRY_MD_SHA512, 0);
  gcry_md_write (ohd, opad, 128);
  gcry_md_write (ohd, ihash, 512 / 8);

  memcpy (hmac, gcry_md_read (ohd, 0), 512 / 8);
}

void
hmac_sha512_k128_gcrypt (const char *data, const char *key128, unsigned char *hmac)
{
  gcry_md_hd_t hd;
  gcry_error_t error = gcry_md_open (&hd, GCRY_MD_SHA512, GCRY_MD_FLAG_HMAC);
  gcry_md_setkey (hd, key128, 128);
  gcry_md_write (hd, data, strlen (data));

  memcpy (hmac, gcry_md_read (hd, 0), 512 / 8);
}


int
main (int argc, char **argv)
{
  if (argc != 2)
    {
      printf ("usage: hmac512 <data>\n");
      return 1;
    }

  const char *data = argv[1];
  const char *key  = "KeyKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKK";
  if (strlen (key) != 128)
    {
      printf ("need 128 byte key");
      return 1;
    }

  gcry_control(GCRYCTL_DISABLE_SECMEM);
  printf ("Key:  %s\nData: %s\n", key, data);

  unsigned char hmac[64];

  /* GCrypt HMAC implementation */
  hmac_sha512_k128_gcrypt (data, key, hmac);
  printf ("GCrypt: ");
  for (int i = 0; i < 64; i++)
    printf ("%02x", hmac[i]);
  printf ("\n");

  /* My HMAC implementation */
  hmac_sha512_k128_my (data, key, hmac);
  printf ("MyHMAC: ");
  for (int i = 0; i < 64; i++)
    printf ("%02x", hmac[i]);
  printf ("\n");

  /* Python HMAC implementation */
  string cmd;
  cmd += "python -c 'import hashlib,hmac;print \"Python: %s\" % hmac.new (\"";
  cmd += key;
  cmd += "\", \"";
  cmd += data;
  cmd += "\", hashlib.sha512).digest().encode (\"hex\")'";
  system (cmd.c_str());
}