[patch i386]: Fix PR target/56186

[Kai Tietz]

Hello,

this patch fixes reported regression about 128-bit return-type for x64 ABI.

ChangeLog 2013-02-04  Kai Tietz  <kti...@redhat.com>

        PR target/56186
        * i386.c (function_value_64): Add additional valtype argument
        and improve checking of return-argument types for 16-byte modes.
        (ix86_function_value_1): Add additional valtype argument on call
        of function_value_64.
        (return_in_memory_ms_64): Sync 16-byte sized mode handling with
        handling infunction_value_64 function.

Tested for x86_64-w64-mingw32, and for x86_64-unknown-linux-gnu.  Ok for apply?

Regards,
Kai

Index: i386.c
===================================================================
--- i386.c      (Revision 195572)
+++ i386.c      (Arbeitskopie)
@@ -7324,7 +7324,8 @@ function_value_64 (enum machine_mode orig_mode, en
 }

 static rtx
-function_value_ms_64 (enum machine_mode orig_mode, enum machine_mode mode)
+function_value_ms_64 (enum machine_mode orig_mode, enum machine_mode mode,
+                     const_tree valtype)
 {
   unsigned int regno = AX_REG;

@@ -7333,6 +7334,12 @@ static rtx
       switch (GET_MODE_SIZE (mode))
         {
         case 16:
+         if (valtype != NULL_TREE
+             && !VECTOR_INTEGER_TYPE_P (valtype)
+              && !VECTOR_INTEGER_TYPE_P (valtype)
+             && !INTEGRAL_TYPE_P (valtype)
+             && !VECTOR_FLOAT_TYPE_P (valtype))
+           break;
           if((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
             && !COMPLEX_MODE_P (mode))
            regno = FIRST_SSE_REG;
@@ -7361,7 +7368,7 @@ ix86_function_value_1 (const_tree valtype, const_t
   fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;

   if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI)
-    return function_value_ms_64 (orig_mode, mode);
+    return function_value_ms_64 (orig_mode, mode, valtype);
   else if (TARGET_64BIT)
     return function_value_64 (orig_mode, mode, valtype);
   else
@@ -7474,7 +7481,9 @@ return_in_memory_ms_64 (const_tree type, enum mach
   HOST_WIDE_INT size = int_size_in_bytes (type);

   /* __m128 is returned in xmm0.  */
-  if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
+  if ((!type || VECTOR_INTEGER_TYPE_P (type) || INTEGRAL_TYPE_P (type)
+       || VECTOR_FLOAT_TYPE_P (type))
+      && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
       && !COMPLEX_MODE_P (mode) && (GET_MODE_SIZE (mode) == 16 || size == 16))
     return false;