[Bug c/89165] New: miscompile calling SSE function from non-SSE code

[nbkolchin at gmail dot com]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89165

            Bug ID: 89165
           Summary: miscompile calling SSE function from non-SSE code
           Product: gcc
           Version: 8.2.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: c
          Assignee: unassigned at gcc dot gnu.org
          Reporter: nbkolchin at gmail dot com
  Target Milestone: ---

BUG: calling SSE function with vector argument
from non-SSE code (admittedly, Bad Idea) silently
miscompiles (breaking calling convention).
I would appreciate an error message instead.

https://godbolt.org/z/pIpM95

GCC Version: 8.2

Command line arguments: -m32 -mno-sse -O2

Source code:

typedef float __attribute__((vector_size(16))) simd4f;

__attribute__((target("sse2")))
simd4f f(simd4f a) {return a*a;}

simd4f g(simd4f a,simd4f b) {return f(a)+f(b);}

Generated assembler:

f(float __vector(4)):
        mulps   %xmm0, %xmm0
        ret
g(float __vector(4), float __vector(4)):
        subl    $76, %esp
        movl    80(%esp), %eax
        leal    16(%esp), %edx
        pushl   108(%esp)
        pushl   108(%esp)
        pushl   108(%esp)
        pushl   108(%esp)
        subl    $12, %esp
        pushl   %edx
        call    f(float __vector(4))
        leal    60(%esp), %edx
        addl    $28, %esp
        pushl   124(%esp)
        pushl   124(%esp)
        pushl   124(%esp)
        pushl   124(%esp)
        subl    $12, %esp
        pushl   %edx
        call    f(float __vector(4))
        flds    44(%esp)
        fadds   60(%esp)
        fstps   28(%esp)
        flds    48(%esp)
        fadds   64(%esp)
        movl    28(%esp), %edx
        fstps   32(%esp)
        flds    52(%esp)
        fadds   68(%esp)
        fstps   36(%esp)
        flds    56(%esp)
        fadds   72(%esp)
        fstps   40(%esp)
        movl    %edx, (%eax)
        movl    32(%esp), %edx
        movl    %edx, 4(%eax)
        movl    36(%esp), %edx
        movl    %edx, 8(%eax)
        movl    40(%esp), %edx
        movl    %edx, 12(%eax)
        addl    $104, %esp
        ret     $4

P.S. Originally found here
https://gamedev.ru/code/forum/?id=233033&page=24#m356