Do not hybrid compile and link source code by using VC and GCC!

来源：互联网发布：linux svn 新建仓库编辑：程序博客网时间：2024/05/29 18:05

Do not hybrid compile and link sourcecode by using VC and GCC!

GCC owns cross-compiling abilities in Windows. Thenormal environment is MinGW or Cygwin. This article focuses on the MinGW, andothers are similar with it.

If comparing with the calling convention of VC, GCChas some gaps in the environment of MinGW/Cygwin. The program will be crashedif SSEx instructions are adopted in the source coded built by GCC, however theyare linked with VC. It is a common case, such as many open source codes are portedfrom Linux.

To ease to understand these serious risks, thearticle will set about an example. If readers follow it step by step, theresult will become obvious.

Preface:

1. foo_gcc.c/foo_gcc.h is compiled by GCC.

2. foo_vc_call_gcc.c is compiled by VC.

C Source codes:

1. foo_gcc.h/c

Header file

Source code

#ifndef _FOO_GCC_H

#define _FOO_GCC_H

#if defined( __cplusplus ) || defined( c_plusplus )

extern "C" {

#endif

#ifdef BUILD_DLL

#define GCC_API __declspec( dllexport )

#else

#define GCC_API __declspec( dllimport )

#endif

GCC_API int foo_gcc( int i );

#if defined( __cplusplus ) || defined( c_plusplus )

}

#endif

#include "foo_gcc.h"

GCC_API int foo_gcc( int i )

{

char unused[ 16 ] __attribute__ ((__aligned__ ( 16 )));

__asm__ __volatile__( "movdqa %%xmm0, %0/n/t" : "=m"(unused) );

return i * i;

}

2. foo_vc_call_gcc.c

#include "foo_gcc.h"

int foo_vc( int i )

{

__declspec( align( 16 ) ) char unused[ 16 ];

_asm

{

lea esi, unused

movdqa xmm0, xmmword ptr[ esi ]

}

i = foo_gcc( i );

return i * i;

}

int main( void )

{

foo_vc( 10 );

return 0;

}

Compile

A bash file aconvenient for this boring task, and please refer it herein.

#!/bin/sh

rm ./foo_gcc.o

rm ./foo_gcc.dll

rm ./foo_gcc.s

rm ./foo_gcc.lib

rm ./libfoo_gcc.a

rm ./foo_gcc.exp

rm ./foo_gcc.def

gcc -S ./foo_gcc.c

gcc -c -O3 -DBUILD_DLL ./foo_gcc.c

gcc -shared -o ./foo_gcc.dll ./foo_gcc.o -Wl,--out-implib,./libfoo_gcc.a

pexports.exe ./foo_gcc.dll > ./foo_gcc.def

lib /machine:i386 /def:foo_gcc.def /out:foo_gcc.lib

rm ./foo_vc_call_gcc.obj

rm ./foo_vc_call_gcc.asm

rm ./foo_vc_call_gcc.exe

cl -Fa -O2 ./foo_vc_call_gcc.c ./foo_gcc.lib

Reader can referto Howto use GCC to build DLL by DEF file in MinGW? and How togenerate DLL files by GCC in the MinGW? articles to build DLL by GCC inMinGW.

Execute

$ ./foo_vc_call_gcc.exe

What will behappened? I think that the program will be crashed in your system, right?!

Herein, you maybe wonder why?????? Please keep patience! It is an issue of the callingconvention. Before making it clear, the source codes of ASM must beinvestigated.

Root cause

1. foo_gcc.s

Foo_gcc.s

.file "foo_gcc.c"

.text

.globl _foo_gcc

.def _foo_gcc; .scl 2; .type 32; .endef

_foo_gcc:

pushl %ebp

movl %esp, %ebp

subl $24, %esp

/APP

# 7 "./foo_gcc.c" 1

movdqa %xmm0, -24(%ebp)

# 0 "" 2

/NO_APP

movl 8(%ebp), %eax

imull 8(%ebp), %eax

leave

ret

A. eax is used totransfer parameter. It is __stdcall calling function of DLL.

B. eax is used toreturn value.

C. GCC uses static strategy toalign stack if SSEx instructions are used. The AMS marked by red color showsthe rule(32bytes = 4bytes (returning address) + 4bytes (save ebp) + 24bytes).It hits that the esp must be aligned when invoking foo_gcc function.

2. foo_vc_call_gcc.asm

foo_icl_call_gcc.asm

; Listing generated by Microsoft (R) Optimizing Compiler Version 14.00.50727.42

TITLE c:/temp/risks/foo_vc_call_gcc.c

.686P

.XMM

include listing.inc

.model flat

INCLUDELIB LIBCMT

INCLUDELIB OLDNAMES

PUBLIC __$ArrayPad$

PUBLIC _foo_vc

EXTRN __imp__foo_gcc:PROC

EXTRN ___security_cookie:DWORD

EXTRN @__security_check_cookie@4:PROC

; Function compile flags: /Ogtpy

; COMDAT _foo_vc

_TEXT SEGMENT

_unused$ = -32 ; size = 16

__$ArrayPad$ = -4 ; size = 4

_i$ = 8 ; size = 4

_foo_vc PROC ; COMDAT

; File c:/temp/risks/foo_vc_call_gcc.c

; Line 4

push ebp

mov ebp, esp

and esp, -16 ; fffffff0H

sub esp, 44 ; 0000002cH

mov eax, DWORD PTR ___security_cookie

xor eax, esp

mov DWORD PTR __$ArrayPad$[esp+44], eax

push esi

; Line 9

lea esi, DWORD PTR _unused$[esp+48]

; Line 10

movdqa xmm0, XMMWORD PTR [esi]

; Line 12

mov eax, DWORD PTR _i$[ebp]

push eax

call DWORD PTR __imp__foo_gcc

; Line 14

imul eax, eax

; Line 15

mov ecx, DWORD PTR __$ArrayPad$[esp+52]

add esp, 4

pop esi

xor ecx, esp

call @__security_check_cookie@4

mov esp, ebp

pop ebp

ret 0

_foo_vc ENDP

_TEXT ENDS

PUBLIC __$ArrayPad$

PUBLIC _main

; Function compile flags: /Ogtpy

; COMDAT _main

_TEXT SEGMENT

_unused$699 = -32 ; size = 16

__$ArrayPad$ = -4 ; size = 4

_main PROC ; COMDAT

; Line 18

push ebp

mov ebp, esp

and esp, -16 ; fffffff0H

sub esp, 44 ; 0000002cH

mov eax, DWORD PTR ___security_cookie

xor eax, esp

mov DWORD PTR __$ArrayPad$[esp+44], eax

push esi

; Line 19

lea esi, DWORD PTR _unused$699[esp+48]

movdqa xmm0, XMMWORD PTR [esi]

push 10 ; 0000000aH

call DWORD PTR __imp__foo_gcc

; Line 22

mov ecx, DWORD PTR __$ArrayPad$[esp+52]

add esp, 4

pop esi

xor ecx, esp

xor eax, eax

call @__security_check_cookie@4

mov esp, ebp

pop ebp

ret 0

_main ENDP

_TEXT ENDS

END

A. Stack is usedto transfer parameter.

B. eax is used toreturn value.

C. Dynamically align the address ofstack. “and esp, -16” equals “and esp, 0xFFFFFFF0”, and it makes address aligned by16bytes. However, when invoking foo_gcc function, the esp does not been aligned((44 + 4 + 4) % 16 = 4, they are marked by red color). The operations will leadto faults of SSEx instruction, and the symptom is that program is crashed inthe runtime.

Summarization:

1. The callingconvention among compilers has gaps, and it will lead to crash if SSExinstructions are adopted.

2. It is a commonissue, so all compilers compatible with VC have same results.

3. If thefunctions using SSEx instructions are not APIs of DLL, maybe the program can berun. In fact, it depends on the optimizing method of GCC. The risks are big!