x86-64 Calling Convention
This is a personal study note — I write these to solidify my own understanding. If you spot anything wrong or have thoughts, feel free to reach out. Layout, formatting, and grammar assisted by Claude.
Passing arguments
Returning values
integer
vector
memory
1strax
2ndrdx
1stxmm0
2ndxmm1
caller's buffer, address inrdi
long
value
double
value
__int128
low
high
struct { long a; long b; }
a
b
struct { double a; double b; }
a
b
struct { long a; double b; }
a
b
struct { double a; long b; }
b
a
struct { long a; long b; long c; }
pointer
a
b
c
Structs
void f(struct S s);
eightbytes · class
integer
vector
memory
struct S
bytes0–7
bytes8–15
1strdi
2ndrsi
1stxmm0
2ndxmm1
a copy on thestack
{ long a; long b; }
a
INTEGERb
INTEGERa
b
{ double a; double b; }
a
SSEb
SSEa
b
{ long a; double b; }
a
INTEGERb
SSEa
b
{ int a; double b; }
a
INTEGERb
SSEa
b
{ int a; int b; int c; }
ab
INTEGERc
INTEGERab
c
{ float a, b, c, d; }
ab
SSEcd
SSEab
cd
{ int a; float b; }
ab
INTEGERab
{ long a; long b; long c; }
24 bytes: more than 16
a b c
The stack: alignment, red zone, variadic calls
long caller(void) {
return callee(1, 2, 3, 4, 5, 6, 7, 8);
}
long callee(long a, long b, long c, long d, long e, long f, long g, long h) {
long local[3] = { a, b, c };
return leaf(local, d) + e + g + h;
}
long leaf(long *p, long i) {
long t[4] = { p[0], p[1], p[2], p[0] + p[1] };
return t[i & 3];
}
int printf(const char *fmt, ...);
long a;
double x, y;
long a;
double x, y;
rdi
rsi
xmm0
xmm1
vector registers usedal
printf("hi\n");
fmt
0
printf("%ld\n", a);
fmt
a
0
printf("%f\n", x);
fmt
x
1
printf("%ld %f %f\n", a, x, y);
fmt
a
x
y
2
typedef struct {
unsigned int gp_offset; // offset of the next integer argument
unsigned int fp_offset; // offset of the next floating-point argument
void *overflow_arg_area; // next argument on the stack
void *reg_save_area; // start of the register save area
} va_list[1];
// what va_arg(ap, long) does
if (ap->gp_offset <= 40) {
value = *(long *)(ap->reg_save_area + ap->gp_offset);
ap->gp_offset += 8;
} else {
value = *(long *)ap->overflow_arg_area;
ap->overflow_arg_area += 8;
}
// what va_arg(ap, double) does
if (ap->fp_offset <= 160) {
value = *(double *)(ap->reg_save_area + ap->fp_offset);
ap->fp_offset += 16;
} else {
value = *(double *)ap->overflow_arg_area;
ap->overflow_arg_area += 8;
}
registers · at the call
Who saves what
callee-savedcaller-saved
0–7
8–15 · REX
16–31 · APX · REX2 or EVEX
xmm0–15 · vector
APX: PUSH2, POP2 and PPX
APX adds pushp and popp. They do the same thing as push and pop, and also tell the processor that they come as a pair: the popp reads back the value its pushp wrote. The processor does not enforce the pair. The hint is only for performance, so a mismatched pair breaks nothing and only loses the potential speed-up.
long ext(long);
long two_saved(long a, long b) {
long r = ext(a);
r += ext(b);
return r + a + b;
}
without APX
push r15
push r14
push rbx
mov rbx, rsi ; b
mov r14, rdi ; a
call ext ; ext(a)
mov r15, rax ; r
mov rdi, rbx
call ext ; ext(b)
add rbx, r14
add rbx, r15
add rax, rbx
pop rbx
pop r14
pop r15
ret
with APX
pushp r15
push2p r14, rbx
mov rbx, rsi ; b
mov r14, rdi ; a
call ext ; ext(a)
mov r15, rax ; r
mov rdi, rbx
call ext ; ext(b)
add rbx, r14
add rbx, r15
add rax, rbx
pop2p rbx, r14
popp r15
ret