← Back to blog

x86-64 Calling Convention

This is a personal study note — I write these to solidify my own understanding. If you spot anything wrong or have thoughts, feel free to reach out. Layout, formatting, and grammar assisted by Claude.


Passing arguments

Returning values

integer
vector
memory
1strax
2ndrdx
1stxmm0
2ndxmm1
caller's buffer, address inrdi
long
value
double
value
__int128
low
high
struct { long a; long b; }
a
b
struct { double a; double b; }
a
b
struct { long a; double b; }
a
b
struct { double a; long b; }
b
a
struct { long a; long b; long c; }
pointer
a
b
c

Structs

void f(struct S s);
eightbytes · class
integer
vector
memory
struct S
bytes0–7
bytes8–15
1strdi
2ndrsi
1stxmm0
2ndxmm1
a copy on thestack
{ long a; long b; }
a
INTEGER
b
INTEGER
a
b
{ double a; double b; }
a
SSE
b
SSE
a
b
{ long a; double b; }
a
INTEGER
b
SSE
a
b
{ int a; double b; }
a
INTEGER
b
SSE
a
b
{ int a; int b; int c; }
ab
INTEGER
c
INTEGER
ab
c
{ float a, b, c, d; }
ab
SSE
cd
SSE
ab
cd
{ int a; float b; }
ab
INTEGER
ab
{ long a; long b; long c; }
24 bytes: more than 16
a b c

The stack: alignment, red zone, variadic calls

long caller(void) { return callee(1, 2, 3, 4, 5, 6, 7, 8); } long callee(long a, long b, long c, long d, long e, long f, long g, long h) { long local[3] = { a, b, c }; return leaf(local, d) + e + g + h; } long leaf(long *p, long i) { long t[4] = { p[0], p[1], p[2], p[0] + p[1] }; return t[i & 3]; }
HIGHER ADDRESSES return address padding: rsp to 16n h, argument 8 [rsp+56] in callee g, argument 7 [rsp+48] return address [rsp+40] saved rbx [rsp+32] padding: rsp to 16n [rsp+24] local[2] [rsp+16] local[1] [rsp+8] local[0] [rsp] return address [rsp] in leaf padding: t aligns to 16 [rsp−8] t[3] [rsp−16] t[2] [rsp−24] t[1] [rsp−32] t[0] [rsp−40] ⋮ [rsp−128] LOWER ADDRESSES rsp before the call · 16n rsp on entry · 16n + 8 rsp in callee · 16n rsp in leaf · 16n + 8 red zone 128 bytes below rsp caller callee leaf
int printf(const char *fmt, ...);
long a;
double x, y;
rdi
rsi
xmm0
xmm1
vector registers usedal
printf("hi\n");
fmt
0
printf("%ld\n", a);
fmt
a
0
printf("%f\n", x);
fmt
x
1
printf("%ld %f %f\n", a, x, y);
fmt
a
x
y
2
typedef struct {
    unsigned int gp_offset;          // offset of the next integer argument
    unsigned int fp_offset;          // offset of the next floating-point argument
    void        *overflow_arg_area;  // next argument on the stack
    void        *reg_save_area;      // start of the register save area
} va_list[1];
// what va_arg(ap, long) does
if (ap->gp_offset <= 40) {
    value = *(long *)(ap->reg_save_area + ap->gp_offset);
    ap->gp_offset += 8;
} else {
    value = *(long *)ap->overflow_arg_area;
    ap->overflow_arg_area += 8;
}

// what va_arg(ap, double) does
if (ap->fp_offset <= 160) {
    value = *(double *)(ap->reg_save_area + ap->fp_offset);
    ap->fp_offset += 16;
} else {
    value = *(double *)ap->overflow_arg_area;
    ap->overflow_arg_area += 8;
}

registers · at the call

Who saves what

callee-savedcaller-saved
0–7
8–15 · REX
16–31 · APX · REX2 or EVEX
xmm0–15 · vector

APX: PUSH2, POP2 and PPX

APX adds pushp and popp. They do the same thing as push and pop, and also tell the processor that they come as a pair: the popp reads back the value its pushp wrote. The processor does not enforce the pair. The hint is only for performance, so a mismatched pair breaks nothing and only loses the potential speed-up.

long ext(long); long two_saved(long a, long b) { long r = ext(a); r += ext(b); return r + a + b; }
without APX
push r15 push r14 push rbx mov rbx, rsi ; b mov r14, rdi ; a call ext ; ext(a) mov r15, rax ; r mov rdi, rbx call ext ; ext(b) add rbx, r14 add rbx, r15 add rax, rbx pop rbx pop r14 pop r15 ret
with APX
pushp r15 push2p r14, rbx mov rbx, rsi ; b mov r14, rdi ; a call ext ; ext(a) mov r15, rax ; r mov rdi, rbx call ext ; ext(b) add rbx, r14 add rbx, r15 add rax, rbx pop2p rbx, r14 popp r15 ret
address return address 16n + 8 saved r15 16n saved r14 16n − 8 saved rbx 16n − 16 WITHOUT APX WITH APX push r15 push r14 push rbx pushp r15 push2p r14, rbx rsp must be a multiple of 16 3 instructions 2 instructions and back pop rbx · pop r14 · pop r15 3 instructions pop2p rbx, r14 · popp r15 2 instructions