ASM/NASM - Return high low of a MUL in a type struct

Question

global mymul
mymul:
    mov rax, rdi
    mul rsi
    ret

#include <stdio.h>

typedef struct {
        unsigned long long high;
            unsigned long long low;
} resmul;

void mymul(unsigned long long, unsigned long long, resmul *res);

int main() {
    resmul res;

    mymul(3, 6, &res);
    printf("mymul(3, 6); res.high=0x%llx, res.low=0x%llx\n", res.high, res.low);
    //mymul(3, 6); res.high=0x0, res.low=0x12
    return 0;
}

the goal is to multiply first arg with the second and send to result to the last arg first arg = RDI / second arg = RSI goal to send result high/low to typestruct

I dont understand why it gives 0 to both results RAX and RDX should be returned but i doesnt

Answer 1

Your mymul function is declared from C as taking a pointer argument, where it should be storing the results in memory. But in fact it is leaving them in the rdx:rax registers and never storing anything in memory at all, ignoring the pointer completely.

The third argument would be passed in the rdx register, which complicates things a little because mul overwrites it. So you have to do something like

global mymul
mymul:
    mov rcx, rdx  ; save argument
    mov rax, rdi
    mul rsi
    mov [rcx], rdx
    mov [rcx+8], rax
    ret

Answer 2

You told the compiler your function returns void, so the caller isn't looking at RDX:RAX.

Use a debugger and/or look at the asm in the caller; it will be loading its uninitialized local variable res from the stack, and that unwritten stack memory just happens to be zero.

---

To properly declare a function returning in RDX:RAX for the x86-64 SysV calling convention, either return a struct, or return unsigned __int128 if you're using GNU extensions.

#include <stdio.h>
#include <stdint.h>

typedef struct {
        uint64_t low;         // low first, the low half of RDX:RAX
        uint64_t high;
} resmul;              // x86-64 System V will return this in RDX:RAX, like __int128

resmul mymul(uint64_t, uint64_t);  // your function doesn't look for a pointer in RDX

int main() {
    resmul res = mymul(3, 6);
    printf("mymul(3, 6); res.high=%#lx, res.low=%#lx\n", res.high, res.low);
    //mymul(3, 6); res.high=0x0, res.low=0x12
    return 0;
}

Godbolt, also including GNU C implementations of your function that compile to the same asm:

// compiles to the same asm
resmul mymul(uint64_t a, uint64_t b){
    unsigned __int128 prod = a * (unsigned __int128)b;
    return (resmul){prod, prod>>64};
}

unsigned __int128 mulv2(uint64_t a, uint64_t b){
    return a * (unsigned __int128)b;
}

mymul(unsigned long, unsigned long):
        mov     rax, rdi
        mul     rsi
        ret
mulv2(unsigned long, unsigned long):
        mov     rax, rdi
        mul     rsi
        ret