diff --git a/src/host_memory.c b/src/host_memory.c index 648a8c66..64dffd23 100644 --- a/src/host_memory.c +++ b/src/host_memory.c @@ -6,24 +6,23 @@ #include #include #include +#include #include -#include -#include #include "host_memory.h" #include "host_validation.h" #include "perftest_parameters.h" #if !defined(__FreeBSD__) +#include #include #endif -#define SHMAT_ADDR (void *)(0x0UL) -#define SHMAT_FLAGS (0) -#define SHMAT_INVALID_PTR ((void *)-1) #define HUGEPAGE_SIZE_2MB (2ULL * 1024 * 1024) #define HUGEPAGE_SIZE_1GB (1ULL * 1024 * 1024 * 1024) +#define ALIGN_SIZE(size, align) (((size) + (align) - 1) & ~((align) - 1)) + /* MAP_HUGE_* flags for specifying huge page size (Linux 3.8+) */ #ifndef MAP_HUGE_2MB #define MAP_HUGE_2MB (21 << 26) /* 2^21 = 2MB */ @@ -32,29 +31,54 @@ #define MAP_HUGE_1GB (30 << 26) /* 2^30 = 1GB */ #endif +#ifndef MFD_HUGETLB +#define MFD_HUGETLB 0x0004U +#endif + +#ifndef MFD_HUGE_2MB +#define MFD_HUGE_2MB MAP_HUGE_2MB +#endif + #if !defined(__FreeBSD__) +static int create_hugepage_memfd(const char *name) +{ +#ifdef SYS_memfd_create + return syscall(SYS_memfd_create, name, MFD_HUGETLB | MFD_HUGE_2MB); +#else + errno = ENOSYS; + return -1; +#endif +} + int alloc_hugepage_region(int alignment, uint64_t size, void **addr) { - int huge_shmid; - uint64_t buf_size = (size + HUGEPAGE_SIZE_2MB - 1) & ~(HUGEPAGE_SIZE_2MB - 1); + int huge_fd; + uint64_t buf_size = ALIGN_SIZE(size, HUGEPAGE_SIZE_2MB); - huge_shmid = shmget(IPC_PRIVATE, buf_size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); - if (huge_shmid < 0) { - fprintf(stderr, "Failed to allocate hugepages. Please configure hugepages\n"); + (void)alignment; + + huge_fd = create_hugepage_memfd("perftest-hugepage"); + if (huge_fd < 0) { + fprintf(stderr, "Failed to create hugepage memfd (errno=%d: %s)\n", + errno, strerror(errno)); return FAILURE; } - *addr = (void *)shmat(huge_shmid, SHMAT_ADDR, SHMAT_FLAGS); - if (*addr == SHMAT_INVALID_PTR) { - fprintf(stderr, "Failed to attach shared memory region\n"); + if (ftruncate(huge_fd, buf_size)) { + fprintf(stderr, "Failed to allocate hugepages. Please configure hugepages (errno=%d: %s)\n", + errno, strerror(errno)); + close(huge_fd); return FAILURE; } - /* Mark for removal so shmem is freed when process detaches */ - if (shmctl(huge_shmid, IPC_RMID, 0) != 0) { - fprintf(stderr, "Failed to mark shm for removal\n"); + *addr = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_SHARED, huge_fd, 0); + if (*addr == MAP_FAILED) { + fprintf(stderr, "Failed to mmap hugepage memfd (errno=%d: %s)\n", + errno, strerror(errno)); + close(huge_fd); return FAILURE; } + close(huge_fd); return SUCCESS; } @@ -78,8 +102,7 @@ static int alloc_huge_pages_with_fallback(uint64_t size, void **addr, for (i = 0; i < 3; i++) { uint64_t aligned = opts[i].page_size - ? (size + opts[i].page_size - 1) & ~(opts[i].page_size - 1) - : size; + ? ALIGN_SIZE(size, opts[i].page_size) : size; void *ptr = mmap(NULL, aligned, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | opts[i].extra_flags, -1, 0); if (ptr != MAP_FAILED) { @@ -123,7 +146,7 @@ int host_memory_allocate_buffer(struct memory_ctx *ctx, int alignment, uint64_t host_ctx->alloc_size = size; #else /* Priority: data_validation -> auto huge pages, - * --use_hugepages -> legacy shmget, otherwise -> memalign */ + * --use_hugepages -> hugetlb memfd, otherwise -> memalign */ if (host_ctx->use_huge_for_validation) { if (alloc_huge_pages_with_fallback(size, addr, &host_ctx->alloc_type, &host_ctx->alloc_size, host_ctx->debug) != SUCCESS) { @@ -135,8 +158,8 @@ int host_memory_allocate_buffer(struct memory_ctx *ctx, int alignment, uint64_t fprintf(stderr, "Failed to allocate hugepage region.\n"); return FAILURE; } - host_ctx->alloc_type = HOST_ALLOC_SHMGET; - host_ctx->alloc_size = size; + host_ctx->alloc_type = HOST_ALLOC_MEMFD_HUGETLB; + host_ctx->alloc_size = ALIGN_SIZE(size, HUGEPAGE_SIZE_2MB); } else { *addr = memalign(alignment, size); host_ctx->alloc_type = HOST_ALLOC_MALLOC; @@ -157,9 +180,7 @@ int host_memory_free_buffer(struct memory_ctx *ctx, int dmabuf_fd, void *addr, u struct host_memory_ctx *host_ctx = container_of(ctx, struct host_memory_ctx, base); switch (host_ctx->alloc_type) { - case HOST_ALLOC_SHMGET: - shmdt(addr); - break; + case HOST_ALLOC_MEMFD_HUGETLB: case HOST_ALLOC_MMAP_REGULAR: case HOST_ALLOC_MMAP_HUGE_2MB: case HOST_ALLOC_MMAP_HUGE_1GB: diff --git a/src/host_memory.h b/src/host_memory.h index 40ce20c5..dd378179 100644 --- a/src/host_memory.h +++ b/src/host_memory.h @@ -15,7 +15,7 @@ struct host_validation_ctx; enum host_alloc_type { HOST_ALLOC_MALLOC, /* Standard malloc/memalign */ - HOST_ALLOC_SHMGET, /* Legacy shmget hugepages */ + HOST_ALLOC_MEMFD_HUGETLB, /* memfd hugetlb pages */ HOST_ALLOC_MMAP_REGULAR, /* mmap without huge pages */ HOST_ALLOC_MMAP_HUGE_2MB, /* mmap with 2MB huge pages */ HOST_ALLOC_MMAP_HUGE_1GB /* mmap with 1GB huge pages */ @@ -23,7 +23,7 @@ enum host_alloc_type { struct host_memory_ctx { struct memory_ctx base; - int use_hugepages; /* Legacy hugepages flag (--use_hugepages) */ + int use_hugepages; /* Hugepages flag (--use_hugepages) */ int use_huge_for_validation; /* Auto huge pages for data validation */ int debug; /* Print allocation diagnostics */ enum host_alloc_type alloc_type; /* How buffer was allocated */