Skip to content

Perftest: use memfd for hugepage host memory #396

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Hong-L666 wants to merge 1 commit into
base: master
Choose a base branch
from
+47 −26
Open

Perftest: use memfd for hugepage host memory #396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 45 additions & 24 deletions src/host_memory.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,23 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include "host_memory.h"
#include "host_validation.h"
#include "perftest_parameters.h"

#if !defined(__FreeBSD__)
#include <sys/syscall.h>
#include <sys/mman.h>
#endif

#define SHMAT_ADDR (void *)(0x0UL)
#define SHMAT_FLAGS (0)
#define SHMAT_INVALID_PTR ((void *)-1)

#define HUGEPAGE_SIZE_2MB (2ULL * 1024 * 1024)
#define HUGEPAGE_SIZE_1GB (1ULL * 1024 * 1024 * 1024)

#define ALIGN_SIZE(size, align) (((size) + (align) - 1) & ~((align) - 1))

/* MAP_HUGE_* flags for specifying huge page size (Linux 3.8+) */
#ifndef MAP_HUGE_2MB
#define MAP_HUGE_2MB (21 << 26) /* 2^21 = 2MB */
Expand All @@ -32,29 +31,54 @@
#define MAP_HUGE_1GB (30 << 26) /* 2^30 = 1GB */
#endif

#ifndef MFD_HUGETLB
#define MFD_HUGETLB 0x0004U
#endif

#ifndef MFD_HUGE_2MB
#define MFD_HUGE_2MB MAP_HUGE_2MB
#endif

#if !defined(__FreeBSD__)
static int create_hugepage_memfd(const char *name)
{
#ifdef SYS_memfd_create
return syscall(SYS_memfd_create, name, MFD_HUGETLB | MFD_HUGE_2MB);
#else
errno = ENOSYS;
return -1;
#endif
}

int alloc_hugepage_region(int alignment, uint64_t size, void **addr)
{
int huge_shmid;
uint64_t buf_size = (size + HUGEPAGE_SIZE_2MB - 1) & ~(HUGEPAGE_SIZE_2MB - 1);
int huge_fd;
uint64_t buf_size = ALIGN_SIZE(size, HUGEPAGE_SIZE_2MB);

huge_shmid = shmget(IPC_PRIVATE, buf_size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
if (huge_shmid < 0) {
fprintf(stderr, "Failed to allocate hugepages. Please configure hugepages\n");
(void)alignment;

huge_fd = create_hugepage_memfd("perftest-hugepage");
if (huge_fd < 0) {
fprintf(stderr, "Failed to create hugepage memfd (errno=%d: %s)\n",
errno, strerror(errno));
return FAILURE;
}

*addr = (void *)shmat(huge_shmid, SHMAT_ADDR, SHMAT_FLAGS);
if (*addr == SHMAT_INVALID_PTR) {
fprintf(stderr, "Failed to attach shared memory region\n");
if (ftruncate(huge_fd, buf_size)) {
fprintf(stderr, "Failed to allocate hugepages. Please configure hugepages (errno=%d: %s)\n",
errno, strerror(errno));
close(huge_fd);
return FAILURE;
}

/* Mark for removal so shmem is freed when process detaches */
if (shmctl(huge_shmid, IPC_RMID, 0) != 0) {
fprintf(stderr, "Failed to mark shm for removal\n");
*addr = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_SHARED, huge_fd, 0);
if (*addr == MAP_FAILED) {
fprintf(stderr, "Failed to mmap hugepage memfd (errno=%d: %s)\n",
errno, strerror(errno));
close(huge_fd);
return FAILURE;
}
close(huge_fd);

return SUCCESS;
}
Expand All @@ -78,8 +102,7 @@ static int alloc_huge_pages_with_fallback(uint64_t size, void **addr,

for (i = 0; i < 3; i++) {
uint64_t aligned = opts[i].page_size
? (size + opts[i].page_size - 1) & ~(opts[i].page_size - 1)
: size;
? ALIGN_SIZE(size, opts[i].page_size) : size;
void *ptr = mmap(NULL, aligned, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | opts[i].extra_flags, -1, 0);
if (ptr != MAP_FAILED) {
Expand Down Expand Up @@ -123,7 +146,7 @@ int host_memory_allocate_buffer(struct memory_ctx *ctx, int alignment, uint64_t
host_ctx->alloc_size = size;
#else
/* Priority: data_validation -> auto huge pages,
* --use_hugepages -> legacy shmget, otherwise -> memalign */
* --use_hugepages -> hugetlb memfd, otherwise -> memalign */
if (host_ctx->use_huge_for_validation) {
if (alloc_huge_pages_with_fallback(size, addr, &host_ctx->alloc_type,
&host_ctx->alloc_size, host_ctx->debug) != SUCCESS) {
Expand All @@ -135,8 +158,8 @@ int host_memory_allocate_buffer(struct memory_ctx *ctx, int alignment, uint64_t
fprintf(stderr, "Failed to allocate hugepage region.\n");
return FAILURE;
}
host_ctx->alloc_type = HOST_ALLOC_SHMGET;
host_ctx->alloc_size = size;
host_ctx->alloc_type = HOST_ALLOC_MEMFD_HUGETLB;
host_ctx->alloc_size = ALIGN_SIZE(size, HUGEPAGE_SIZE_2MB);
} else {
*addr = memalign(alignment, size);
host_ctx->alloc_type = HOST_ALLOC_MALLOC;
Expand All @@ -157,9 +180,7 @@ int host_memory_free_buffer(struct memory_ctx *ctx, int dmabuf_fd, void *addr, u
struct host_memory_ctx *host_ctx = container_of(ctx, struct host_memory_ctx, base);

switch (host_ctx->alloc_type) {
case HOST_ALLOC_SHMGET:
shmdt(addr);
break;
case HOST_ALLOC_MEMFD_HUGETLB:
case HOST_ALLOC_MMAP_REGULAR:
case HOST_ALLOC_MMAP_HUGE_2MB:
case HOST_ALLOC_MMAP_HUGE_1GB:
Expand Down
4 changes: 2 additions & 2 deletions src/host_memory.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ struct host_validation_ctx;

enum host_alloc_type {
HOST_ALLOC_MALLOC, /* Standard malloc/memalign */
HOST_ALLOC_SHMGET, /* Legacy shmget hugepages */
HOST_ALLOC_MEMFD_HUGETLB, /* memfd hugetlb pages */
HOST_ALLOC_MMAP_REGULAR, /* mmap without huge pages */
HOST_ALLOC_MMAP_HUGE_2MB, /* mmap with 2MB huge pages */
HOST_ALLOC_MMAP_HUGE_1GB /* mmap with 1GB huge pages */
};

struct host_memory_ctx {
struct memory_ctx base;
int use_hugepages; /* Legacy hugepages flag (--use_hugepages) */
int use_hugepages; /* Hugepages flag (--use_hugepages) */
int use_huge_for_validation; /* Auto huge pages for data validation */
int debug; /* Print allocation diagnostics */
enum host_alloc_type alloc_type; /* How buffer was allocated */
Expand Down