/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2009 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2007 Voltaire. All rights reserved. * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010-2015 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2012-2023 NVIDIA Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file */ #ifndef MCA_BTL_SMCUDA_H #define MCA_BTL_SMCUDA_H #include "opal_config.h" #include #include #include #include #ifdef HAVE_SCHED_H # include #endif /* HAVE_SCHED_H */ #include "opal/class/opal_free_list.h" #include "opal/mca/btl/btl.h" #include "opal/mca/common/sm/common_sm.h" #include "opal/util/bit_ops.h" BEGIN_C_DECLS /* * Shared Memory FIFOs * * The FIFO is implemented as a circular queue with head and tail pointers * (integer indices). For efficient wraparound indexing, the size of the * queue is constrained to be a power of two and we "&" indices with a "mask". * * More than one process can write to the FIFO head. Therefore, there is a head * lock. One cannot write until the head slot is empty, indicated by the special * queue entry SM_FIFO_FREE. * * Only the receiver can read the FIFO tail. Therefore, the tail lock is * required only in multithreaded applications. If a tail read returns the * SM_FIFO_FREE value, that means the FIFO is empty. Once a non-FREE value * has been read, the queue slot is *not* automatically reset to SM_FIFO_FREE. * Rather, read tail slots are reset "lazily" (see "lazy_free" and "num_to_clear") * to reduce the number of memory barriers and improve performance. * * Since the FIFO lives in shared memory that is mapped differently into * each address space, the "queue" pointer is relative (each process must * add its own offset) and the queue_recv pointer is meaningful only in the * receiver's address space. * * Since multiple processes access different parts of the FIFO structure in * different ways, we introduce padding to keep different parts on different * cachelines. */ #define SM_FIFO_FREE (void *) (-2) /* We can't use opal_cache_line_size here because we need a compile-time constant for padding the struct. We can't really have a compile-time constant that is portable, either (e.g., compile on one machine and run on another). So just use a big enough cache line that should hopefully be good in most places. */ #define SM_CACHE_LINE_PAD 128 struct sm_fifo_t { /* This queue pointer is used only by the heads. */ volatile void **queue; char pad0[SM_CACHE_LINE_PAD - sizeof(void **)]; /* This lock is used by the heads. */ opal_atomic_lock_t head_lock; char pad1[SM_CACHE_LINE_PAD - sizeof(opal_atomic_lock_t)]; /* This index is used by the head holding the head lock. */ volatile int head; char pad2[SM_CACHE_LINE_PAD - sizeof(int)]; /* This mask is used "read only" by all processes. */ unsigned int mask; char pad3[SM_CACHE_LINE_PAD - sizeof(int)]; /* The following are used only by the tail. */ volatile void **queue_recv; opal_atomic_lock_t tail_lock; volatile int tail; int num_to_clear; int lazy_free; char pad4[SM_CACHE_LINE_PAD - sizeof(void **) - sizeof(opal_atomic_lock_t) - sizeof(int) * 3]; }; typedef struct sm_fifo_t sm_fifo_t; /* * Shared Memory resource management */ #if OPAL_ENABLE_PROGRESS_THREADS == 1 # define DATA (char) 0 # define DONE (char) 1 #endif typedef struct mca_btl_smcuda_mem_node_t { mca_mpool_base_module_t *sm_mpool; /**< shared memory pool */ } mca_btl_smcuda_mem_node_t; /** * Shared Memory (SM) BTL module. */ struct mca_btl_smcuda_component_t { mca_btl_base_component_2_0_0_t super; /**< base BTL component */ int sm_free_list_num; /**< initial size of free lists */ int sm_free_list_max; /**< maximum size of free lists */ int sm_free_list_inc; /**< number of elements to alloc when growing free lists */ int sm_max_procs; /**< upper limit on the number of processes using the shared memory pool */ int sm_extra_procs; /**< number of extra procs to allow */ char *sm_mpool_name; /**< name of shared memory pool module */ mca_mpool_base_module_t **sm_mpools; /**< shared memory pools (one for each memory node) */ mca_mpool_base_module_t *sm_mpool; /**< mpool on local node */ void *sm_mpool_base; /**< base address of shared memory pool */ size_t eager_limit; /**< first fragment size */ size_t max_frag_size; /**< maximum (second and beyond) fragment size */ opal_mutex_t sm_lock; mca_common_sm_module_t *sm_seg; /**< description of shared memory segment */ volatile sm_fifo_t **shm_fifo; /**< pointer to fifo 2D array in shared memory */ char **shm_bases; /**< pointer to base pointers in shared memory */ uint16_t *shm_mem_nodes; /**< pointer to mem noded in shared memory */ sm_fifo_t **fifo; /**< cached copy of the pointer to the 2D fifo array. The address in the shared memory segment sm_ctl_header is a relative, but this one, in process private memory, is a real virtual address */ uint16_t *mem_nodes; /**< cached copy of mem nodes of each local rank */ unsigned int fifo_size; /**< number of FIFO queue entries */ unsigned int fifo_lazy_free; /**< number of reads before lazy fifo free is triggered */ int nfifos; /**< number of FIFOs per receiver */ int32_t num_smp_procs; /**< current number of smp procs on this host */ int32_t my_smp_rank; /**< My SMP process rank. Used for accessing * SMP specific data structures. */ opal_free_list_t sm_frags_eager; /**< free list of sm first */ opal_free_list_t sm_frags_max; /**< free list of sm second */ opal_free_list_t sm_frags_user; opal_free_list_t sm_first_frags_to_progress; /**< list of first fragments that are awaiting resources */ struct mca_btl_base_endpoint_t **sm_peers; opal_free_list_t pending_send_fl; opal_atomic_int32_t num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */ opal_atomic_int32_t num_pending_sends; /**< total number on all of my pending-send queues */ int mem_node; int num_mem_nodes; #if OPAL_ENABLE_PROGRESS_THREADS == 1 char sm_fifo_path[OPAL_PATH_MAX]; /**< path to fifo used to signal this process */ int sm_fifo_fd; /**< file descriptor corresponding to opened fifo */ opal_thread_t sm_fifo_thread; #endif struct mca_btl_smcuda_t **sm_btls; struct mca_btl_smcuda_frag_t **table; size_t sm_num_btls; size_t sm_max_btls; /** MCA: should we be using knem or not? neg=try but continue if not available, 0=don't try, 1=try and fail if not available */ int use_knem; /** MCA: minimal message size (bytes) to offload on DMA engine when using knem */ unsigned int knem_dma_min; /** MCA: how many simultaneous ongoing knem operations to support */ int knem_max_simultaneous; /** If we want DMA and DMA is supported, this will be loaded with KNEM_FLAG_DMA. Otherwise, it'll be 0. */ int knem_dma_flag; /** MCA: should we be using CMA or not? 0 = no, 1 = yes */ int use_cma; /* /// well-known file names for sm and sm mpool init /// */ char *sm_mpool_ctl_file_name; char *sm_mpool_rndv_file_name; char *sm_ctl_file_name; char *sm_rndv_file_name; int cuda_ipc_verbose; int cuda_ipc_output; int use_cuda_ipc; int use_cuda_ipc_same_gpu; unsigned long mpool_min_size; char *allocator; }; typedef struct mca_btl_smcuda_component_t mca_btl_smcuda_component_t; OPAL_DECLSPEC extern mca_btl_smcuda_component_t mca_btl_smcuda_component; /** * SM BTL Interface */ struct mca_btl_smcuda_t { mca_btl_base_module_t super; /**< base BTL interface */ bool btl_inited; /**< flag indicating if btl has been inited */ mca_btl_base_module_error_cb_fn_t error_cb; mca_rcache_base_module_t *rcache; }; typedef struct mca_btl_smcuda_t mca_btl_smcuda_t; OPAL_DECLSPEC extern mca_btl_smcuda_t mca_btl_smcuda; struct btl_smcuda_pending_send_item_t { opal_free_list_item_t super; void *data; }; typedef struct btl_smcuda_pending_send_item_t btl_smcuda_pending_send_item_t; /*** * FIFO support for sm BTL. */ /*** * One or more FIFO components may be a pointer that must be * accessed by multiple processes. Since the shared region may * be mmapped differently into each process's address space, * these pointers will be relative to some base address. Here, * we define macros to translate between relative addresses and * virtual addresses. */ #define VIRTUAL2RELATIVE(VADDR) \ ((long) (VADDR) \ - (long) mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank]) #define RELATIVE2VIRTUAL(OFFSET) \ ((long) (OFFSET) \ + (long) mca_btl_smcuda_component.shm_bases[mca_btl_smcuda_component.my_smp_rank]) static inline int sm_fifo_init(int fifo_size, mca_mpool_base_module_t *mpool, sm_fifo_t *fifo, int lazy_free) { int i, qsize; /* figure out the queue size (a power of two that is at least 1) */ qsize = opal_next_poweroftwo_inclusive(fifo_size); /* allocate the queue in the receiver's address space */ fifo->queue_recv = (volatile void **) mpool->mpool_alloc(mpool, sizeof(void *) * qsize, opal_cache_line_size, 0); if (NULL == fifo->queue_recv) { return OPAL_ERR_OUT_OF_RESOURCE; } /* initialize the queue */ for (i = 0; i < qsize; i++) fifo->queue_recv[i] = SM_FIFO_FREE; /* shift queue address to be relative */ fifo->queue = (volatile void **) VIRTUAL2RELATIVE(fifo->queue_recv); /* initialize the locks */ opal_atomic_lock_init(&(fifo->head_lock), OPAL_ATOMIC_LOCK_UNLOCKED); opal_atomic_lock_init(&(fifo->tail_lock), OPAL_ATOMIC_LOCK_UNLOCKED); opal_atomic_unlock(&(fifo->head_lock)); /* should be unnecessary */ opal_atomic_unlock(&(fifo->tail_lock)); /* should be unnecessary */ /* other initializations */ fifo->head = 0; fifo->mask = qsize - 1; fifo->tail = 0; fifo->num_to_clear = 0; fifo->lazy_free = lazy_free; return OPAL_SUCCESS; } static inline int sm_fifo_write(void *value, sm_fifo_t *fifo) { volatile void **q = (volatile void **) RELATIVE2VIRTUAL(fifo->queue); /* if there is no free slot to write, report exhausted resource */ opal_atomic_rmb(); if (SM_FIFO_FREE != q[fifo->head]) return OPAL_ERR_OUT_OF_RESOURCE; /* otherwise, write to the slot and advance the head index */ q[fifo->head] = value; opal_atomic_wmb(); fifo->head = (fifo->head + 1) & fifo->mask; return OPAL_SUCCESS; } static inline void *sm_fifo_read(sm_fifo_t *fifo) { void *value; /* read the next queue entry */ value = (void *) fifo->queue_recv[fifo->tail]; opal_atomic_rmb(); /* if you read a non-empty slot, advance the tail pointer */ if (SM_FIFO_FREE != value) { fifo->tail = (fifo->tail + 1) & fifo->mask; fifo->num_to_clear += 1; /* check if it's time to free slots, which we do lazily */ if (fifo->num_to_clear >= fifo->lazy_free) { int i = (fifo->tail - fifo->num_to_clear) & fifo->mask; while (fifo->num_to_clear > 0) { fifo->queue_recv[i] = SM_FIFO_FREE; i = (i + 1) & fifo->mask; fifo->num_to_clear -= 1; } opal_atomic_wmb(); } } return value; } /** * shared memory component progress. */ extern int mca_btl_smcuda_component_progress(void); /** * Register a callback function that is called on error.. * * @param btl (IN) BTL module * @return Status indicating if cleanup was successful */ int mca_btl_smcuda_register_error_cb(struct mca_btl_base_module_t *btl, mca_btl_base_module_error_cb_fn_t cbfunc); /** * Cleanup any resources held by the BTL. * * @param btl BTL instance. * @return OPAL_SUCCESS or error status on failure. */ extern int mca_btl_smcuda_finalize(struct mca_btl_base_module_t *btl); /** * PML->BTL notification of change in the process list. * PML->BTL Notification that a receive fragment has been matched. * Called for message that is send from process with the virtual * address of the shared memory segment being different than that of * the receiver. * * @param btl (IN) * @param proc (IN) * @param peer (OUT) * @return OPAL_SUCCESS or error status on failure. * */ extern int mca_btl_smcuda_add_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers, struct opal_bitmap_t *reachability); /** * PML->BTL notification of change in the process list. * * @param btl (IN) BTL instance * @param proc (IN) Peer process * @param peer (IN) Peer addressing information. * @return Status indicating if cleanup was successful * */ extern int mca_btl_smcuda_del_procs(struct mca_btl_base_module_t *btl, size_t nprocs, struct opal_proc_t **procs, struct mca_btl_base_endpoint_t **peers); /** * Allocate a segment. * * @param btl (IN) BTL module * @param size (IN) Request segment size. */ extern mca_btl_base_descriptor_t *mca_btl_smcuda_alloc(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, uint8_t order, size_t size, uint32_t flags); /** * Return a segment allocated by this BTL. * * @param btl (IN) BTL module * @param segment (IN) Allocated segment. */ extern int mca_btl_smcuda_free(struct mca_btl_base_module_t *btl, mca_btl_base_descriptor_t *segment); /** * Pack data * * @param btl (IN) BTL module * @param peer (IN) BTL peer addressing */ struct mca_btl_base_descriptor_t * mca_btl_smcuda_prepare_src(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct opal_convertor_t *convertor, uint8_t order, size_t reserve, size_t *size, uint32_t flags); /** * Initiate an inlined send to the peer or return a descriptor. * * @param btl (IN) BTL module * @param peer (IN) BTL peer addressing */ extern int mca_btl_smcuda_sendi(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct opal_convertor_t *convertor, void *header, size_t header_size, size_t payload_size, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t **descriptor); /** * Initiate a send to the peer. * * @param btl (IN) BTL module * @param peer (IN) BTL peer addressing */ extern int mca_btl_smcuda_send(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag); /** * Remote get using device memory. */ int mca_btl_smcuda_get_cuda(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *ep, void *local_address, uint64_t remote_address, struct mca_btl_base_registration_handle_t *local_handle, struct mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata); /* CUDA IPC control message tags */ enum ipcCtrlMsg { IPC_REQ = 10, IPC_ACK, IPC_NOTREADY, }; /* CUDA IPC control message */ typedef struct ctrlhdr_st { enum ipcCtrlMsg ctag; int cudev; } ctrlhdr_t; /* State of setting up CUDA IPC on an endpoint */ enum ipcState { IPC_INIT = 1, IPC_SENT, IPC_ACKING, IPC_ACKED, IPC_OK, IPC_BAD }; extern void mca_btl_smcuda_dump(struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, int verbose); #if OPAL_ENABLE_PROGRESS_THREADS == 1 void mca_btl_smcuda_component_event_thread(opal_object_t *); #endif #if OPAL_ENABLE_PROGRESS_THREADS == 1 # define MCA_BTL_SMCUDA_SIGNAL_PEER(peer) \ { \ unsigned char cmd = DATA; \ if (write(peer->fifo_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) { \ opal_output(0, "mca_btl_smcuda_send: write fifo failed: errno=%d\n", errno); \ } \ } #else # define MCA_BTL_SMCUDA_SIGNAL_PEER(peer) #endif END_C_DECLS #endif