/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2013 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef MCA_SPML_H #define MCA_SPML_H #include "oshmem_config.h" #include "oshmem/types.h" #include "oshmem/constants.h" #include "opal_stdint.h" #include "oshmem/mca/mca.h" #include "opal/mca/btl/btl.h" #include "oshmem/proc/proc.h" #include "oshmem/mca/sshmem/sshmem.h" BEGIN_C_DECLS /* * SPML component types */ /** * MCA->PML Called by MCA framework to initialize the component. * * @param priority (OUT) Relative priority or ranking used by MCA to * selected a component. * * @param enable_progress_threads (IN) Whether this component is * allowed to run a hidden/progress thread or not. * * @param enable_mpi_threads (IN) Whether support for multiple MPI * threads is enabled or not (i.e., MPI_THREAD_MULTIPLE), which * indicates whether multiple threads may invoke this component * simultaneously or not. */ typedef enum { MCA_SPML_BASE_PUT_SYNCHRONOUS, MCA_SPML_BASE_PUT_COMPLETE, MCA_SPML_BASE_PUT_BUFFERED, MCA_SPML_BASE_PUT_READY, MCA_SPML_BASE_PUT_STANDARD, MCA_SPML_BASE_PUT_SIZE } mca_spml_base_put_mode_t; typedef struct mca_spml_base_module_1_0_0_t * (*mca_spml_base_component_init_fn_t)(int *priority, bool enable_progress_threads, bool enable_mpi_threads); typedef int (*mca_spml_base_component_finalize_fn_t)(void); /** * SPML component version and interface functions. */ struct mca_spml_base_component_2_0_0_t { mca_base_component_t spmlm_version; mca_base_component_data_t spmlm_data; mca_spml_base_component_init_fn_t spmlm_init; mca_spml_base_component_finalize_fn_t spmlm_finalize; }; typedef struct mca_spml_base_component_2_0_0_t mca_spml_base_component_2_0_0_t; typedef mca_spml_base_component_2_0_0_t mca_spml_base_component_t; /** * MCA management functions. */ static inline char *mca_spml_base_mkey2str(sshmem_mkey_t *mkey) { static char buf[64]; if (mkey->len == 0) { snprintf(buf, sizeof(buf), "mkey: base=%p len=%d key=%" PRIu64, mkey->va_base, mkey->len, mkey->u.key); } else { snprintf(buf, sizeof(buf), "mkey: base=%p len=%d data=0x%p", mkey->va_base, mkey->len, mkey->u.data); } return buf; } /** * Downcall from MCA layer to enable the PML/BTLs. * * @param enable Enable/Disable SPML forwarding * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_enable_fn_t)(bool enable); /** * Waits for an int variable to change on the local PE. * Blocked until the variable is not equal to value. * * @param addr Address of the variable to pool on. * @param value The value to pool on. Pool until the value held in addr is different than value. * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_wait_fn_t)(void* addr, int cmp, void* value, int datatype); /** * Test for an int variable to change on the local PE. * * @param addr Address of the variable to pool on. * @param value The value to pool on. Pool until the value held in addr is different than value. * @param out_value Return value to indicated if variable is equal to given cmp value. * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_test_fn_t)(void* addr, int cmp, void* value, int datatype, int *out_value); /** * deserialize remote mkey * * @param mkey remote mkey */ typedef void (*mca_spml_base_module_mkey_unpack_fn_t)(shmem_ctx_t ctx, sshmem_mkey_t *, uint32_t segno, int remote_pe, int tr_id); /** * If possible, get a pointer to the remote memory described by the mkey * * @param dst_addr address of the symmetric variable * @param mkey remote memory key * @param pe remote PE * * @return pointer to remote memory or NULL */ typedef void * (*mca_spml_base_module_mkey_ptr_fn_t)(const void *dst_addr, sshmem_mkey_t *mkey, int pe); /** * free resources used by deserialized remote mkey * * @param mkey remote mkey */ typedef void (*mca_spml_base_module_mkey_free_fn_t)(sshmem_mkey_t *, int pe); /** * Register (Pinn) a buffer of 'size' bits starting in address addr * * @param addr base address of the registered buffer. * @param size the size of the buffer to be registered. * @param seg_id sysv segment id * @param count number of internal transports (btls) that registered memory * @return array of mkeys (one mkey per "btl") or NULL on failure * */ typedef sshmem_mkey_t * (*mca_spml_base_module_register_fn_t)(void *addr, size_t size, uint64_t shmid, int *count); /** * deregister memory pinned by register() */ typedef int (*mca_spml_base_module_deregister_fn_t)(sshmem_mkey_t *mkeys); /** * try to fill up mkeys that can be used to reach remote pe. * @param pe remote pe * @param seg 0 - symmetric heap, 1 - static data, everything else are static data in .so * @param mkeys mkeys array * * @return OSHMEM_SUCCSESS if keys are found */ typedef int (*mca_spml_base_module_oob_get_mkeys_fn_t)(shmem_ctx_t ctx, int pe, uint32_t seg, sshmem_mkey_t *mkeys); /** * For each proc setup a datastructure that indicates the BTLs * that can be used to reach the destination. * * @param procs A list of all procs participating in the parallel application. * @param nprocs The number of procs in the parallel application. * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_add_procs_fn_t)(struct oshmem_group_t* group, size_t nprocs); typedef int (*mca_spml_base_module_del_procs_fn_t)(struct oshmem_group_t* group, size_t nprocs); /** * Create a communication context. * * @param options The set of options requested for the given context. * @param ctx A handle to the newly created context. * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_ctx_create_fn_t)(long options, shmem_ctx_t *ctx); /** * Destroy a communication context. * * @param ctx Handle to the context that will be destroyed. */ typedef void (*mca_spml_base_module_ctx_destroy_fn_t)(shmem_ctx_t ctx); /** * Transfer data to a remote pe. * * @param ctx The context object this routine is working on. * @param dst_addr The address in the remote PE of the object being written. * @param size The number of bytes to be written. * @param src_addr An address on the local PE holdng the value to be written. * @param dst The remote PE to be written to. * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_put_fn_t)(shmem_ctx_t ctx, void *dst_addr, size_t size, void *src_addr, int dst); /** * These routines provide the means for copying contiguous data to another PE without * blocking the caller. These routines return before the data has been delivered to the * remote PE. * * @param ctx The context object this routine is working on. * @param dst_addr The address in the remote PE of the object being written. * @param size The number of bytes to be written. * @param src_addr An address on the local PE holdng the value to be written. * @param dst The remote PE to be written to. * @param handle The address of a handle to be passed to shmem_wait_nb() or * shmem_test_nb() to wait or poll for the completion of the transfer. * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_put_nb_fn_t)(shmem_ctx_t ctx, void *dst_addr, size_t size, void *src_addr, int dst, void **handle); /** * The put-with-signal routines provide a method for copying data from a * contiguous local data object to a data object on a specified PE and * subsequently updating a remote flag to signal completion. * * @param ctx A context handle specifying the context on which to perform the * operation. When this argument is not provided, the operation is * performed on the default context. * @param dst_addr The address in the remote PE of the object being written. * @param size The number of bytes to be written. * @param src_addr An address on the local PE holdng the value to be written. * @param sig_addr Symmetric address of the signal data object to be updated on the * remote PE as a signal. * @param signal Unsigned 64-bit value that is used for updating the remote sig_addr * signal data object. * @param sig_op Signal operator that represents the type of update to be performed * on the remote sig_addr signal data object. * @param pe PE number of the remote PE. * * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_put_signal_fn_t)(shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_addr, uint64_t *sig_addr, uint64_t signal, int sig_op, int dst); /** * The nonblocking put-with-signal routines provide a method for copying data * from a contiguous local data object to a data object on a specified PE and * subsequently updating a remote flag to signal completion. * * @param ctx A context handle specifying the context on which to perform the * operation. When this argument is not provided, the operation is * performed on the default context. * @param dst_addr The address in the remote PE of the object being written. * @param size The number of bytes to be written. * @param src_addr An address on the local PE holdng the value to be written. * @param sig_addr Symmetric address of the signal data object to be updated on the * remote PE as a signal. * @param signal Unsigned 64-bit value that is used for updating the remote sig_addr * signal data object. * @param sig_op Signal operator that represents the type of update to be performed * on the remote sig_addr signal data object. * @param pe PE number of the remote PE. * * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_put_signal_nb_fn_t) (shmem_ctx_t ctx, void* dst_addr, size_t size, void* src_addr, uint64_t *sig_addr, uint64_t signal, int sig_op, int dst); /* * Wait on an array of variables on the local PE until all variables * meet the specified wait condition. * * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_value The value to be compared with the objects pointed to by ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the wait set. * @param datatype Type of the objects * * @return None */ typedef void(*mca_spml_base_module_wait_until_all_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, const int *status, int datatype); /* * Wait on an array of variables on the local PE until any one variable * meets the specified wait condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_value The value to be compared with the objects pointed to by ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the wait set. * @param datatype Type of the objects * * @return Returns the index of an element in the ivars array that satisfies the * wait condition. If the wait set is empty, this routine returns SIZE_MAX. */ typedef size_t (*mca_spml_base_module_wait_until_any_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, const int *status, int datatype); /* * Wait on an array of variables on the local PE until at least one variable * meets the specified wait condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_value The value to be compared with the objects pointed to by ivars. * @param nelems The number of elements in the ivars array. * @param indices Local address of an array of indices of length at least nelems into * ivars that satisfied the wait condition. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the wait set. * @param datatype Type of the objects * * @return Returns the number of indices returned in the indices array. If the wait * set is empty, this routine returns 0. */ typedef size_t (*mca_spml_base_module_wait_until_some_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, size_t *indices, const int *status, int datatype); /* * Wait on an array of variables on the local PE until all variables meet the * specified wait conditions. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with elements * of cmp_values. * @param cmp_values Local address of an array of length nelems containing values to be * compared with the respective objects in ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the wait set. * @param datatype Type of the objects * * @return None * */ typedef void (*mca_spml_base_module_wait_until_all_vector_fn_t)(void *ivars, int cmp, void *cmp_values, size_t nelems, const int *status, int datatype); /* * Wait on an array of variables on the local PE until any one variable * meets the specified wait condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_value The value to be compared with the objects pointed to by ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the wait set. * @param datatype Type of the objects * * @return Returns the index of an element in the ivars array that satisfies the * test condition. If the test set is empty or no conditions in the test * set are satisfied, this routine returns SIZE_MAX. */ typedef size_t (*mca_spml_base_module_wait_until_any_vector_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, const int *status, int datatype); /* * Wait on an array of variables on the local PE until at least one variable meets the * its specified wait condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with elements * of cmp_values. * @param cmp_values Local address of an array of length nelems containing values to be * compared with the respective objects in ivars. * @param nelems The number of elements in the ivars array. * @param indices Local address of an array of indices of length at least nelems into ivars * that satisfied the wait condition. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the wait set. * @param datatype Type of the objects * * @return Returns the number of indices returned in the indices array. If the test * set is empty, this routine returns 0. */ typedef size_t (*mca_spml_base_module_wait_until_some_vector_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, size_t *indices, const int *status, int datatype); /* * Indicate whether all variables within an array of variables on the local PE meet * a specified test condition. * * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_value The value to be compared with the objects pointed to by ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the test set. * @param datatype Type of the objects * * @return Returns 1 if all variables in ivars satisfy the test condition or if * nelems is 0, otherwise this routine returns 0. */ typedef int (*mca_spml_base_module_test_all_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, const int *status, int datatype); /* * Indicate whether any one variable within an array of variables on the local PE meets * a specified test condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_value The value to be compared with the objects pointed to by ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the test set. * @param datatype Type of the objects * * @return Returns the index of an element in the ivars array that satisfies the * test condition. If the test set is empty or no conditions in the test * set are satisfied, this routine returns SIZE_MAX.. */ typedef size_t (*mca_spml_base_module_test_any_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, const int *status, int datatype); /* * Indicate whether at least one variable within an array of variables on the local PE meets * a specified test condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_value The value to be compared with the objects pointed to by ivars. * @param nelems The number of elements in the ivars array. * @param indices Local address of an array of indices of length at least nelems into * ivars that satisfied the wait condition. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the test set. * @param datatype Type of the objects * * @return Returns the number of indices returned in the indices array. If the test * set is empty, this routine returns 0. */ typedef size_t (*mca_spml_base_module_test_some_fn_t)(void *ivars, int cmp, void *cmp_value, size_t nelems, size_t *indices, const int *status, int datatype); /* * Indicate whether all variables within an array of variables on the local PE meet the * specified test conditions. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with elements * of cmp_values. * @param cmp_values Local address of an array of length nelems containing values to be * compared with the respective objects in ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the test set. * @param datatype Type of the objects * * @return Returns 1 if all variables in ivars satisfy the test conditions or if * nelems is 0, otherwise this routine returns 0. */ typedef int (*mca_spml_base_module_test_all_vector_fn_t)(void *ivars, int cmp, void *cmp_values, size_t nelems, const int *status, int datatype); /* * Indicate whether any one variable within an array of variables on the local PE meets * its specified test condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with cmp_value. * @param cmp_values Local address of an array of length nelems containing values to be * compared with the respective objects in ivars. * @param nelems The number of elements in the ivars array. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the test set. * @param datatype Type of the objects * * @return OSHMEM_SUCCESS or failure status. */ typedef size_t (*mca_spml_base_module_test_any_vector_fn_t)(void *ivars, int cmp, void *cmp_values, size_t nelems, const int *status, int datatype); /* * Indicate whether at least one variable within an array of variables on the local PE meets * its specified test condition. * * @param ivars Symmetric address of an array of remotely accessible * data objects. The type of ivars should match that * implied in the SYNOPSIS section. * @param cmp A comparison operator that compares elements of ivars with elements * of cmp_values. * @param cmp_values Local address of an array of length nelems containing values to be * compared with the respective objects in ivars. * @param nelems The number of elements in the ivars array. * @param indices Local address of an array of indices of length at least nelems into ivars * that satisfied the wait condition. * @param status Local address of an optional mask array of length nelems that indicates * which elements in ivars are excluded from the test set. * @param datatype Type of the objects * * @return OSHMEM_SUCCESS or failure status. */ typedef size_t (*mca_spml_base_module_test_some_vector_fn_t)(void *ivars, int cmp, void *cmp_values, size_t nelems, size_t *indices, const int *status, int datatype); /* * Registers the arrival of a PE at a synchronization point. * This routine does not return until all other PEs in a given * OpenSHMEM team or active set arrive at this synchronization point. * * * @param team An OpenSHMEM team handle. * * @return OSHMEM_SUCCESS or failure status. * Zero on successful local completion. Nonzero otherwise. */ typedef int (*mca_spml_base_module_team_sync_fn_t)(shmem_team_t team); /* * Returns the number of the calling PE within a specified team. * * @param team An OpenSHMEM team handle. * * @return The number of the calling PE within the specified * team, or the value -1 if the team handle compares * equal to SHMEM_TEAM_INVALID */ typedef int (*mca_spml_base_module_team_my_pe_fn_t)(shmem_team_t team); /* * Returns the number of PEs in a specified team. * * @param team An OpenSHMEM team handle. * * @return The number of PEs in the specified team, or the * value -1 if the team handle compares equal to * SHMEM_TEAM_INVALID. */ typedef int (*mca_spml_base_module_team_n_pes_fn_t)(shmem_team_t team); /* * Return the configuration parameters of a given team * * @param team An OpenSHMEM team handle. * * @param config_mask The bitwise mask representing the set of * configuration parameters to fetch from the * given team. * * @param config A pointer to the configuration parameters for the * given team. * * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_get_config_fn_t)(shmem_team_t team, long config_mask, shmem_team_config_t *config); /* * Translate a given PE number from one team to the corresponding PE number in * another team. * * @param src_team An OpenSHMEM team handle. * @param src_pe A PE number in src_team. * @param dest_team An OpenSHMEM team handle. * * * @return The specified PE’s number in the dest_team, or a value * of -1 if any team handle arguments are invalid or the * src_pe is not in both the source and destination teams. */ typedef int (*mca_spml_base_module_team_translate_pe_fn_t)(shmem_team_t src_team, int src_pe, shmem_team_t dest_team); /* * Create a new OpenSHMEM team from a subset of the existing parent team PEs, * where the subset is defined by the PE triplet (start, stride, and size) * supplied to the routine. * * @param parent_team An OpenSHMEM team handle. * @param start The lowest PE number of the subset of PEs from the parent team * that will form the new team. * @param stride The stride between team PE numbers in the parent team that comprise the subset * of PEs that will form the new team. * @param size The number of PEs from the parent team in the subset of PEs that * will form the new team. size must be a positive integer. * @param config A pointer to the configuration parameters for the new team. * @param config_mask The bitwise mask representing the set of configuration parameters * to use from config. * @param new_team An OpenSHMEM team handle. Upon successful creation, it references an OpenSHMEM * team that contains the subset of all PEs in the parent team * specified by the PE triplet provided.m * * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_split_strided_fn_t)(shmem_team_t parent_team, int start, int stride, int size, const shmem_team_config_t *config, long config_mask, shmem_team_t *new_team); /* * Create two new teams by splitting an existing parent team into two subsets * based on a 2D Cartesian space defined by the xrange argument and a y * dimension that is derived from xrange and the parent team size. * * @param parent_team An OpenSHMEM team handle. * @param xrange A positive integer representing the number of elements in the first dimension. * @param xaxis_config A pointer to the configuration parameters for the new x-axis team. * @param xaxis_mask The bitwise mask representing the set of configuration parameters to * use from xaxis_config. * @param xaxis_team A new PE team handle representing a PE subset consisting of all the * PEs that have the same coordinate along the y-axis as the calling PE.. * @param yaxis_config A pointer to the configuration parameters for the new y-axis team. * @param yaxis_mask The bitwise mask representing the set of configuration parameters to use * from yaxis_config. * @param yaxis_team A new PE team handle representing a PE subset consisting of all the PEs * that have the same coordinate along the x-axis as the calling PE. * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_split_2d_fn_t)(shmem_team_t parent_team, int xrange, const shmem_team_config_t *xaxis_config, long xaxis_mask, shmem_team_t *xaxis_team, const shmem_team_config_t *yaxis_config, long yaxis_mask, shmem_team_t *yaxis_team); /* * Destroy an existing team. * * @param team An OpenSHMEM team handle. * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_destroy_fn_t)(shmem_team_t team); /* * Retrieve the team associated with the communication context. * * @param ctx A handle to a communication context. * @param team A pointer to a handle to the associated PE team. * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_get_fn_t)(shmem_ctx_t ctx, shmem_team_t *team); /* * Create a communication context from a team. * * @param team An OpenSHMEM team handle. * @param options The set of options requested for the given context. * @param ctx A handle to the newly created context. * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_create_ctx_fn_t)(shmem_team_t team, long options, shmem_ctx_t *ctx); /* * Exchanges a fixed amount of contiguous data blocks between all pairs of * PEs participating in the collective routine.. * * @param team An OpenSHMEM team handle. * @param dest Symmetric address of a data object large enough to * receive the combined total of nelems elements from each PE in the active set. * @param source Symmetric address of a data object that contains nelems elements of data * for each PE in the active set, ordered according to destination PE. * @param nelems The number of elements to exchange for each PE. * @param datatype Datatype of the elements * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_alltoall_fn_t)(shmem_team_t team, void *dest, const void *source, size_t nelems, int datatype); /* * Exchanges a fixed amount of strided data blocks between all pairs of PEs * participating in the collective routine. * * @param team An OpenSHMEM team handle. * @param dest Symmetric address of a data object large enough to * receive the combined total of nelems elements from each PE in the active set. * @param source Symmetric address of a data object that contains nelems elements of data * for each PE in the active set, ordered according to destination PE. * @param dst The stride between consecutive elements of the dest data object. The stride * is scaled by the element size. A value of 1 indicates contiguous data. * @param sst The stride between consecutive elements of the source data object. The stride * is scaled by the element size. A value of 1 indicates contiguous data * @param nelems The number of elements to exchange for each PE. * @param datatype Datatype of the elements * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_alltoalls_fn_t)(shmem_team_t team, void *dest, const void *source, ptrdiff_t dst, ptrdiff_t sst, size_t nelems, int datatype); /* * Broadcasts a block of data from one PE to one or more destination PEs. * * @param team An OpenSHMEM team handle. * @param dest Symmetric address of destination data object. * @param source Symmetric address of the source data object. * @param nelems The number of elements in source and dest arrays * @param PE_root Zero-based ordinal of the PE, with respect to the team or * active set, from which the data is copied.. * @param datatype Datatype of the elements * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_broadcast_fn_t)(shmem_team_t team, void *dest, const void *source, size_t nelems, int PE_root, int datatype); /* * Concatenates blocks of data from multiple PEs to an array in every PE participating in * the collective routine. * * @param team An OpenSHMEM team handle. * @param dest Symmetric address of an array large enough to accept the * concatenation of the source arrays on all participating PEs. * @param source Symmetric address of the source data object. * @param nelems The number of elements in source array. * @param datatype Datatype of the elements * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_collect_fn_t)(shmem_team_t team, void *dest, const void *source, size_t nelems, int datatype); typedef int (*mca_spml_base_module_team_fcollect_fn_t)(shmem_team_t team, void *dest, const void *source, size_t nelems, int datatype); /* * Performs a math reduction across a set of PEs. * * @param team An OpenSHMEM team handle. * @param dest Symmetric address of an array, of length nreduce elements, * to receive the result of the reduction routines. * @param source Symmetric address of an array, of length nreduce elements, that * contains one element for each separate reduction routine. * @param nreduce The number of elements in the dest and source arrays. * @param operation Operations from list of supported oshmem ops * @param datatype Datatype of the elements * * @return OSHMEM_SUCCESS or failure status. * */ typedef int (*mca_spml_base_module_team_reduce_fn_t)(shmem_team_t team, void *dest, const void *source, size_t nreduce, int operation, int datatype); /** * Blocking data transfer from remote PE. * Read data from remote PE. * * @param ctx The context object this routine is working on. * @param dst_addr The address on the local PE, to write the result of the get operation to. * @param size The number of bytes to be read. * @param src_addr The address on the remote PE, to read from. * @param src The ID of the remote PE. * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_get_fn_t)(shmem_ctx_t ctx, void *dst_addr, size_t size, void *src_addr, int src); /** * Non-blocking data transfer from remote PE. * Read data from remote PE. * * @param ctx The context object this routine is working on. * @param dst_addr The address on the local PE, to write the result of the get operation to. * @param size The number of bytes to be read. * @param src_addr The address on the remote PE, to read from. * @param src The ID of the remote PE. * @param handle The address of a handle to be passed to shmem_wait_nb() or * shmem_test_nb() to wait or poll for the completion of the transfer. * @return - OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_get_nb_fn_t)(shmem_ctx_t ctx, void *dst_addr, size_t size, void *src_addr, int src, void **handle); /** * Post a receive and wait for completion. * * @param buf (IN) User buffer. * @param count (IN) The number of bytes to be sent. * @param src (IN) The ID of the remote PE. * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_recv_fn_t)(void *buf, size_t count, int src); /** * Post a send request and wait for completion. * * @param buf (IN) User buffer. * @param count (IN) The number of bytes to be sent. * @param dst (IN) The ID of the remote PE. * @param mode (IN) Send mode (STANDARD,BUFFERED,SYNCHRONOUS,READY) * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_send_fn_t)(void *buf, size_t count, int dst, mca_spml_base_put_mode_t mode); /** * The routine transfers the data asynchronously from the source PE to all * PEs in the OpenSHMEM job. The routine returns immediately. The source and * target buffers are reusable only after the completion of the routine. * After the data is transferred to the target buffers, the counter object * is updated atomically. The counter object can be read either using atomic * operations such as shmem_atomic_fetch or can use point-to-point synchronization * routines such as shmem_wait_until and shmem_test. * * Shmem_quiet may be used for completing the operation, but not required for * progress or completion. In a multithreaded OpenSHMEM program, the user * (the OpenSHMEM program) should ensure the correct ordering of * shmemx_alltoall_global calls. * * @param dest A symmetric data object that is large enough to receive * “size” bytes of data from each PE in the OpenSHMEM job. * @param source A symmetric data object that contains “size” bytes of data * for each PE in the OpenSHMEM job. * @param size The number of bytes to be sent to each PE in the job. * @param counter A symmetric data object to be atomically incremented after * the target buffer is updated. * * @return OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_put_all_nb_fn_t)(void *dest, const void *source, size_t size, long *counter); /** * Assures ordering of delivery of put() requests * * @param ctx - The context object this routine is working on. * @return - OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_fence_fn_t)(shmem_ctx_t ctx); /** * Wait for completion of all outstanding put() requests * * @param ctx - The context object this routine is working on. * @return - OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_quiet_fn_t)(shmem_ctx_t ctx); /** * Waits for completion of a non-blocking put or get issued by the calling PE. * * @return - OSHMEM_SUCCESS or failure status. */ typedef int (*mca_spml_base_module_wait_nb_fn_t)(void *); /** * Called by memheap when memory is allocated by shmalloc(), * shcalloc(), shmemalign() or shrealloc() * * @param addr base address of the registered buffer. * @param size the size of the buffer to be registered. */ typedef void (*mca_spml_base_module_memuse_hook_fn_t)(void *, size_t); /** * SPML instance. */ struct mca_spml_base_module_1_0_0_t { mca_spml_base_module_add_procs_fn_t spml_add_procs; mca_spml_base_module_del_procs_fn_t spml_del_procs; mca_spml_base_module_enable_fn_t spml_enable; mca_spml_base_module_register_fn_t spml_register; mca_spml_base_module_deregister_fn_t spml_deregister; mca_spml_base_module_oob_get_mkeys_fn_t spml_oob_get_mkeys; mca_spml_base_module_ctx_create_fn_t spml_ctx_create; mca_spml_base_module_ctx_destroy_fn_t spml_ctx_destroy; mca_spml_base_module_put_fn_t spml_put; mca_spml_base_module_put_nb_fn_t spml_put_nb; mca_spml_base_module_put_signal_fn_t spml_put_signal; mca_spml_base_module_put_signal_nb_fn_t spml_put_signal_nb; mca_spml_base_module_get_fn_t spml_get; mca_spml_base_module_get_nb_fn_t spml_get_nb; mca_spml_base_module_recv_fn_t spml_recv; mca_spml_base_module_send_fn_t spml_send; mca_spml_base_module_wait_fn_t spml_wait; mca_spml_base_module_wait_nb_fn_t spml_wait_nb; mca_spml_base_module_wait_until_all_fn_t spml_wait_until_all; mca_spml_base_module_wait_until_any_fn_t spml_wait_until_any; mca_spml_base_module_wait_until_some_fn_t spml_wait_until_some; mca_spml_base_module_wait_until_all_vector_fn_t spml_wait_until_all_vector; mca_spml_base_module_wait_until_any_vector_fn_t spml_wait_until_any_vector; mca_spml_base_module_wait_until_some_vector_fn_t spml_wait_until_some_vector; mca_spml_base_module_test_fn_t spml_test; mca_spml_base_module_test_all_fn_t spml_test_all; mca_spml_base_module_test_any_fn_t spml_test_any; mca_spml_base_module_test_some_fn_t spml_test_some; mca_spml_base_module_test_all_vector_fn_t spml_test_all_vector; mca_spml_base_module_test_any_vector_fn_t spml_test_any_vector; mca_spml_base_module_test_some_vector_fn_t spml_test_some_vector; mca_spml_base_module_team_sync_fn_t spml_team_sync; mca_spml_base_module_team_my_pe_fn_t spml_team_my_pe; mca_spml_base_module_team_n_pes_fn_t spml_team_n_pes; mca_spml_base_module_team_get_config_fn_t spml_team_get_config; mca_spml_base_module_team_translate_pe_fn_t spml_team_translate_pe; mca_spml_base_module_team_split_strided_fn_t spml_team_split_strided; mca_spml_base_module_team_split_2d_fn_t spml_team_split_2d; mca_spml_base_module_team_destroy_fn_t spml_team_destroy; mca_spml_base_module_team_get_fn_t spml_team_get; mca_spml_base_module_team_create_ctx_fn_t spml_team_create_ctx; mca_spml_base_module_team_alltoall_fn_t spml_team_alltoall; mca_spml_base_module_team_alltoalls_fn_t spml_team_alltoalls; mca_spml_base_module_team_broadcast_fn_t spml_team_broadcast; mca_spml_base_module_team_collect_fn_t spml_team_collect; mca_spml_base_module_team_fcollect_fn_t spml_team_fcollect; mca_spml_base_module_team_reduce_fn_t spml_team_reduce; mca_spml_base_module_fence_fn_t spml_fence; mca_spml_base_module_quiet_fn_t spml_quiet; mca_spml_base_module_mkey_unpack_fn_t spml_rmkey_unpack; mca_spml_base_module_mkey_free_fn_t spml_rmkey_free; mca_spml_base_module_mkey_ptr_fn_t spml_rmkey_ptr; mca_spml_base_module_memuse_hook_fn_t spml_memuse_hook; mca_spml_base_module_put_all_nb_fn_t spml_put_all_nb; void *self; }; typedef struct mca_spml_base_module_1_0_0_t mca_spml_base_module_1_0_0_t; typedef mca_spml_base_module_1_0_0_t mca_spml_base_module_t; /* * Macro for use in components that are of type spml */ #define MCA_SPML_BASE_VERSION_2_0_0 \ OSHMEM_MCA_BASE_VERSION_2_1_0("spml", 2, 0, 0) /* * macro for doing direct call / call through struct */ #if MCA_oshmem_spml_DIRECT_CALL #include MCA_oshmem_spml_DIRECT_CALL_HEADER #define MCA_SPML_CALL_STAMP(a, b) mca_spml_ ## a ## _ ## b #define MCA_SPML_CALL_EXPANDER(a, b) MCA_SPML_CALL_STAMP(a,b) #define MCA_SPML_CALL(a) MCA_SPML_CALL_EXPANDER(MCA_oshmem_spml_DIRECT_CALL_COMPONENT, a) #else #define MCA_SPML_CALL(a) mca_spml.spml_ ## a #endif OSHMEM_DECLSPEC extern mca_spml_base_module_t mca_spml; END_C_DECLS #endif /* MCA_SPML_H */