/* * Copyright (c) 2011-2017 Sandia National Laboratories. All rights reserved. * Copyright (c) 2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "ompi_config.h" #include "ompi/mca/osc/base/base.h" #include "ompi/mca/osc/base/osc_base_obj_convert.h" #include "ompi/mca/osc/osc.h" #include "osc_portals4.h" #include "osc_portals4_request.h" static int ompi_osc_portals4_get_op(struct ompi_op_t *op, ptl_op_t *ptl_op) { if (MPI_MAX == op) { *ptl_op = PTL_MAX; } else if (MPI_MIN == op) { *ptl_op = PTL_MIN; } else if (MPI_SUM == op) { *ptl_op = PTL_SUM; } else if (MPI_PROD == op) { *ptl_op = PTL_PROD; } else if (MPI_LAND == op) { *ptl_op = PTL_LAND; } else if (MPI_BAND == op) { *ptl_op = PTL_BAND; } else if (MPI_LOR == op) { *ptl_op = PTL_LOR; } else if (MPI_BOR == op) { *ptl_op = PTL_BOR; } else if (MPI_LXOR == op) { *ptl_op = PTL_LXOR; } else if (MPI_BXOR == op) { *ptl_op = PTL_BXOR; } else { return OMPI_ERROR; } return OMPI_SUCCESS; } static int get_sized_type(bool sign, size_t size, ptl_datatype_t *ptl_dt) { if (sign) { switch (size) { case 1: *ptl_dt = PTL_INT8_T; break; case 2: *ptl_dt = PTL_INT16_T; break; case 4: *ptl_dt = PTL_INT32_T; break; case 8: *ptl_dt = PTL_INT64_T; break; default: return OMPI_ERROR; } } else { switch (size) { case 1: *ptl_dt = PTL_UINT8_T; break; case 2: *ptl_dt = PTL_UINT16_T; break; case 4: *ptl_dt = PTL_UINT32_T; break; case 8: *ptl_dt = PTL_UINT64_T; break; default: return OMPI_ERROR; } } return OMPI_SUCCESS; } static int ompi_osc_portals4_get_dt(struct ompi_datatype_t *dt, ptl_datatype_t *ptl_dt) { ompi_datatype_t *base_dt = ompi_datatype_get_single_predefined_type_from_args(dt); if (MPI_BYTE == base_dt) { *ptl_dt = PTL_INT8_T; } else if (MPI_CHAR == base_dt) { *ptl_dt = PTL_INT8_T; } else if (MPI_SHORT == base_dt) { return get_sized_type(true, sizeof(short), ptl_dt); } else if (MPI_INT == base_dt) { return get_sized_type(true, sizeof(int), ptl_dt); } else if (MPI_LONG == base_dt) { return get_sized_type(true, sizeof(long), ptl_dt); } else if (MPI_FLOAT == base_dt) { *ptl_dt = PTL_FLOAT; } else if (MPI_DOUBLE == base_dt) { *ptl_dt = PTL_DOUBLE; } else if (MPI_LONG_DOUBLE == base_dt) { *ptl_dt = PTL_LONG_DOUBLE; } else if (MPI_UNSIGNED_CHAR == base_dt) { *ptl_dt = PTL_UINT8_T; } else if (MPI_SIGNED_CHAR == base_dt) { *ptl_dt = PTL_UINT8_T; } else if (MPI_UNSIGNED_SHORT == base_dt) { return get_sized_type(false, sizeof(short), ptl_dt); } else if (MPI_UNSIGNED_LONG == base_dt) { return get_sized_type(false, sizeof(long), ptl_dt); } else if (MPI_UNSIGNED == base_dt) { return get_sized_type(false, sizeof(int), ptl_dt); } else if (MPI_LONG_LONG_INT == base_dt) { return get_sized_type(true, sizeof(long long int), ptl_dt); } else if (MPI_LONG_LONG == base_dt) { return get_sized_type(true, sizeof(long long), ptl_dt); } else if (MPI_INT8_T == base_dt) { *ptl_dt = PTL_INT8_T; } else if (MPI_UINT8_T == base_dt) { *ptl_dt = PTL_UINT8_T; } else if (MPI_INT16_T == base_dt) { *ptl_dt = PTL_INT16_T; } else if (MPI_UINT16_T == base_dt) { *ptl_dt = PTL_UINT16_T; } else if (MPI_INT32_T == base_dt) { *ptl_dt = PTL_INT32_T; } else if (MPI_UINT32_T == base_dt) { *ptl_dt = PTL_UINT32_T; } else if (MPI_INT64_T == base_dt) { *ptl_dt = PTL_INT64_T; } else if (MPI_UINT64_T == base_dt) { *ptl_dt = PTL_UINT64_T; } else if (MPI_C_COMPLEX == base_dt) { *ptl_dt = PTL_DOUBLE_COMPLEX; } else if (MPI_C_FLOAT_COMPLEX == base_dt) { *ptl_dt = PTL_FLOAT_COMPLEX; } else if (MPI_C_DOUBLE_COMPLEX == base_dt) { *ptl_dt = PTL_DOUBLE_COMPLEX; } else if (MPI_C_LONG_DOUBLE_COMPLEX == base_dt) { *ptl_dt = PTL_LONG_DOUBLE_COMPLEX; } else if (MPI_AINT == base_dt) { if (sizeof(MPI_Aint) == 2) { *ptl_dt = PTL_UINT16_T; } else if (sizeof(MPI_Aint) == 4) { *ptl_dt = PTL_UINT32_T; } else if (sizeof(MPI_Aint) == 8) { *ptl_dt = PTL_UINT64_T; } } else { return OMPI_ERROR; } return 0; } static ptl_size_t number_of_fragments(ptl_size_t length, ptl_size_t maxlength) { ptl_size_t nb_frag = length == 0 ? 1 : (length - 1) / maxlength + 1; OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d : %ld fragment(s)", __FILE__, __LINE__, nb_frag)); return nb_frag; } /* put in segments no larger than segment_length */ static int segmentedPut(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t put_length, ptl_size_t segment_length, ptl_ack_req_t ack_req, ptl_process_t target_id, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, ptl_size_t target_offset, void *user_ptr, ptl_hdr_data_t hdr_data) { int ret; ptl_size_t bytes_put = 0; do { opal_atomic_add_fetch_64(opcount, 1); ptl_size_t frag_length = MIN(put_length, segment_length); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "Put size : %lu/%lu, offset:%lu", frag_length, put_length, bytes_put)); ret = PtlPut(md_h, origin_offset + bytes_put, frag_length, ack_req, target_id, pt_index, match_bits, target_offset + bytes_put, user_ptr, hdr_data); if (PTL_OK != ret) { opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlPut failed with return value %d", __FILE__, __LINE__, ret); return ret; } put_length -= frag_length; bytes_put += frag_length; } while (put_length); return PTL_OK; } /* get in segments no larger than segment_length */ static int segmentedGet(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t get_length, ptl_size_t segment_length, ptl_process_t target_id, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, ptl_size_t target_offset, void *user_ptr) { int ret; ptl_size_t bytes_gotten = 0; do { opal_atomic_add_fetch_64(opcount, 1); ptl_size_t frag_length = MIN(get_length, segment_length); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "Get size : %lu/%lu, offset:%lu", frag_length, get_length, bytes_gotten)); ret = PtlGet(md_h, (ptl_size_t) origin_offset + bytes_gotten, frag_length, target_id, pt_index, match_bits, target_offset + bytes_gotten, user_ptr); if (PTL_OK != ret) { opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlGet failed with return value %d", __FILE__, __LINE__, ret); return ret; } get_length -= frag_length; bytes_gotten += frag_length; } while (get_length); return PTL_OK; } /* atomic op in segments no larger than segment_length */ static int segmentedAtomic(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t length, ptl_size_t segment_length, ptl_process_t target_id, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, ptl_size_t target_offset, void *user_ptr, ptl_op_t ptl_op, ptl_datatype_t ptl_dt) { int ret; ptl_size_t sent = 0; do { opal_atomic_add_fetch_64(opcount, 1); ptl_size_t frag_length = MIN(length, segment_length); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "Atomic size : %lu/%lu, offset:%lu", frag_length, length, sent)); ret = PtlAtomic(md_h, (ptl_size_t) origin_offset + sent, frag_length, PTL_ACK_REQ, target_id, pt_index, match_bits, target_offset + sent, user_ptr, 0, ptl_op, ptl_dt); if (PTL_OK != ret) { opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlAtomic failed with return value %d", __FILE__, __LINE__, ret); return ret; } length -= frag_length; sent += frag_length; } while (length); return PTL_OK; } /* atomic op in segments no larger than segment_length */ static int segmentedFetchAtomic(opal_atomic_int64_t *opcount, ptl_handle_md_t result_md_h, ptl_size_t result_offset, ptl_handle_md_t origin_md_h, ptl_size_t origin_offset, ptl_size_t length, ptl_size_t segment_length, ptl_process_t target_id, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, ptl_size_t target_offset, void *user_ptr, ptl_op_t ptl_op, ptl_datatype_t ptl_dt) { int ret; ptl_size_t sent = 0; do { opal_atomic_add_fetch_64(opcount, 1); ptl_size_t frag_length = MIN(length, segment_length); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "Atomic size : %lu/%lu, offset:%lu", frag_length, length, sent)); ret = PtlFetchAtomic(result_md_h, result_offset + sent, origin_md_h, origin_offset + sent, frag_length, target_id, pt_index, match_bits, target_offset + sent, user_ptr, 0, ptl_op, ptl_dt); if (PTL_OK != ret) { opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlFetchAtomic failed with return value %d", __FILE__, __LINE__, ret); return ret; } length -= frag_length; sent += frag_length; } while (length); return PTL_OK; } /* swap in segments no larger than segment_length */ static int segmentedSwap(opal_atomic_int64_t *opcount, ptl_handle_md_t result_md_h, ptl_size_t result_offset, ptl_handle_md_t origin_md_h, ptl_size_t origin_offset, ptl_size_t length, ptl_size_t segment_length, ptl_process_t target_id, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, ptl_size_t target_offset, void *user_ptr, ptl_datatype_t ptl_dt) { int ret; ptl_size_t sent = 0; do { opal_atomic_add_fetch_64(opcount, 1); ptl_size_t frag_length = MIN(length, segment_length); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "Swap size : %lu/%lu, offset:%lu", frag_length, length, sent)); ret = PtlSwap(result_md_h, result_offset + sent, origin_md_h, (ptl_size_t) origin_offset + sent, frag_length, target_id, pt_index, match_bits, target_offset + sent, user_ptr, 0, NULL, PTL_SWAP, ptl_dt); if (PTL_OK != ret) { opal_atomic_add_fetch_64(opcount, -1); opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlSwap failed with return value %d", __FILE__, __LINE__, ret); return ret; } length -= frag_length; sent += frag_length; } while (length); return PTL_OK; } static int create_iov_list(const void *address, int count, ompi_datatype_t *datatype, ptl_iovec_t **ptl_iovec, ptl_size_t *ptl_iovec_count) { struct iovec iov[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t convertor; uint32_t iov_count; uint32_t iov_index, ptl_iovec_index; /* needed for opal_convertor_raw but not used */ size_t size; int ret; bool done; OBJ_CONSTRUCT(&convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &datatype->super, count, address, 0, &convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } *ptl_iovec_count = 0; ptl_iovec_index = 0; do { /* decode segments of the data */ iov_count = OSC_PORTALS4_IOVEC_MAX; iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&convertor, iov, &iov_count, &size); *ptl_iovec_count += iov_count; *ptl_iovec = (ptl_iovec_t *)realloc(*ptl_iovec, *ptl_iovec_count * sizeof(ptl_iovec_t)); while (iov_index != iov_count) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "adding iov[%d].[%p,%lu] to ptl_iovec", iov_index, iov[iov_index].iov_base, iov[iov_index].iov_len)); (*ptl_iovec)[ptl_iovec_index].iov_base = iov[iov_index].iov_base; (*ptl_iovec)[ptl_iovec_index].iov_len = iov[iov_index].iov_len; ptl_iovec_index++; iov_index++; } assert(*ptl_iovec_count == ptl_iovec_index); } while (!done); return OMPI_SUCCESS; } /* get from a contiguous remote to an iovec local */ static int get_to_iovec(ompi_osc_portals4_module_t *module, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { int ret; size_t size; ptrdiff_t length, origin_lb, target_lb, extent; ptl_md_t md; if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->origin_iovec_md_h); free(module->origin_iovec_list); module->origin_iovec_md_h = PTL_INVALID_HANDLE; module->origin_iovec_list = NULL; } ptl_size_t iovec_count=0; create_iov_list( origin_address, origin_count, origin_datatype, &module->origin_iovec_list, &iovec_count); ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_datatype, &size); length = size * origin_count; md.start = module->origin_iovec_list; md.length = iovec_count; if (user_ptr) { md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } else { md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); ret = PtlGet(module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, peer, module->pt_idx, module->match_bits, offset + target_lb, user_ptr); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d PtlGet() failed: ret = %d", __FILE__, __LINE__, ret)); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } return OMPI_SUCCESS; } /* get to an iovec MD from a contiguous target using fragments no larger * than max_fetch_atomic_size to guarantee atomic writes at the origin */ static int atomic_get_to_iovec(ompi_osc_portals4_module_t *module, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { int ret; size_t size; ptrdiff_t length, origin_lb, target_lb, extent; ptl_md_t md; if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->origin_iovec_md_h); free(module->origin_iovec_list); module->origin_iovec_md_h = PTL_INVALID_HANDLE; module->origin_iovec_list = NULL; } ptl_size_t iovec_count=0; create_iov_list( origin_address, origin_count, origin_datatype, &module->origin_iovec_list, &iovec_count); ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_datatype, &size); length = size * origin_count; md.start = module->origin_iovec_list; md.length = iovec_count; if (user_ptr) { md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } else { md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); ret = segmentedGet(&module->opcount, module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, offset + target_lb, user_ptr); if (PTL_OK != ret) { return ret; } return OMPI_SUCCESS; } /* put from an iovec MD into a contiguous target */ static int put_from_iovec(ompi_osc_portals4_module_t *module, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { int ret; size_t size; ptrdiff_t length, origin_lb, target_lb, extent; ptl_md_t md; if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->origin_iovec_md_h); free(module->origin_iovec_list); module->origin_iovec_md_h = PTL_INVALID_HANDLE; module->origin_iovec_list = NULL; } ptl_size_t iovec_count=0; create_iov_list( origin_address, origin_count, origin_datatype, &module->origin_iovec_list, &iovec_count); ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_datatype, &size); length = size * origin_count; md.start = module->origin_iovec_list; md.length = iovec_count; if (user_ptr) { md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } else { md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); ret = PtlPut(module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + target_lb, user_ptr, 0); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d PtlPut() failed: ret = %d", __FILE__, __LINE__, ret)); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } return OMPI_SUCCESS; } /* put from an iovec MD into a contiguous target using fragments no larger * than max_atomic_size to guarantee atomic writes at the target */ static int atomic_put_from_iovec(ompi_osc_portals4_module_t *module, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { int ret; size_t size; ptrdiff_t length, origin_lb, target_lb, extent; ptl_md_t md; if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->origin_iovec_md_h); free(module->origin_iovec_list); module->origin_iovec_md_h = PTL_INVALID_HANDLE; module->origin_iovec_list = NULL; } ptl_size_t iovec_count=0; create_iov_list( origin_address, origin_count, origin_datatype, &module->origin_iovec_list, &iovec_count); ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_datatype, &size); length = size * origin_count; md.start = module->origin_iovec_list; md.length = iovec_count; if (user_ptr) { md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } else { md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); ret = segmentedPut(&module->opcount, module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, module->atomic_max, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + target_lb, NULL, 0); if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; } /* perform atomic operation on iovec local and contiguous remote */ static int atomic_from_iovec(ompi_osc_portals4_module_t *module, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, struct ompi_op_t *op, void *user_ptr) { int ret; size_t size; ptrdiff_t length, origin_lb, target_lb, extent; ptl_md_t md; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->origin_iovec_md_h); free(module->origin_iovec_list); module->origin_iovec_md_h = PTL_INVALID_HANDLE; module->origin_iovec_list = NULL; } ptl_size_t iovec_count=0; create_iov_list( origin_address, origin_count, origin_datatype, &module->origin_iovec_list, &iovec_count); ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_datatype, &size); length = size * origin_count; md.start = module->origin_iovec_list; md.length = iovec_count; if (user_ptr) { md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } else { md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } ret = segmentedAtomic(&module->opcount, module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, module->atomic_max, peer, module->pt_idx, module->match_bits, offset + target_lb, user_ptr, ptl_op, ptl_dt); if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; } /* perform atomic operation on iovec local and contiguous remote */ static int swap_to_iovec(ompi_osc_portals4_module_t *module, const void *result_address, int result_count, ompi_datatype_t *result_datatype, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { int ret; size_t size; ptl_size_t iovec_count=0; ptrdiff_t length, result_lb, origin_lb, target_lb, extent; ptl_md_t md; ptl_datatype_t ptl_dt; if (module->result_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->result_iovec_md_h); free(module->result_iovec_list); module->result_iovec_md_h = PTL_INVALID_HANDLE; module->result_iovec_list = NULL; } create_iov_list( result_address, result_count, result_datatype, &module->result_iovec_list, &iovec_count); md.start = module->result_iovec_list; md.length = iovec_count; if (user_ptr) { md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } else { md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->result_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->origin_iovec_md_h); free(module->origin_iovec_list); module->origin_iovec_md_h = PTL_INVALID_HANDLE; module->origin_iovec_list = NULL; } create_iov_list( origin_address, origin_count, origin_datatype, &module->origin_iovec_list, &iovec_count); md.start = module->origin_iovec_list; md.length = iovec_count; md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_datatype_get_true_extent(result_datatype, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_datatype, &size); length = size * origin_count; ret = segmentedSwap(&module->opcount, module->result_iovec_md_h, (ptl_size_t) result_lb, module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, offset + target_lb, user_ptr, ptl_dt); if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; } /* perform fetch atomic operation on iovec local and contiguous remote */ static int fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module, const void *result_address, int result_count, ompi_datatype_t *result_datatype, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, struct ompi_op_t *op, void *user_ptr) { int ret; size_t size; ptl_size_t iovec_count=0; ptrdiff_t length, result_lb, origin_lb, target_lb, extent; ptl_md_t md; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; if (module->result_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->result_iovec_md_h); free(module->result_iovec_list); module->result_iovec_md_h = PTL_INVALID_HANDLE; module->result_iovec_list = NULL; } create_iov_list( result_address, result_count, result_datatype, &module->result_iovec_list, &iovec_count); md.start = module->result_iovec_list; md.length = iovec_count; if (user_ptr) { md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } else { md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; } md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->result_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { PtlMDRelease(module->origin_iovec_md_h); free(module->origin_iovec_list); module->origin_iovec_md_h = PTL_INVALID_HANDLE; module->origin_iovec_list = NULL; } create_iov_list( origin_address, origin_count, origin_datatype, &module->origin_iovec_list, &iovec_count); md.start = module->origin_iovec_list; md.length = iovec_count; md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; md.eq_handle = mca_osc_portals4_component.matching_eq_h; md.ct_handle = module->ct_h; ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d: PtlMDBind(iovec) failed: %d\n", __FILE__, __LINE__, ret); return ret; } ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_datatype_get_true_extent(result_datatype, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_datatype, &size); length = size * origin_count; ret = segmentedFetchAtomic(&module->opcount, module->result_iovec_md_h, (ptl_size_t) result_lb, module->origin_iovec_md_h, (ptl_size_t) origin_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, offset + target_lb, user_ptr, ptl_op, ptl_dt); if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; } /* * Derived from ompi_osc_rdma_master_noncontig() */ /* put in the largest chunks possible given the noncontiguous restriction */ static int put_to_noncontig(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t origin_convertor, target_convertor; uint32_t origin_iov_count, target_iov_count; uint32_t origin_iov_index, target_iov_index; /* needed for opal_convertor_raw but not used */ size_t origin_size, target_size, rdma_len; size_t max_rdma_len = mca_osc_portals4_component.ptl_max_msg_size; int ret; bool done; /* prepare convertors for the source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, (void*)origin_address, 0, &origin_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *)NULL, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } origin_iov_index = 0; origin_iov_count = 0; do { /* decode segments of the remote data */ target_iov_count = OSC_PORTALS4_IOVEC_MAX; target_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); /* loop on the target segments until we have exhausted the decoded source data */ while (target_iov_index != target_iov_count) { if (origin_iov_index == origin_iov_count) { /* decode segments of the target buffer */ origin_iov_count = OSC_PORTALS4_IOVEC_MAX; origin_iov_index = 0; (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != origin_iov_count); /* determine how much to transfer in this operation */ rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); opal_atomic_add_fetch_64(opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len)); ret = PtlPut(md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, rdma_len, PTL_ACK_REQ, peer, pt_index, match_bits, offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, user_ptr, 0); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_atomic_add_fetch_64(opcount, -1); return ret; } /* adjust io vectors */ origin_iovec[origin_iov_index].iov_len -= rdma_len; target_iovec[target_iov_index].iov_len -= rdma_len; origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); /* clean up convertors */ opal_convertor_cleanup (&origin_convertor); OBJ_DESTRUCT(&origin_convertor); opal_convertor_cleanup (&target_convertor); OBJ_DESTRUCT(&target_convertor); return OMPI_SUCCESS; } /* put in fragments no larger than max_atomic_size to guarantee atomic writes at the target */ static int atomic_put_to_noncontig(ompi_osc_portals4_module_t *module, ptl_handle_md_t md_h, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t origin_convertor, target_convertor; uint32_t origin_iov_count, target_iov_count; uint32_t origin_iov_index, target_iov_index; /* needed for opal_convertor_raw but not used */ size_t origin_size, target_size, rdma_len; size_t max_rdma_len = module->atomic_max; int ret; bool done; /* prepare convertors for the source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, (void*)origin_address, 0, &origin_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *)NULL, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } origin_iov_index = 0; origin_iov_count = 0; do { /* decode segments of the remote data */ target_iov_count = OSC_PORTALS4_IOVEC_MAX; target_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); /* loop on the target segments until we have exhausted the decoded source data */ while (target_iov_index != target_iov_count) { if (origin_iov_index == origin_iov_count) { /* decode segments of the target buffer */ origin_iov_count = OSC_PORTALS4_IOVEC_MAX; origin_iov_index = 0; (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != origin_iov_count); /* determine how much to transfer in this operation */ rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len)); ret = PtlPut(md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, rdma_len, PTL_ACK_REQ, peer, pt_index, match_bits, offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, user_ptr, 0); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } /* adjust io vectors */ origin_iovec[origin_iov_index].iov_len -= rdma_len; target_iovec[target_iov_index].iov_len -= rdma_len; origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); return OMPI_SUCCESS; } /* perform atomic operation on (non)contiguous local and noncontiguous remote */ static int atomic_to_noncontig(ompi_osc_portals4_module_t *module, ptl_handle_md_t md_h, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, struct ompi_op_t *op, void *user_ptr) { struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t origin_convertor, target_convertor; uint32_t origin_iov_count, target_iov_count; uint32_t origin_iov_index, target_iov_index; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; /* needed for opal_convertor_raw but not used */ size_t origin_size, target_size, atomic_len; int ret; bool done; /* prepare convertors for the source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, (void*)origin_address, 0, &origin_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *)NULL, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } origin_iov_index = 0; origin_iov_count = 0; do { /* decode segments of the remote data */ target_iov_count = OSC_PORTALS4_IOVEC_MAX; target_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); /* loop on the target segments until we have exhausted the decoded source data */ while (target_iov_index != target_iov_count) { if (origin_iov_index == origin_iov_count) { /* decode segments of the target buffer */ origin_iov_count = OSC_PORTALS4_IOVEC_MAX; origin_iov_index = 0; (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != origin_iov_count); /* determine how much to transfer in this operation */ atomic_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), module->atomic_max); opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len)); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Atomic", __FILE__, __LINE__)); ret = PtlAtomic(md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, atomic_len, PTL_ACK_REQ, peer, pt_index, match_bits, offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, user_ptr, 0, ptl_op, ptl_dt); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } /* adjust io vectors */ origin_iovec[origin_iov_index].iov_len -= atomic_len; target_iovec[target_iov_index].iov_len -= atomic_len; origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + atomic_len); target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + atomic_len); origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); return OMPI_SUCCESS; } /* get from a noncontiguous remote to an (non)contiguous local */ static int get_from_noncontig(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t origin_convertor, target_convertor; uint32_t origin_iov_count, target_iov_count; uint32_t origin_iov_index, target_iov_index; /* needed for opal_convertor_raw but not used */ size_t origin_size, target_size, rdma_len; size_t max_rdma_len = mca_osc_portals4_component.ptl_max_msg_size; int ret; bool done; /* prepare convertors for the source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, (void*)origin_address, 0, &origin_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *)NULL, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } origin_iov_index = 0; origin_iov_count = 0; do { /* decode segments of the remote data */ target_iov_count = OSC_PORTALS4_IOVEC_MAX; target_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); /* loop on the target segments until we have exhausted the decoded source data */ while (target_iov_index != target_iov_count) { if (origin_iov_index == origin_iov_count) { /* decode segments of the target buffer */ origin_iov_count = OSC_PORTALS4_IOVEC_MAX; origin_iov_index = 0; (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != origin_iov_count); /* determine how much to transfer in this operation */ rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); opal_atomic_add_fetch_64(opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len)); ret = PtlGet(md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, rdma_len, peer, pt_index, match_bits, offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, user_ptr); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_atomic_add_fetch_64(opcount, -1); return ret; } /* adjust io vectors */ origin_iovec[origin_iov_index].iov_len -= rdma_len; target_iovec[target_iov_index].iov_len -= rdma_len; origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); return OMPI_SUCCESS; } /* get from a noncontiguous remote to an (non)contiguous local */ static int atomic_get_from_noncontig(ompi_osc_portals4_module_t *module, ptl_handle_md_t md_h, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t origin_convertor, target_convertor; uint32_t origin_iov_count, target_iov_count; uint32_t origin_iov_index, target_iov_index; /* needed for opal_convertor_raw but not used */ size_t origin_size, target_size, rdma_len; size_t max_rdma_len = module->fetch_atomic_max; int ret; bool done; /* prepare convertors for the source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, (void*)origin_address, 0, &origin_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *)NULL, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } origin_iov_index = 0; origin_iov_count = 0; do { /* decode segments of the remote data */ target_iov_count = OSC_PORTALS4_IOVEC_MAX; target_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); /* loop on the target segments until we have exhausted the decoded source data */ while (target_iov_index != target_iov_count) { if (origin_iov_index == origin_iov_count) { /* decode segments of the target buffer */ origin_iov_count = OSC_PORTALS4_IOVEC_MAX; origin_iov_index = 0; (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != origin_iov_count); /* determine how much to transfer in this operation */ rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len)); ret = PtlGet(md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, rdma_len, peer, pt_index, match_bits, offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, user_ptr); if (OPAL_UNLIKELY(PTL_OK != ret)) { opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } /* adjust io vectors */ origin_iovec[origin_iov_index].iov_len -= rdma_len; target_iovec[target_iov_index].iov_len -= rdma_len; origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); return OMPI_SUCCESS; } /* swap from a noncontiguous remote to an (non)contiguous local */ static int swap_from_noncontig(ompi_osc_portals4_module_t *module, ptl_handle_md_t result_md_h, const void *result_address, int result_count, ompi_datatype_t *result_datatype, ptl_handle_md_t origin_md_h, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, void *user_ptr) { struct iovec result_iovec[OSC_PORTALS4_IOVEC_MAX], origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t result_convertor, origin_convertor, target_convertor; uint32_t result_iov_count, origin_iov_count, target_iov_count; uint32_t result_iov_index, origin_iov_index, target_iov_index; /* needed for opal_convertor_raw but not used */ size_t result_size, origin_size, target_size, rdma_len; size_t max_rdma_len = module->fetch_atomic_max; ptl_datatype_t ptl_dt; int ret; bool done; /* prepare convertors for the result, source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ OBJ_CONSTRUCT(&result_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &result_datatype->super, result_count, (void*)result_address, 0, &result_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, (void*)origin_address, 0, &origin_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *)NULL, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } result_iov_index = 0; result_iov_count = 0; origin_iov_index = 0; origin_iov_count = 0; do { /* decode segments of the remote data */ target_iov_count = OSC_PORTALS4_IOVEC_MAX; target_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); /* loop on the target segments until we have exhausted the decoded source data */ while (target_iov_index != target_iov_count) { if (result_iov_index == result_iov_count) { /* decode segments of the target buffer */ result_iov_count = OSC_PORTALS4_IOVEC_MAX; result_iov_index = 0; (void) opal_convertor_raw (&result_convertor, result_iovec, &result_iov_count, &result_size); } if (origin_iov_index == origin_iov_count) { /* decode segments of the target buffer */ origin_iov_count = OSC_PORTALS4_IOVEC_MAX; origin_iov_index = 0; (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != result_iov_count); assert (0 != origin_iov_count); /* determine how much to transfer in this operation */ rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "performing swap on contiguous region. result: %p origin: %p, target: %p, len: %lu", result_iovec[result_iov_index].iov_base, origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len)); ret = PtlSwap(result_md_h, (ptl_size_t)result_iovec[result_iov_index].iov_base, origin_md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, rdma_len, peer, pt_index, match_bits, offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, user_ptr, 0, NULL, PTL_SWAP, ptl_dt); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlSwap failed with return value %d", __FILE__, __LINE__, ret); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } /* adjust io vectors */ result_iovec[result_iov_index].iov_len -= rdma_len; origin_iovec[origin_iov_index].iov_len -= rdma_len; target_iovec[target_iov_index].iov_len -= rdma_len; result_iovec[result_iov_index].iov_base = (void *)((intptr_t) result_iovec[result_iov_index].iov_base + rdma_len); origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); result_iov_index += (0 == result_iovec[result_iov_index].iov_len); origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); return OMPI_SUCCESS; } /* swap from a noncontiguous remote to an (non)contiguous local */ static int fetch_atomic_from_noncontig(ompi_osc_portals4_module_t *module, ptl_handle_md_t result_md_h, const void *result_address, int result_count, ompi_datatype_t *result_datatype, ptl_handle_md_t origin_md_h, const void *origin_address, int origin_count, ompi_datatype_t *origin_datatype, ptl_process_t peer, int target_count, ompi_datatype_t *target_datatype, size_t offset, ptl_pt_index_t pt_index, ptl_match_bits_t match_bits, struct ompi_op_t *op, void *user_ptr) { struct iovec result_iovec[OSC_PORTALS4_IOVEC_MAX], origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; opal_convertor_t result_convertor, origin_convertor, target_convertor; uint32_t result_iov_count, origin_iov_count, target_iov_count; uint32_t result_iov_index, origin_iov_index, target_iov_index; /* needed for opal_convertor_raw but not used */ size_t result_size, origin_size, target_size, rdma_len; size_t max_rdma_len = module->fetch_atomic_max; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; int ret; bool done; /* prepare convertors for the result, source and target. these convertors will be used to determine the * contiguous segments within the source and target. */ OBJ_CONSTRUCT(&result_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &result_datatype->super, result_count, (void*)result_address, 0, &result_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, (void*)origin_address, 0, &origin_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, (void *)NULL, 0, &target_convertor); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } result_iov_index = 0; result_iov_count = 0; origin_iov_index = 0; origin_iov_count = 0; do { /* decode segments of the remote data */ target_iov_count = OSC_PORTALS4_IOVEC_MAX; target_iov_index = 0; /* opal_convertor_raw returns done when it has reached the end of the data */ done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); /* loop on the target segments until we have exhausted the decoded source data */ while (target_iov_index != target_iov_count) { if (result_iov_index == result_iov_count) { /* decode segments of the target buffer */ result_iov_count = OSC_PORTALS4_IOVEC_MAX; result_iov_index = 0; (void) opal_convertor_raw (&result_convertor, result_iovec, &result_iov_count, &result_size); } if (origin_iov_index == origin_iov_count) { /* decode segments of the target buffer */ origin_iov_count = OSC_PORTALS4_IOVEC_MAX; origin_iov_index = 0; (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); } /* we already checked that the target was large enough. this should be impossible */ assert (0 != result_iov_count); assert (0 != origin_iov_count); /* determine how much to transfer in this operation */ rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "performing swap on contiguous region. result: %p origin: %p, target: %p, len: %lu", result_iovec[result_iov_index].iov_base, origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, (unsigned long) target_iovec[target_iov_index].iov_len)); ret = PtlFetchAtomic(result_md_h, (ptl_size_t)result_iovec[result_iov_index].iov_base, origin_md_h, (ptl_size_t)origin_iovec[origin_iov_index].iov_base, rdma_len, peer, pt_index, match_bits, offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, user_ptr, 0, ptl_op, ptl_dt); if (PTL_OK != ret) { opal_output_verbose(1, ompi_osc_base_framework.framework_output, "%s:%d PtlFetchAtomic failed with return value %d", __FILE__, __LINE__, ret); opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } /* adjust io vectors */ result_iovec[result_iov_index].iov_len -= rdma_len; origin_iovec[origin_iov_index].iov_len -= rdma_len; target_iovec[target_iov_index].iov_len -= rdma_len; result_iovec[result_iov_index].iov_base = (void *)((intptr_t) result_iovec[result_iov_index].iov_base + rdma_len); origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); result_iov_index += (0 == result_iovec[result_iov_index].iov_len); origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } } while (!done); return OMPI_SUCCESS; } int ompi_osc_portals4_rput(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_win_t *win, struct ompi_request_t **ompi_req) { int ret; ompi_osc_portals4_request_t *request; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t size, offset; ptrdiff_t length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rput: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, (unsigned long) win)); OMPI_OSC_PORTALS4_REQUEST_ALLOC(win, request); if (NULL == request) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; *ompi_req = &request->super; offset = get_displacement(module, target) * target_disp; if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { ret = put_to_noncontig(&module->opcount, module->req_md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_to_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { ret = put_from_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_from_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; request->ops_expected += number_of_fragments(length, mca_osc_portals4_component.ptl_max_msg_size); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d RPut(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); ret = segmentedPut(&module->opcount, module->req_md_h, (ptl_size_t) origin_addr + origin_lb, length, mca_osc_portals4_component.ptl_max_msg_size, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + target_lb, request, 0); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } return OMPI_SUCCESS; } int ompi_osc_portals4_rget(void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_win_t *win, struct ompi_request_t **ompi_req) { int ret; ompi_osc_portals4_request_t *request; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t offset, size; ptrdiff_t length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, (unsigned long) win)); OMPI_OSC_PORTALS4_REQUEST_ALLOC(win, request); if (NULL == request) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; *ompi_req = &request->super; offset = get_displacement(module, target) * target_disp; if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { ret = get_from_noncontig(&module->opcount, module->req_md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { ret = get_to_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; request->ops_expected += number_of_fragments(length, mca_osc_portals4_component.ptl_max_msg_size); OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, "%s,%d RGet", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->req_md_h, (ptl_size_t) origin_addr + origin_lb, length, mca_osc_portals4_component.ptl_max_msg_size, peer, module->pt_idx, module->match_bits, offset + target_lb, request); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } return OMPI_SUCCESS; } int ompi_osc_portals4_raccumulate(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_op_t *op, struct ompi_win_t *win, struct ompi_request_t **ompi_req) { int ret; ompi_osc_portals4_request_t *request; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; ptrdiff_t sent, length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "raccumulate: 0x%lx, %d, %s, %d, %lu, %d, %s, %s 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, op->o_name, (unsigned long) win)); OMPI_OSC_PORTALS4_REQUEST_ALLOC(win, request); if (NULL == request) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; *ompi_req = &request->super; offset = get_displacement(module, target) * target_disp; if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { if (MPI_REPLACE == op) { ret = atomic_put_to_noncontig(module, module->req_md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_to_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = atomic_to_noncontig(module, module->req_md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, op, request); if (PTL_OK != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_to_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { if (MPI_REPLACE == op) { ret = atomic_put_from_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_from_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = atomic_from_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, op, request); if (PTL_OK != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_from_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } } else { ptl_size_t md_offset; ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; sent = 0; md_offset = (ptl_size_t) origin_addr; request->ops_expected += number_of_fragments(length, module->atomic_max); if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put", __FILE__, __LINE__)); ret = segmentedPut(&module->opcount, module->req_md_h, md_offset + origin_lb, length, module->atomic_max, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + target_lb, request, 0); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } else { ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); opal_output(ompi_osc_base_framework.framework_output, "operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } do { size_t msg_length = MIN(module->atomic_max, length - sent); (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Atomic", __FILE__, __LINE__)); ret = PtlAtomic(module->req_md_h, md_offset + sent + origin_lb, msg_length, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + sent + target_lb, request, 0, ptl_op, ptl_dt); if (OMPI_SUCCESS != ret) { (void)opal_atomic_add_fetch_64(&module->opcount, -1); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } sent += msg_length; } while (sent < length); } } return OMPI_SUCCESS; } int ompi_osc_portals4_rget_accumulate(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, void *result_addr, int result_count, struct ompi_datatype_t *result_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_op_t *op, struct ompi_win_t *win, struct ompi_request_t **ompi_req) { int ret; ompi_osc_portals4_request_t *request; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t target_offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; ptrdiff_t length, origin_lb, target_lb, result_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, (unsigned long) result_addr, result_count, result_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, op->o_name, (unsigned long) win)); OMPI_OSC_PORTALS4_REQUEST_ALLOC(win, request); if (NULL == request) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; *ompi_req = &request->super; target_offset = get_displacement(module, target) * target_disp; if (target_count > 0 && !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: MPI_REPLACE non-contiguous target")); ret = swap_from_noncontig(module, module->req_md_h, result_addr, result_count, result_dt, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } else if (MPI_NO_OP == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: MPI_NO_OP non-contiguous target")); ret = atomic_get_from_noncontig(module, module->req_md_h, result_addr, result_count, result_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } else { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: other-op non-contiguous target")); ret = fetch_atomic_from_noncontig(module, module->req_md_h, result_addr, result_count, result_dt, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, op, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } } else if ((origin_count > 0 && !ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) || (result_count > 0 && !ompi_datatype_is_contiguous_memory_layout(result_dt, result_count))) { if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: MPI_REPLACE non-contiguous origin/result")); ret = swap_to_iovec(module, result_addr, result_count, result_dt, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } else if (MPI_NO_OP == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: MPI_NO_OP non-contiguous origin/result")); ret = atomic_get_to_iovec(module, result_addr, result_count, result_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } else { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: other-op non-contiguous origin/result")); ret = fetch_atomic_to_iovec(module, result_addr, result_count, result_dt, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, op, request); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } } else { if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: MPI_REPLACE contiguous")); ptl_size_t result_md_offset, origin_md_offset; ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return OMPI_ERR_NOT_SUPPORTED; } result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; request->ops_expected += number_of_fragments(length, module->fetch_atomic_max); ret = segmentedSwap(&module->opcount, module->req_md_h, result_md_offset + result_lb, module->md_h, origin_md_offset + origin_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, target_offset + target_lb, request, ptl_dt); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } else if (MPI_NO_OP == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: MPI_NO_OP contiguous")); ptl_size_t md_offset; ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ompi_datatype_type_size(target_dt, &size); length = size * target_count; md_offset = (ptl_size_t) result_addr; request->ops_expected += number_of_fragments(length, module->fetch_atomic_max); OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, "%s,%d MPI_Get_accumulate", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->req_md_h, (ptl_size_t) md_offset + result_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, target_offset + target_lb, request); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } else { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: other-op contiguous")); ptl_size_t result_md_offset, origin_md_offset; ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "operation is not currently supported"); OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return OMPI_ERR_NOT_SUPPORTED; } request->ops_expected += number_of_fragments(length, module->fetch_atomic_max); ret = segmentedFetchAtomic(&module->opcount, module->req_md_h, result_md_offset + result_lb, module->md_h, origin_md_offset + origin_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, target_offset + target_lb, request, ptl_op, ptl_dt); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } } } return OMPI_SUCCESS; } int ompi_osc_portals4_put(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_win_t *win) { int ret; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t offset, size; ptrdiff_t length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "put: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, (unsigned long) win)); offset = get_displacement(module, target) * target_disp; if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { ret = put_to_noncontig(&module->opcount, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_to_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { ret = put_from_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d put_from_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", __FILE__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); ret = segmentedPut(&module->opcount, module->md_h, (ptl_size_t) origin_addr + origin_lb, length, mca_osc_portals4_component.ptl_max_msg_size, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + target_lb, NULL, 0); if (OMPI_SUCCESS != ret) { return ret; } } return OMPI_SUCCESS; } int ompi_osc_portals4_get(void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_win_t *win) { int ret; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t offset, size; ptrdiff_t length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, (unsigned long) win)); offset = get_displacement(module, target) * target_disp; if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { ret = get_from_noncontig(&module->opcount, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { ret = get_to_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d get_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, "%s,%d Get", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->md_h, (ptl_size_t) origin_addr + origin_lb, length, mca_osc_portals4_component.ptl_max_msg_size, peer, module->pt_idx, module->match_bits, offset + target_lb, NULL); if (OMPI_SUCCESS != ret) { return ret; } } return OMPI_SUCCESS; } int ompi_osc_portals4_accumulate(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_op_t *op, struct ompi_win_t *win) { int ret; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; ptrdiff_t sent, length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "accumulate: 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, op->o_name, (unsigned long) win)); offset = get_displacement(module, target) * target_disp; if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { if (MPI_REPLACE == op) { ret = atomic_put_to_noncontig(module, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_to_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = atomic_to_noncontig(module, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, op, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_to_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { if (MPI_REPLACE == op) { ret = atomic_put_from_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_put_from_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { ret = atomic_from_iovec(module, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, offset, module->pt_idx, module->match_bits, op, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_from_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } } else { ptl_size_t md_offset; ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; sent = 0; md_offset = (ptl_size_t) origin_addr; if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put", __FILE__, __LINE__)); ret = segmentedPut(&module->opcount, module->md_h, md_offset + origin_lb, length, module->atomic_max, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + target_lb, NULL, 0); if (OMPI_SUCCESS != ret) { return ret; } } else { ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } do { size_t msg_length = MIN(module->atomic_max, length - sent); (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Atomic", __FILE__, __LINE__)); ret = PtlAtomic(module->md_h, md_offset + sent + origin_lb, msg_length, PTL_ACK_REQ, peer, module->pt_idx, module->match_bits, offset + sent + target_lb, NULL, 0, ptl_op, ptl_dt); if (OMPI_SUCCESS != ret) { (void)opal_atomic_add_fetch_64(&module->opcount, -1); return ret; } sent += msg_length; } while (sent < length); } } return OMPI_SUCCESS; } int ompi_osc_portals4_get_accumulate(const void *origin_addr, int origin_count, struct ompi_datatype_t *origin_dt, void *result_addr, int result_count, struct ompi_datatype_t *result_dt, int target, ptrdiff_t target_disp, int target_count, struct ompi_datatype_t *target_dt, struct ompi_op_t *op, struct ompi_win_t *win) { int ret; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t target_offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; ptrdiff_t length, origin_lb, target_lb, result_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx", (unsigned long) origin_addr, origin_count, origin_dt->name, (unsigned long) result_addr, result_count, result_dt->name, target, (unsigned long) target_disp, target_count, target_dt->name, op->o_name, (unsigned long) win)); target_offset = get_displacement(module, target) * target_disp; if (target_count > 0 && !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: MPI_REPLACE non-contiguous target")); ret = swap_from_noncontig(module, module->md_h, result_addr, result_count, result_dt, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else if (MPI_NO_OP == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: MPI_NO_OP non-contiguous target")); ret = atomic_get_from_noncontig(module, module->md_h, result_addr, result_count, result_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: other-op non-contiguous target")); ret = fetch_atomic_from_noncontig(module, module->md_h, result_addr, result_count, result_dt, module->md_h, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, op, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_from_noncontig() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } } else if ((origin_count > 0 && !ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) || (result_count > 0 && !ompi_datatype_is_contiguous_memory_layout(result_dt, result_count))) { if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: MPI_REPLACE non-contiguous origin/result")); ret = swap_to_iovec(module, result_addr, result_count, result_dt, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d swap_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else if (MPI_NO_OP == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: MPI_NO_OP non-contiguous origin/result")); ret = atomic_get_to_iovec(module, result_addr, result_count, result_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d atomic_get_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } else { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: other-op non-contiguous origin/result")); ret = fetch_atomic_to_iovec(module, result_addr, result_count, result_dt, origin_addr, origin_count, origin_dt, peer, target_count, target_dt, target_offset, module->pt_idx, module->match_bits, op, NULL); if (PTL_OK != ret) { OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d fetch_atomic_to_iovec() failed: ret = %d", __FILE__, __LINE__, ret)); return ret; } } } else { if (MPI_REPLACE == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: MPI_REPLACE contiguous")); ptl_size_t result_md_offset, origin_md_offset; ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "MPI_Get_accumulate: datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; ret = segmentedSwap(&module->opcount, module->md_h, result_md_offset + result_lb, module->md_h, origin_md_offset + origin_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, target_offset + target_lb, NULL, ptl_dt); if (OMPI_SUCCESS != ret) { return ret; } } else if (MPI_NO_OP == op) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: MPI_NO_OP contiguous")); ptl_size_t md_offset; ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(target_dt, &size); length = size * target_count; md_offset = (ptl_size_t) result_addr; OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, "%s,%d MPI_Get_accumulate", __FILE__, __LINE__)); ret = segmentedGet(&module->opcount, module->md_h, (ptl_size_t) md_offset + result_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, target_offset + target_lb, NULL); if (OMPI_SUCCESS != ret) { return ret; } } else { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: other-op contiguous")); ptl_size_t result_md_offset, origin_md_offset; ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "MPI_Get_accumulate: datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "MPI_Get_accumulate: operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } ret = segmentedFetchAtomic(&module->opcount, module->md_h, result_md_offset + result_lb, module->md_h, origin_md_offset + origin_lb, length, module->fetch_atomic_max, peer, module->pt_idx, module->match_bits, target_offset + target_lb, NULL, ptl_op, ptl_dt); if (OMPI_SUCCESS != ret) { return ret; } } } return OMPI_SUCCESS; } int ompi_osc_portals4_compare_and_swap(const void *origin_addr, const void *compare_addr, void *result_addr, struct ompi_datatype_t *dt, int target, ptrdiff_t target_disp, struct ompi_win_t *win) { int ret; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t length; size_t offset; ptl_datatype_t ptl_dt; ptl_size_t result_md_offset, origin_md_offset; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "compare_and_swap: 0x%lx, 0x%lx, 0x%lx, %s, %d, %lu, 0x%lx", (unsigned long) origin_addr, (unsigned long) compare_addr, (unsigned long) result_addr, dt->name, target, (unsigned long) target_disp, (unsigned long) win)); ret = ompi_osc_portals4_get_dt(dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "MPI_Compare_and_swap: datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } offset = get_displacement(module, target) * target_disp; ret = ompi_datatype_type_size(dt, &length); if (OMPI_SUCCESS != ret) return ret; assert(length <= module->fetch_atomic_max); result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, "%s,%d Swap", __FILE__, __LINE__)); ret = PtlSwap(module->md_h, result_md_offset, module->md_h, origin_md_offset, length, peer, module->pt_idx, module->match_bits, offset, NULL, 0, compare_addr, PTL_CSWAP, ptl_dt); if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; } int ompi_osc_portals4_fetch_and_op(const void *origin_addr, void *result_addr, struct ompi_datatype_t *dt, int target, ptrdiff_t target_disp, struct ompi_op_t *op, struct ompi_win_t *win) { int ret; ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); size_t length; size_t offset; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "fetch_and_op: 0x%lx, 0x%lx, %s, %d, %lu, %s, 0x%lx", (unsigned long) origin_addr, (unsigned long) result_addr, dt->name, target, (unsigned long) target_disp, op->o_name, (unsigned long) win)); ret = ompi_osc_portals4_get_dt(dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "MPI_Fetch_and_op: datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } offset = get_displacement(module, target) * target_disp; ret = ompi_datatype_type_size(dt, &length); if (OMPI_SUCCESS != ret) return ret; assert(length <= module->fetch_atomic_max); if (MPI_REPLACE == op) { ptl_size_t result_md_offset, origin_md_offset; result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Swap", __FILE__, __LINE__)); ret = PtlSwap(module->md_h, result_md_offset, module->md_h, origin_md_offset, length, peer, module->pt_idx, module->match_bits, offset, NULL, 0, NULL, PTL_SWAP, ptl_dt); } else if (MPI_NO_OP == op) { ptl_size_t md_offset; md_offset = (ptl_size_t) result_addr; (void)opal_atomic_add_fetch_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Get", __FILE__, __LINE__)); ret = PtlGet(module->md_h, md_offset, length, peer, module->pt_idx, module->match_bits, offset, NULL); } else { ptl_size_t result_md_offset, origin_md_offset; (void)opal_atomic_add_fetch_64(&module->opcount, 1); ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "MPI_Fetch_and_op: operation is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d FetchAtomic", __FILE__, __LINE__)); ret = PtlFetchAtomic(module->md_h, result_md_offset, module->md_h, origin_md_offset, length, peer, module->pt_idx, module->match_bits, offset, NULL, 0, ptl_op, ptl_dt); } if (OMPI_SUCCESS != ret) { return ret; } return OMPI_SUCCESS; }