/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
 * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
 *                         reserved.
 * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

#include "btl_ugni_smsg.h"
#include "btl_ugni_rdma.h"

static void mca_btl_ugni_smsg_mbox_construct(mca_btl_ugni_smsg_mbox_t *mbox)
{
    struct mca_btl_ugni_reg_t *ugni_reg = (struct mca_btl_ugni_reg_t *) mbox->super.registration;
    mca_rcache_base_registration_t *base_reg = (mca_rcache_base_registration_t *) ugni_reg;

    /* initialize mailbox attributes */
    mbox->attr.smsg_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
    mbox->attr.smsg_attr.msg_maxsize = mca_btl_ugni_component.ugni_smsg_limit;
    mbox->attr.smsg_attr.mbox_maxcredit = mca_btl_ugni_component.smsg_max_credits;
    mbox->attr.smsg_attr.mbox_offset = (uintptr_t) mbox->super.ptr - (uintptr_t) base_reg->base;
    mbox->attr.smsg_attr.msg_buffer = base_reg->base;
    mbox->attr.smsg_attr.buff_size = mca_btl_ugni_component.smsg_mbox_size;
    mbox->attr.smsg_attr.mem_hndl = ugni_reg->handle.gni_handle;
    mbox->attr.proc_name = OPAL_PROC_MY_NAME;
    mbox->attr.rmt_irq_mem_hndl = mca_btl_ugni_component.modules[0].devices[0].smsg_irq_mhndl;
}

OBJ_CLASS_INSTANCE(mca_btl_ugni_smsg_mbox_t, opal_free_list_item_t,
                   mca_btl_ugni_smsg_mbox_construct, NULL);

int mca_btl_ugni_smsg_init(mca_btl_ugni_module_t *ugni_module)
{
    gni_return_t rc;

    for (int i = 0; i < mca_btl_ugni_component.virtual_device_count; ++i) {
        rc = GNI_SmsgSetMaxRetrans(ugni_module->devices[i].dev_handle,
                                   mca_btl_ugni_component.smsg_max_retries);
        if (GNI_RC_SUCCESS != rc) {
            BTL_ERROR(("error setting maximum SMSG retries %s", gni_err_str[rc]));
            return mca_btl_rc_ugni_to_opal(rc);
        }
    }

    return OPAL_SUCCESS;
}

/* progress */
int mca_btl_ugni_smsg_process(mca_btl_base_endpoint_t *ep)
{
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl(ep);
    mca_btl_active_message_callback_t *reg;
    mca_btl_ugni_base_frag_t frag;
    bool disconnect = false;
    int32_t _tmp_value = 0;
    uintptr_t data_ptr;
    gni_return_t rc;
    uint32_t len;
    int count = 0;

    if (!opal_atomic_compare_exchange_strong_32(&ep->smsg_progressing, &_tmp_value, 1)) {
        /* already progressing (we can't support reentry here) */
        return 0;
    }

    /* per uGNI documentation we loop until the mailbox is empty */
    do {
        uint8_t tag = GNI_SMSG_ANY_TAG;

        rc = mca_btl_ugni_smsg_get_next_wtag(&ep->smsg_ep_handle, &data_ptr, &tag);
        if (GNI_RC_SUCCESS != rc) {
            if (OPAL_LIKELY(GNI_RC_NOT_DONE == rc)) {
                BTL_VERBOSE(("no smsg message waiting. rc = %s", gni_err_str[rc]));

                ep->smsg_progressing = 0;
                return count;
            }

            BTL_ERROR(("unhandled GNI_SmsgGetNextWTag error"));
            return OPAL_ERROR;
        }

        assert(0 != data_ptr);

        count++;

        BTL_VERBOSE(("got smsg fragment. tag = %d\n", tag));

        switch (tag) {
        case MCA_BTL_UGNI_TAG_SEND:

            frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0];

            tag = frag.hdr.send.lag >> 24;
            len = frag.hdr.send.lag & 0x00ffffff;

            BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag));

            reg = mca_btl_base_active_message_trigger + tag;

            mca_btl_base_segment_t seg
                = {.seg_addr.pval = (void *) ((uintptr_t) data_ptr
                                              + sizeof(mca_btl_ugni_send_frag_hdr_t)),
                   seg.seg_len = len};
            const mca_btl_base_receive_descriptor_t desc = {.endpoint = ep,
                                                            .des_segments = &seg,
                                                            .des_segment_count = 1,
                                                            .tag = tag,
                                                            .cbdata = reg->cbdata};
            reg->cbfunc(&ugni_module->super, &desc);

            break;
        case MCA_BTL_UGNI_TAG_GET_INIT:
            frag.hdr.eager_ex = ((mca_btl_ugni_eager_ex_frag_hdr_t *) data_ptr)[0];

            mca_btl_ugni_start_eager_get(ep, frag.hdr.eager_ex, NULL);
            break;
        case MCA_BTL_UGNI_TAG_RDMA_COMPLETE:
            frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0];

            if (((mca_btl_ugni_base_frag_t *) frag.hdr.rdma.ctx)->flags
                & MCA_BTL_UGNI_FRAG_SMSG_COMPLETE) {
                mca_btl_ugni_frag_complete(frag.hdr.rdma.ctx, OPAL_SUCCESS);
            } else {
                /* let the local smsg completion finish this frag */
                ((mca_btl_ugni_base_frag_t *) frag.hdr.rdma.ctx)->flags
                    &= ~MCA_BTL_UGNI_FRAG_IGNORE;
            }
            break;
        case MCA_BTL_UGNI_TAG_DISCONNECT:
            /* remote endpoint has disconnected */
            disconnect = true;
            break;
        default:
            BTL_ERROR(("unknown tag %d\n", tag));
            break;
        }

        rc = mca_btl_ugni_smsg_release(&ep->smsg_ep_handle);
        if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
            BTL_ERROR(("Smsg release failed! rc = %d", rc));
            return OPAL_ERROR;
        }
    } while (!disconnect);

    ep->smsg_progressing = 0;

    /* disconnect if we get here */
    opal_mutex_lock(&ep->lock);

    mca_btl_ugni_ep_disconnect(ep, false);

    opal_mutex_unlock(&ep->lock);

    return count;
}

static inline int mca_btl_ugni_handle_remote_smsg_overrun(mca_btl_ugni_module_t *btl)
{
    size_t endpoint_count;
    unsigned int ep_index;
    int count, rc;

    BTL_VERBOSE(("btl/ugni_component detected SMSG CQ overrun. "
                 "processing message backlog..."));

    /* we don't know which endpoint lost an smsg completion. clear the
       smsg remote cq and check all mailboxes */

    /* clear out remote cq */
    mca_btl_ugni_cq_clear(btl->devices, btl->smsg_remote_cq);

    endpoint_count = opal_pointer_array_get_size(&btl->endpoints);

    for (ep_index = 0, count = 0; ep_index < endpoint_count; ++ep_index) {
        mca_btl_base_endpoint_t *ep;

        ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item(&btl->endpoints, ep_index);

        if (NULL == ep || MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state) {
            continue;
        }

        /* clear out smsg mailbox */
        rc = mca_btl_ugni_smsg_process(ep);
        if (OPAL_LIKELY(rc >= 0)) {
            count += rc;
        }
    }

    return count;
}

int mca_btl_ugni_progress_remote_smsg(mca_btl_ugni_module_t *btl)
{
    mca_btl_base_endpoint_t *ep;
    gni_cq_entry_t event_data;
    gni_return_t grc;
    uint64_t inst_id;

    grc = mca_btl_ugni_gni_cq_get_event(btl->devices, btl->smsg_remote_cq, &event_data);
    if (GNI_RC_NOT_DONE == grc) {
        return 0;
    }

    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc || !GNI_CQ_STATUS_OK(event_data)
                      || GNI_CQ_OVERRUN(event_data))) {
        if (GNI_RC_ERROR_RESOURCE == grc || (GNI_RC_SUCCESS == grc && GNI_CQ_OVERRUN(event_data))) {
            /* recover from smsg cq overrun */
            return mca_btl_ugni_handle_remote_smsg_overrun(btl);
        }

        BTL_ERROR(("unhandled error in GNI_CqGetEvent"));

        /* unhandled error: crash */
        assert(0);
        return mca_btl_rc_ugni_to_opal(grc);
    }

    BTL_VERBOSE(("REMOTE CQ: Got event 0x%" PRIx64 ". msg id = %" PRIu64
                 ". ok = %d, type = %" PRIu64,
                 (uint64_t) event_data, GNI_CQ_GET_INST_ID(event_data),
                 GNI_CQ_STATUS_OK(event_data), GNI_CQ_GET_TYPE(event_data)));

    inst_id = GNI_CQ_GET_INST_ID(event_data);

    ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item(&btl->endpoints, inst_id);

    if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state)) {
        /* due to the nature of datagrams we may get a smsg completion before
           we get mailbox info from the peer */
        BTL_VERBOSE(("event occurred on an unconnected endpoint! ep state = %d", ep->state));
        return 0;
    }

    return mca_btl_ugni_smsg_process(ep);
}