/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2005 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2006-2007 Voltaire All rights reserved.
 * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
 * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
 * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
 * Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights
 *                         reserved.
 *
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

#include "opal_config.h"

#include <stdio.h>

#include "opal/constants.h"
#include "opal/mca/btl/base/base.h"
#include "opal/mca/btl/btl.h"
#include "opal/util/output.h"

int mca_btl_base_param_register(mca_base_component_t *version, mca_btl_base_module_t *module)
{
    /* If this is ever triggered change the uint32_ts in mca_btl_base_module_t to unsigned ints */
    assert(sizeof(unsigned int) == sizeof(uint32_t));

    (void) mca_base_component_var_register(version, "exclusivity", "BTL exclusivity (must be >= 0)",
                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
                                           OPAL_INFO_LVL_7, MCA_BASE_VAR_SCOPE_READONLY,
                                           &module->btl_exclusivity);

    (void) mca_base_component_var_register(
        version, "flags",
        "BTL bit flags (general flags: send, put, get, in-place, hetero-rdma, "
        "atomics, fetching-atomics)",
        MCA_BASE_VAR_TYPE_UNSIGNED_INT, &mca_btl_base_flag_enum->super, 0, 0, OPAL_INFO_LVL_5,
        MCA_BASE_VAR_SCOPE_READONLY, &module->btl_flags);

    (void) mca_base_component_var_register(version, "atomic_flags", "BTL atomic support flags",
                                           MCA_BASE_VAR_TYPE_UNSIGNED_INT,
                                           &mca_btl_base_atomic_enum->super, 0,
                                           MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_5,
                                           MCA_BASE_VAR_SCOPE_CONSTANT, &module->btl_atomic_flags);

    (void) mca_base_component_var_register(
        version, "rndv_eager_limit",
        "Size (in bytes, including header) of \"phase 1\" fragment sent for all large messages "
        "(must be >= 0 and <= eager_limit)",
        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
        &module->btl_rndv_eager_limit);

    (void) mca_base_component_var_register(
        version, "eager_limit",
        "Maximum size (in bytes, including header) of \"short\" messages (must be >= 1).",
        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
        &module->btl_eager_limit);

    if ((module->btl_flags & MCA_BTL_FLAGS_GET) && module->btl_get) {
        if (0 == module->btl_get_limit) {
            module->btl_get_limit = SIZE_MAX;
        }

        (void) mca_base_component_var_register(version, "get_limit",
                                               "Maximum size (in bytes) for btl get",
                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
                                               OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
                                               &module->btl_get_limit);

        /* Allow the user to set the alignment. The BTL should double-check the alignment in its
         * open function. */
        (void) mca_base_component_var_register(version, "get_alignment",
                                               "Alignment required for btl get",
                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
                                               OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_CONSTANT,
                                               &module->btl_get_alignment);
    }

    if ((module->btl_flags & MCA_BTL_FLAGS_PUT) && module->btl_put) {
        if (0 == module->btl_put_limit) {
            module->btl_put_limit = SIZE_MAX;
        }
        (void) mca_base_component_var_register(version, "put_limit",
                                               "Maximum size (in bytes) for btl put",
                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
                                               OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
                                               &module->btl_put_limit);

        /* Allow the user to set the alignment. The BTL should double-check the alignment in its
         * open function. */
        (void) mca_base_component_var_register(version, "put_alignment",
                                               "Alignment required for btl put",
                                               MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0,
                                               OPAL_INFO_LVL_6, MCA_BASE_VAR_SCOPE_CONSTANT,
                                               &module->btl_put_alignment);
    }

#if OPAL_CUDA_GDR_SUPPORT
    /* If no CUDA RDMA support, zero them out */
    if (!(MCA_BTL_FLAGS_ACCELERATOR_GET & module->btl_flags)) {
        module->btl_accelerator_eager_limit = 0;
        module->btl_accelerator_rdma_limit = SIZE_MAX;
    }
    (void) mca_base_component_var_register(
        version, "accelerator_eager_limit",
        "Maximum size (in bytes, including header) of \"GPU short\" messages (must be >= 1).",
        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
        &module->btl_accelerator_eager_limit);
    (void) mca_base_component_var_register(version, "accelerator_rdma_limit",
                                           "Size (in bytes, including header) of GPU buffer when "
                                           "switch to rndv protocol and pipeline.",
                                           MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_5,
                                           MCA_BASE_VAR_SCOPE_READONLY,
                                           &module->btl_accelerator_rdma_limit);
#endif /* OPAL_CUDA_GDR_SUPPORT */
    module->btl_accelerator_max_send_size = 0;
    (void) mca_base_component_var_register(
        version, "accelerator_max_send_size",
        "Maximum size (in bytes) of a single GPU \"phase 2\" fragment of a long message when using "
        "the pipeline protocol (must be >= 1) (only valid on smcuda btl)",
        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
        &module->btl_accelerator_max_send_size);

    (void) mca_base_component_var_register(
        version, "max_send_size",
        "Maximum size (in bytes) of a single \"phase 2\" fragment of a long message when using the "
        "pipeline protocol (must be >= 1)",
        MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
        &module->btl_max_send_size);

    if (NULL != module->btl_put) {
        (void) mca_base_component_var_register(
            version, "rdma_pipeline_send_length",
            "Length of the \"phase 2\" portion of a large message (in bytes) when using the "
            "pipeline protocol.  This part of the message will be split into fragments of size "
            "max_send_size and sent using send/receive semantics (must be >= 0; only relevant when "
            "the PUT flag is set)",
            MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
            &module->btl_rdma_pipeline_send_length);

        (void) mca_base_component_var_register(
            version, "rdma_pipeline_frag_size",
            "Maximum size (in bytes) of a single \"phase 3\" fragment from a long message when "
            "using the pipeline protocol.  These fragments will be sent using RDMA semantics (must "
            "be >= 1; only relevant when the PUT flag is set)",
            MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
            &module->btl_rdma_pipeline_frag_size);

        (void) mca_base_component_var_register(
            version, "min_rdma_pipeline_size",
            "Messages smaller than this size (in bytes) will not use the RDMA pipeline protocol.  "
            "Instead, they will be split into fragments of max_send_size and sent using "
            "send/receive semantics (must be >=0, and is automatically adjusted up to at least "
            "(eager_limit+btl_rdma_pipeline_send_length); only relevant when the PUT flag is set)",
            MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY,
            &module->btl_min_rdma_pipeline_size);

        (void) mca_base_component_var_register(
            version, "latency",
            "Approximate latency of interconnect (0 = auto-detect value at run-time [not supported "
            "in all BTL modules], >= 1 = latency in microseconds)",
            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
            MCA_BASE_VAR_SCOPE_READONLY, &module->btl_latency);
        (void) mca_base_component_var_register(
            version, "bandwidth",
            "Approximate maximum bandwidth of interconnect (0 = auto-detect value at run-time [not "
            "supported in all BTL modules], >= 1 = bandwidth in Mbps)",
            MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
            MCA_BASE_VAR_SCOPE_READONLY, &module->btl_bandwidth);
    }

    return mca_btl_base_param_verify(module);
}

/* Verify btl parameters make sense */
int mca_btl_base_param_verify(mca_btl_base_module_t *module)
{
    if (module->btl_min_rdma_pipeline_size
        < (module->btl_eager_limit + module->btl_rdma_pipeline_send_length)) {
        module->btl_min_rdma_pipeline_size = module->btl_eager_limit
                                             + module->btl_rdma_pipeline_send_length;
    }

    if (NULL == module->btl_put) {
        module->btl_flags &= ~MCA_BTL_FLAGS_PUT;
    }

    if (NULL == module->btl_get) {
        module->btl_flags &= ~MCA_BTL_FLAGS_GET;
    }

    if (NULL == module->btl_flush) {
        module->btl_flags &= ~MCA_BTL_FLAGS_RDMA_FLUSH;
    }

    if (0 == module->btl_atomic_flags) {
        module->btl_flags &= ~MCA_BTL_FLAGS_ATOMIC_OPS;
    }

    if (0 == module->btl_get_limit) {
        module->btl_get_limit = SIZE_MAX;
    }

    if (0 == module->btl_put_limit) {
        module->btl_put_limit = SIZE_MAX;
    }

    return OPAL_SUCCESS;
}