/* * Copyright (c) 2018-2020 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef OMPI_SPC #define OMPI_SPC #include "ompi_config.h" #include "opal/sys/atomic.h" #include "opal/include/opal/prefetch.h" #include "opal/mca/threads/thread_usage.h" #include MCA_timer_IMPLEMENTATION_HEADER /* INSTRUCTIONS FOR ADDING COUNTERS * 1.) Add a new counter name in the ompi_spc_counters_t enum before * OMPI_SPC_NUM_COUNTERS below. * 2.) Add corresponding counter name and descriptions to the * counter_names and counter_descriptions arrays in * ompi_spc.c NOTE: The names and descriptions * MUST be in the same array location as where you added the * counter name in step 1. * 3.) If this counter is based on a timer, add its enum name to * the logic for timer-based counters in the ompi_spc_init * function in ompi_spc.c * 4.) Instrument the Open MPI code base where it makes sense for * your counter to be modified using the SPC_RECORD macro. * Note: If your counter is timer-based you should use the * SPC_TIMER_START and SPC_TIMER_STOP macros to record * the time in cycles to then be converted to microseconds later * in the ompi_spc_get_count function when requested by MPI_T */ /* This enumeration serves as event ids for the various events */ typedef enum ompi_spc_counters { OMPI_SPC_SEND, OMPI_SPC_BSEND, OMPI_SPC_RSEND, OMPI_SPC_SSEND, OMPI_SPC_RECV, OMPI_SPC_MRECV, OMPI_SPC_ISEND, OMPI_SPC_IBSEND, OMPI_SPC_IRSEND, OMPI_SPC_ISSEND, OMPI_SPC_IRECV, OMPI_SPC_SENDRECV, OMPI_SPC_SENDRECV_REPLACE, OMPI_SPC_PUT, OMPI_SPC_RPUT, OMPI_SPC_GET, OMPI_SPC_RGET, OMPI_SPC_PROBE, OMPI_SPC_IPROBE, OMPI_SPC_BCAST, OMPI_SPC_IBCAST, OMPI_SPC_BCAST_INIT, OMPI_SPC_REDUCE, OMPI_SPC_REDUCE_SCATTER, OMPI_SPC_REDUCE_SCATTER_BLOCK, OMPI_SPC_IREDUCE, OMPI_SPC_IREDUCE_SCATTER, OMPI_SPC_IREDUCE_SCATTER_BLOCK, OMPI_SPC_REDUCE_INIT, OMPI_SPC_REDUCE_SCATTER_INIT, OMPI_SPC_REDUCE_SCATTER_BLOCK_INIT, OMPI_SPC_ALLREDUCE, OMPI_SPC_IALLREDUCE, OMPI_SPC_ALLREDUCE_INIT, OMPI_SPC_SCAN, OMPI_SPC_EXSCAN, OMPI_SPC_ISCAN, OMPI_SPC_IEXSCAN, OMPI_SPC_SCAN_INIT, OMPI_SPC_EXSCAN_INIT, OMPI_SPC_SCATTER, OMPI_SPC_SCATTERV, OMPI_SPC_ISCATTER, OMPI_SPC_ISCATTERV, OMPI_SPC_SCATTER_INIT, OMPI_SPC_SCATTERV_INIT, OMPI_SPC_GATHER, OMPI_SPC_GATHERV, OMPI_SPC_IGATHER, OMPI_SPC_IGATHERV, OMPI_SPC_GATHER_INIT, OMPI_SPC_GATHERV_INIT, OMPI_SPC_ALLTOALL, OMPI_SPC_ALLTOALLV, OMPI_SPC_ALLTOALLW, OMPI_SPC_IALLTOALL, OMPI_SPC_IALLTOALLV, OMPI_SPC_IALLTOALLW, OMPI_SPC_ALLTOALL_INIT, OMPI_SPC_ALLTOALLV_INIT, OMPI_SPC_ALLTOALLW_INIT, OMPI_SPC_NEIGHBOR_ALLTOALL, OMPI_SPC_NEIGHBOR_ALLTOALLV, OMPI_SPC_NEIGHBOR_ALLTOALLW, OMPI_SPC_INEIGHBOR_ALLTOALL, OMPI_SPC_INEIGHBOR_ALLTOALLV, OMPI_SPC_INEIGHBOR_ALLTOALLW, OMPI_SPC_NEIGHBOR_ALLTOALL_INIT, OMPI_SPC_NEIGHBOR_ALLTOALLV_INIT, OMPI_SPC_NEIGHBOR_ALLTOALLW_INIT, OMPI_SPC_ALLGATHER, OMPI_SPC_ALLGATHERV, OMPI_SPC_IALLGATHER, OMPI_SPC_IALLGATHERV, OMPI_SPC_ALLGATHER_INIT, OMPI_SPC_ALLGATHERV_INIT, OMPI_SPC_NEIGHBOR_ALLGATHER, OMPI_SPC_NEIGHBOR_ALLGATHERV, OMPI_SPC_INEIGHBOR_ALLGATHER, OMPI_SPC_INEIGHBOR_ALLGATHERV, OMPI_SPC_NEIGHBOR_ALLGATHER_INIT, OMPI_SPC_NEIGHBOR_ALLGATHERV_INIT, OMPI_SPC_TEST, OMPI_SPC_TESTALL, OMPI_SPC_TESTANY, OMPI_SPC_TESTSOME, OMPI_SPC_WAIT, OMPI_SPC_WAITALL, OMPI_SPC_WAITANY, OMPI_SPC_WAITSOME, OMPI_SPC_BARRIER, OMPI_SPC_IBARRIER, OMPI_SPC_BARRIER_INIT, OMPI_SPC_WTIME, OMPI_SPC_CANCEL, OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI, OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI, OMPI_SPC_BYTES_PUT, OMPI_SPC_BYTES_GET, OMPI_SPC_UNEXPECTED, OMPI_SPC_OUT_OF_SEQUENCE, OMPI_SPC_MATCH_TIME, OMPI_SPC_UNEXPECTED_IN_QUEUE, OMPI_SPC_OOS_IN_QUEUE, OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE, OMPI_SPC_MAX_OOS_IN_QUEUE, OMPI_SPC_ISENDRECV, OMPI_SPC_ISENDRECV_REPLACE, OMPI_SPC_PARRIVED, OMPI_SPC_PREADY, OMPI_SPC_NUM_COUNTERS /* This serves as the number of counters. It must be last. */ } ompi_spc_counters_t; /* There is currently no support for atomics on long long values so we will default to * size_t for now until support for such atomics is implemented. */ typedef long long ompi_spc_value_t; /* A structure for storing the event data */ typedef struct ompi_spc_s{ opal_atomic_int64_t value; opal_atomic_int32_t num_attached; bool is_high_watermark; bool is_timer_event; } ompi_spc_t; /* Definitions for using the SPC utility functions throughout the codebase. * If SPC_ENABLE is not 1, the macros become no-ops. */ #if SPC_ENABLE == 1 /* OMPI SPC utility functions */ void ompi_spc_init(void); void ompi_spc_fini(void); void ompi_spc_cycles_to_usecs(opal_timer_t *cycles); /* An array of event structures to store the event data value, attachments, flags) * The memory is statically allocated to reduce the number of loads required. */ OPAL_DECLSPEC extern ompi_spc_t ompi_spc_events[OMPI_SPC_NUM_COUNTERS] __opal_attribute_aligned__(sizeof(ompi_spc_t)); #define SPC_INIT() \ ompi_spc_init() #define SPC_FINI() \ ompi_spc_fini() #define SPC_RECORD(event_id, value) \ ompi_spc_record(event_id, value) #define SPC_TIMER_START(event_id, usec) \ ompi_spc_timer_start(event_id, usec) #define SPC_TIMER_STOP(event_id, usec) \ ompi_spc_timer_stop(event_id, usec) #define SPC_USER_OR_MPI(tag, value, enum_if_user, enum_if_mpi) \ ompi_spc_user_or_mpi(tag, value, enum_if_user, enum_if_mpi) #define SPC_CYCLES_TO_USECS(cycles) \ ompi_spc_cycles_to_usecs(cycles) #define SPC_UPDATE_WATERMARK(watermark_enum, value_enum) \ ompi_spc_update_watermark(watermark_enum, value_enum) /* Records an update to a counter using an atomic add operation. */ static inline void ompi_spc_record(unsigned int event_id, ompi_spc_value_t value) { /* Denoted unlikely because counters will often be turned off. */ if( ompi_spc_events[event_id].num_attached > 0 ) { OPAL_THREAD_ADD_FETCH64(&(ompi_spc_events[event_id].value), value); } } /* Checks a tag, and records the user version of the counter if it's greater * than or equal to 0 and records the mpi version of the counter otherwise. */ static inline void ompi_spc_user_or_mpi(int tag, ompi_spc_value_t value, unsigned int user_enum, unsigned int mpi_enum) { ompi_spc_record( (tag >= 0 ? user_enum : mpi_enum), value); } /* Checks whether the counter denoted by value_enum exceeds the current value of the * counter denoted by watermark_enum, and if so sets the watermark_enum counter to the * value of the value_enum counter. */ static inline void ompi_spc_update_watermark(unsigned int watermark_enum, unsigned int value_enum) { ompi_spc_t *watermark_event = &ompi_spc_events[watermark_enum]; ompi_spc_t *value_event = &ompi_spc_events[value_enum]; /* Denoted unlikely because counters will often be turned off. */ if( watermark_event->num_attached && value_event->num_attached ) { int64_t watermark = watermark_event->value; int64_t value = watermark_event->value; /* Try to atomically replace the watermark while the value is larger * (i.e, while no thread has replaced it with a larger value, including this thread) */ while (value > watermark && !OPAL_THREAD_COMPARE_EXCHANGE_STRONG_64(&watermark_event->value, &watermark, value)) { } } } /* * Starts cycle-precision timer and stores the start value in the 'cycles' argument. * The value is always stored in 'cycles' to avoid race conditions with other threads * activating a previously inactive timer counter in between start and stop. */ static inline void ompi_spc_timer_start(unsigned int event_id, opal_timer_t *cycles) { (void)event_id; /* unused */ *cycles = 0; if( (ompi_spc_events[event_id].num_attached > 0) ) { *cycles = opal_timer_base_get_cycles(); } } /* Stops a cycle-precision timer and calculates the total elapsed time * based on the starting time in 'cycles' and stores the result in the * 'cycles' argument. */ static inline void ompi_spc_timer_stop(unsigned int event_id, opal_timer_t *cycles) { if( ompi_spc_events[event_id].num_attached > 0 && *cycles > 0 ) { *cycles = opal_timer_base_get_cycles() - *cycles; OPAL_THREAD_ADD_FETCH64(&ompi_spc_events[event_id].value, *cycles); } } #else /* SPCs are not enabled */ #define SPC_INIT() \ ((void)0) #define SPC_FINI() \ ((void)0) #define SPC_RECORD(event_id, value) \ ((void)0) #define SPC_TIMER_START(event_id, usec) \ ((void)0) #define SPC_TIMER_STOP(event_id, usec) \ ((void)0) #define SPC_USER_OR_MPI(tag, value, enum_if_user, enum_if_mpi) \ ((void)0) #define SPC_CYCLES_TO_USECS(cycles) \ ((void)0) #define SPC_UPDATE_WATERMARK(watermark_enum, value_enum) \ ((void)0) #endif #endif