Program Listing for File EventCount.h

↰ Return to documentation for file (ADS/concurrent/EventCount.h)

/*--------------------------------------------------------------------------*/
/*--------------------------- File EventCount.h ----------------------------*/
/*--------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
/*----------------------------- DEFINITIONS --------------------------------*/
/*--------------------------------------------------------------------------*/

#ifndef __ADS_EventCount
 #define __ADS_EventCount
                      /* self-identification: #endif constant */

#if defined( __SANITIZE_THREAD__ )
 #define ADS_EVENTCOUNT_RMW
#elif defined( __has_feature )
 #if __has_feature( thread_sanitizer )
  #define ADS_EVENTCOUNT_RMW
 #endif
#endif

/*--------------------------------------------------------------------------*/
/*------------------------------ INCLUDES ----------------------------------*/
/*--------------------------------------------------------------------------*/

#include <atomic>
#include <chrono>
#include <cstdint>

#if defined( __linux__ ) && ! defined( ADS_EVENTCOUNT_RMW ) && \
    __has_include( <linux/membarrier.h> )
 #define ADS_EVENTCOUNT_MEMBARRIER
 #include <linux/membarrier.h>
 #include <sys/syscall.h>
 #include <unistd.h>
#endif

/*--------------------------------------------------------------------------*/
/*--------------------------- NAMESPACE ------------------------------------*/
/*--------------------------------------------------------------------------*/

namespace ADS {

namespace detail {

inline bool asymmetricFences( void ) noexcept
{
#ifdef ADS_EVENTCOUNT_MEMBARRIER
 static const bool ok = [] {
  const long q = syscall( SYS_membarrier , MEMBARRIER_CMD_QUERY , 0 );
  return( ( q >= 0 ) && ( q & MEMBARRIER_CMD_PRIVATE_EXPEDITED ) &&
          ( syscall( SYS_membarrier ,
                     MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED , 0 ) == 0 ) );
  }();
 return( ok );
#else
 return( false );
#endif
 }

inline void lightFence( void ) noexcept
{
 if( asymmetricFences() )
  std::atomic_signal_fence( std::memory_order_seq_cst );
 else
  std::atomic_thread_fence( std::memory_order_seq_cst );
 }

inline void heavyFence( void ) noexcept
{
#ifdef ADS_EVENTCOUNT_MEMBARRIER
 if( asymmetricFences() &&
     ( syscall( SYS_membarrier , MEMBARRIER_CMD_PRIVATE_EXPEDITED , 0 ) ==
       0 ) )
  return;
#endif
 std::atomic_thread_fence( std::memory_order_seq_cst );
 }

}  // end( namespace detail )

/*--------------------------------------------------------------------------*/
/*--------------------------- CLASS EventCount -----------------------------*/
/*--------------------------------------------------------------------------*/

class EventCount {

/*--------------------------------------------------------------------------*/
 public:

 using Key = std::uint32_t;

 EventCount( void ) = default;
 EventCount( const EventCount & ) = delete;
 EventCount & operator=( const EventCount & ) = delete;

 [[nodiscard]] Key prepareWait( void ) noexcept {
  const std::uint64_t v = f_state.fetch_add( 1 , std::memory_order_seq_cst );
#ifndef ADS_EVENTCOUNT_RMW
  detail::heavyFence();
#endif
  return( Key( v >> 32 ) );
  }

 void cancelWait( Key k ) noexcept {
  std::uint64_t v = f_state.load( std::memory_order_relaxed );
  while( ( Key( v >> 32 ) == k ) &&
         ! f_state.compare_exchange_weak( v , v - 1 ,
                                          std::memory_order_relaxed ) ) {}
  }

 void commitWait( Key k ) noexcept {
  while( f_epoch.load( std::memory_order_acquire ) == k )
   f_epoch.wait( k , std::memory_order_acquire );
  }

 void notify( void ) noexcept {
#ifndef ADS_EVENTCOUNT_RMW
  detail::lightFence();
  std::uint64_t v = f_state.load( std::memory_order_relaxed );
#else
  std::uint64_t v = f_state.fetch_add( 0 , std::memory_order_seq_cst );
#endif
  while( v & waiterMask ) [[unlikely]]
   if( f_state.compare_exchange_weak( v , ( ( v >> 32 ) + 1 ) << 32 ,
                                      std::memory_order_acq_rel ,
                                      std::memory_order_relaxed ) ) {
    f_epoch.store( Key( ( v >> 32 ) + 1 ) , std::memory_order_release );
    f_epoch.notify_all();
    return;
    }
  }

 template< class P >
 void await( P && p , std::int64_t spin = 20000 ) {
  if( p() )
   return;
  const auto end = std::chrono::steady_clock::now() +
                   std::chrono::nanoseconds( spin );
  for( unsigned i = 1 ; ; ++i ) {           // the clock every 64 looks
   relax();
   if( p() )
    return;
   if( ( ( i & 63 ) == 0 ) && ( std::chrono::steady_clock::now() >= end ) )
    break;
   }
  while( ! p() ) {
   const Key k = prepareWait();
   if( p() ) {
    cancelWait( k );
    return;
    }
   commitWait( k );
   }
  }

 [[nodiscard]] std::uint32_t waiters( void ) const noexcept {
  return( std::uint32_t( f_state.load( std::memory_order_relaxed ) &
                         waiterMask ) );
  }

/*--------------------------------------------------------------------------*/
 private:

 static void relax( void ) noexcept {
#if defined( __x86_64__ ) || defined( __i386__ )
  __builtin_ia32_pause();
#elif defined( __aarch64__ )
  asm volatile( "yield" );
#endif
  }

 static constexpr std::uint64_t waiterMask = 0xFFFFFFFFu;

 alignas( 64 ) std::atomic< std::uint64_t > f_state{ 0 };
 std::atomic< Key > f_epoch{ 0 };
 [[maybe_unused]]
 char f_pad[ 64 - sizeof( std::uint64_t ) - sizeof( Key ) ] = {};

 };  // end( class EventCount )

/*--------------------------------------------------------------------------*/

}  // end( namespace ADS )

/*--------------------------------------------------------------------------*/

#endif  /* EventCount.h included */

/*--------------------------------------------------------------------------*/
/*------------------------- End File EventCount.h --------------------------*/
/*--------------------------------------------------------------------------*/