BVB Source Codes

mars Show ops_msvc_x86.hpp Source code

Return Download mars: download ops_msvc_x86.hpp Source code - Download mars Source code - Type:.hpp
  1. /*
  2.  * Distributed under the Boost Software License, Version 1.0.
  3.  * (See accompanying file LICENSE_1_0.txt or copy at
  4.  * http://www.boost.org/LICENSE_1_0.txt)
  5.  *
  6.  * Copyright (c) 2009 Helge Bahmann
  7.  * Copyright (c) 2012 Tim Blechmann
  8.  * Copyright (c) 2014 Andrey Semashev
  9.  */
  10. /*!
  11.  * \file   atomic/detail/ops_msvc_x86.hpp
  12.  *
  13.  * This header contains implementation of the \c operations template.
  14.  */
  15.  
  16. #ifndef BOOST_ATOMIC_DETAIL_OPS_MSVC_X86_HPP_INCLUDED_
  17. #define BOOST_ATOMIC_DETAIL_OPS_MSVC_X86_HPP_INCLUDED_
  18.  
  19. #include <boost/memory_order.hpp>
  20. #include <boost/type_traits/make_signed.hpp>
  21. #include <boost/atomic/detail/config.hpp>
  22. #include <boost/atomic/detail/interlocked.hpp>
  23. #include <boost/atomic/detail/storage_type.hpp>
  24. #include <boost/atomic/detail/operations_fwd.hpp>
  25. #include <boost/atomic/capabilities.hpp>
  26. #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B) || defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
  27. #include <boost/cstdint.hpp>
  28. #include <boost/atomic/detail/ops_cas_based.hpp>
  29. #endif
  30. #include <boost/atomic/detail/ops_msvc_common.hpp>
  31. #if !defined(_M_IX86) && !(defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8) && defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16))
  32. #include <boost/atomic/detail/ops_extending_cas_based.hpp>
  33. #endif
  34.  
  35. #ifdef BOOST_HAS_PRAGMA_ONCE
  36. #pragma once
  37. #endif
  38.  
  39. #if defined(BOOST_MSVC)
  40. #pragma warning(push)
  41. // frame pointer register 'ebx' modified by inline assembly code. See the note below.
  42. #pragma warning(disable: 4731)
  43. #endif
  44.  
  45. #if defined(_MSC_VER) && (defined(_M_AMD64) || (defined(_M_IX86) && defined(_M_IX86_FP) && _M_IX86_FP >= 2))
  46. extern "C" void _mm_mfence(void);
  47. #if defined(BOOST_MSVC)
  48. #pragma intrinsic(_mm_mfence)
  49. #endif
  50. #endif
  51.  
  52. namespace mars_boost {} namespace boost = mars_boost; namespace mars_boost {
  53. namespace atomics {
  54. namespace detail {
  55.  
  56. /*
  57.  * Implementation note for asm blocks.
  58.  *
  59.  * http://msdn.microsoft.com/en-us/data/k1a8ss06%28v=vs.105%29
  60.  *
  61.  * Some SSE types require eight-byte stack alignment, forcing the compiler to emit dynamic stack-alignment code.
  62.  * To be able to access both the local variables and the function parameters after the alignment, the compiler
  63.  * maintains two frame pointers. If the compiler performs frame pointer omission (FPO), it will use EBP and ESP.
  64.  * If the compiler does not perform FPO, it will use EBX and EBP. To ensure code runs correctly, do not modify EBX
  65.  * in asm code if the function requires dynamic stack alignment as it could modify the frame pointer.
  66.  * Either move the eight-byte aligned types out of the function, or avoid using EBX.
  67.  *
  68.  * Since we have no way of knowing that the compiler uses FPO, we have to always save and restore ebx
  69.  * whenever we have to clobber it. Additionally, we disable warning C4731 above so that the compiler
  70.  * doesn't spam about ebx use.
  71.  */
  72.  
  73. struct msvc_x86_operations_base
  74. {
  75.     static BOOST_FORCEINLINE void hardware_full_fence() BOOST_NOEXCEPT
  76.     {
  77. #if defined(_MSC_VER) && (defined(_M_AMD64) || (defined(_M_IX86) && defined(_M_IX86_FP) && _M_IX86_FP >= 2))
  78.         // Use mfence only if SSE2 is available
  79.         _mm_mfence();
  80. #else
  81.         long tmp;
  82.         BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&tmp, 0);
  83. #endif
  84.     }
  85.  
  86.     static BOOST_FORCEINLINE void fence_before(memory_order) BOOST_NOEXCEPT
  87.     {
  88.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  89.     }
  90.  
  91.     static BOOST_FORCEINLINE void fence_after(memory_order) BOOST_NOEXCEPT
  92.     {
  93.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  94.     }
  95.  
  96.     static BOOST_FORCEINLINE void fence_after_load(memory_order) BOOST_NOEXCEPT
  97.     {
  98.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  99.  
  100.         // On x86 and x86_64 there is no need for a hardware barrier,
  101.         // even if seq_cst memory order is requested, because all
  102.         // seq_cst writes are implemented with lock-prefixed operations
  103.         // or xchg which has implied lock prefix. Therefore normal loads
  104.         // are already ordered with seq_cst stores on these architectures.
  105.     }
  106. };
  107.  
  108. template< typename T, typename Derived >
  109. struct msvc_x86_operations :
  110.     public msvc_x86_operations_base
  111. {
  112.     typedef T storage_type;
  113.  
  114.     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  115.     {
  116.         if (order != memory_order_seq_cst)
  117.         {
  118.             fence_before(order);
  119.             storage = v;
  120.             fence_after(order);
  121.         }
  122.         else
  123.         {
  124.             Derived::exchange(storage, v, order);
  125.         }
  126.     }
  127.  
  128.     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
  129.     {
  130.         storage_type v = storage;
  131.         fence_after_load(order);
  132.         return v;
  133.     }
  134.  
  135.     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  136.     {
  137.         typedef typename make_signed< storage_type >::type signed_storage_type;
  138.         return Derived::fetch_add(storage, static_cast< storage_type >(-static_cast< signed_storage_type >(v)), order);
  139.     }
  140.  
  141.     static BOOST_FORCEINLINE bool compare_exchange_weak(
  142.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
  143.     {
  144.         return Derived::compare_exchange_strong(storage, expected, desired, success_order, failure_order);
  145.     }
  146.  
  147.     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  148.     {
  149.         return !!Derived::exchange(storage, (storage_type)1, order);
  150.     }
  151.  
  152.     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  153.     {
  154.         store(storage, (storage_type)0, order);
  155.     }
  156.  
  157.     static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
  158.     {
  159.         return true;
  160.     }
  161. };
  162.  
  163. template< bool Signed >
  164. struct operations< 4u, Signed > :
  165.     public msvc_x86_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > >
  166. {
  167.     typedef msvc_x86_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > > base_type;
  168.     typedef typename base_type::storage_type storage_type;
  169.     typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type;
  170.  
  171.     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  172.     {
  173.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD(&storage, v));
  174.     }
  175.  
  176.     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  177.     {
  178.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&storage, v));
  179.     }
  180.  
  181.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  182.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
  183.     {
  184.         storage_type previous = expected;
  185.         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE(&storage, desired, previous));
  186.         expected = old_val;
  187.         return (previous == old_val);
  188.     }
  189.  
  190. #if defined(BOOST_ATOMIC_INTERLOCKED_AND)
  191.     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  192.     {
  193.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND(&storage, v));
  194.     }
  195. #else
  196.     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  197.     {
  198.         storage_type res = storage;
  199.         while (!compare_exchange_strong(storage, res, res & v, order, memory_order_relaxed)) {}
  200.         return res;
  201.     }
  202. #endif
  203.  
  204. #if defined(BOOST_ATOMIC_INTERLOCKED_OR)
  205.     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  206.     {
  207.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR(&storage, v));
  208.     }
  209. #else
  210.     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  211.     {
  212.         storage_type res = storage;
  213.         while (!compare_exchange_strong(storage, res, res | v, order, memory_order_relaxed)) {}
  214.         return res;
  215.     }
  216. #endif
  217.  
  218. #if defined(BOOST_ATOMIC_INTERLOCKED_XOR)
  219.     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  220.     {
  221.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR(&storage, v));
  222.     }
  223. #else
  224.     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  225.     {
  226.         storage_type res = storage;
  227.         while (!compare_exchange_strong(storage, res, res ^ v, order, memory_order_relaxed)) {}
  228.         return res;
  229.     }
  230. #endif
  231. };
  232.  
  233. #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8)
  234.  
  235. template< bool Signed >
  236. struct operations< 1u, Signed > :
  237.     public msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > >
  238. {
  239.     typedef msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > > base_type;
  240.     typedef typename base_type::storage_type storage_type;
  241.     typedef typename make_storage_type< 1u, Signed >::aligned aligned_storage_type;
  242.  
  243.     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  244.     {
  245.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD8(&storage, v));
  246.     }
  247.  
  248.     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  249.     {
  250.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE8(&storage, v));
  251.     }
  252.  
  253.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  254.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
  255.     {
  256.         storage_type previous = expected;
  257.         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8(&storage, desired, previous));
  258.         expected = old_val;
  259.         return (previous == old_val);
  260.     }
  261.  
  262.     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  263.     {
  264.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND8(&storage, v));
  265.     }
  266.  
  267.     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  268.     {
  269.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR8(&storage, v));
  270.     }
  271.  
  272.     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  273.     {
  274.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR8(&storage, v));
  275.     }
  276. };
  277.  
  278. #elif defined(_M_IX86)
  279.  
  280. template< bool Signed >
  281. struct operations< 1u, Signed > :
  282.     public msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > >
  283. {
  284.     typedef msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > > base_type;
  285.     typedef typename base_type::storage_type storage_type;
  286.     typedef typename make_storage_type< 1u, Signed >::aligned aligned_storage_type;
  287.  
  288.     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  289.     {
  290.         base_type::fence_before(order);
  291.         __asm
  292.         {
  293.             mov edx, storage
  294.             movzx eax, v
  295.             lock xadd byte ptr [edx], al
  296.             mov v, al
  297.         };
  298.         base_type::fence_after(order);
  299.         return v;
  300.     }
  301.  
  302.     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  303.     {
  304.         base_type::fence_before(order);
  305.         __asm
  306.         {
  307.             mov edx, storage
  308.             movzx eax, v
  309.             xchg byte ptr [edx], al
  310.             mov v, al
  311.         };
  312.         base_type::fence_after(order);
  313.         return v;
  314.     }
  315.  
  316.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  317.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order) BOOST_NOEXCEPT
  318.     {
  319.         base_type::fence_before(success_order);
  320.         bool success;
  321.         __asm
  322.         {
  323.             mov esi, expected
  324.             mov edi, storage
  325.             movzx eax, byte ptr [esi]
  326.             movzx edx, desired
  327.             lock cmpxchg byte ptr [edi], dl
  328.             mov byte ptr [esi], al
  329.             sete success
  330.         };
  331.         // The success and failure fences are equivalent anyway
  332.         base_type::fence_after(success_order);
  333.         return success;
  334.     }
  335.  
  336.     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  337.     {
  338.         base_type::fence_before(order);
  339.         int backup;
  340.         __asm
  341.         {
  342.             mov backup, ebx
  343.             xor edx, edx
  344.             mov edi, storage
  345.             movzx ebx, v
  346.             movzx eax, byte ptr [edi]
  347.             align 16
  348.         again:
  349.             mov dl, al
  350.             and dl, bl
  351.             lock cmpxchg byte ptr [edi], dl
  352.             jne again
  353.             mov v, al
  354.             mov ebx, backup
  355.         };
  356.         base_type::fence_after(order);
  357.         return v;
  358.     }
  359.  
  360.     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  361.     {
  362.         base_type::fence_before(order);
  363.         int backup;
  364.         __asm
  365.         {
  366.             mov backup, ebx
  367.             xor edx, edx
  368.             mov edi, storage
  369.             movzx ebx, v
  370.             movzx eax, byte ptr [edi]
  371.             align 16
  372.         again:
  373.             mov dl, al
  374.             or dl, bl
  375.             lock cmpxchg byte ptr [edi], dl
  376.             jne again
  377.             mov v, al
  378.             mov ebx, backup
  379.         };
  380.         base_type::fence_after(order);
  381.         return v;
  382.     }
  383.  
  384.     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  385.     {
  386.         base_type::fence_before(order);
  387.         int backup;
  388.         __asm
  389.         {
  390.             mov backup, ebx
  391.             xor edx, edx
  392.             mov edi, storage
  393.             movzx ebx, v
  394.             movzx eax, byte ptr [edi]
  395.             align 16
  396.         again:
  397.             mov dl, al
  398.             xor dl, bl
  399.             lock cmpxchg byte ptr [edi], dl
  400.             jne again
  401.             mov v, al
  402.             mov ebx, backup
  403.         };
  404.         base_type::fence_after(order);
  405.         return v;
  406.     }
  407. };
  408.  
  409. #else
  410.  
  411. template< bool Signed >
  412. struct operations< 1u, Signed > :
  413.     public extending_cas_based_operations< operations< 4u, Signed >, 1u, Signed >
  414. {
  415. };
  416.  
  417. #endif
  418.  
  419. #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16)
  420.  
  421. template< bool Signed >
  422. struct operations< 2u, Signed > :
  423.     public msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > >
  424. {
  425.     typedef msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > > base_type;
  426.     typedef typename base_type::storage_type storage_type;
  427.     typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type;
  428.  
  429.     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  430.     {
  431.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD16(&storage, v));
  432.     }
  433.  
  434.     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  435.     {
  436.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE16(&storage, v));
  437.     }
  438.  
  439.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  440.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
  441.     {
  442.         storage_type previous = expected;
  443.         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16(&storage, desired, previous));
  444.         expected = old_val;
  445.         return (previous == old_val);
  446.     }
  447.  
  448.     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  449.     {
  450.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND16(&storage, v));
  451.     }
  452.  
  453.     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  454.     {
  455.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR16(&storage, v));
  456.     }
  457.  
  458.     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  459.     {
  460.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR16(&storage, v));
  461.     }
  462. };
  463.  
  464. #elif defined(_M_IX86)
  465.  
  466. template< bool Signed >
  467. struct operations< 2u, Signed > :
  468.     public msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > >
  469. {
  470.     typedef msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > > base_type;
  471.     typedef typename base_type::storage_type storage_type;
  472.     typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type;
  473.  
  474.     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  475.     {
  476.         base_type::fence_before(order);
  477.         __asm
  478.         {
  479.             mov edx, storage
  480.             movzx eax, v
  481.             lock xadd word ptr [edx], ax
  482.             mov v, ax
  483.         };
  484.         base_type::fence_after(order);
  485.         return v;
  486.     }
  487.  
  488.     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  489.     {
  490.         base_type::fence_before(order);
  491.         __asm
  492.         {
  493.             mov edx, storage
  494.             movzx eax, v
  495.             xchg word ptr [edx], ax
  496.             mov v, ax
  497.         };
  498.         base_type::fence_after(order);
  499.         return v;
  500.     }
  501.  
  502.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  503.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order) BOOST_NOEXCEPT
  504.     {
  505.         base_type::fence_before(success_order);
  506.         bool success;
  507.         __asm
  508.         {
  509.             mov esi, expected
  510.             mov edi, storage
  511.             movzx eax, word ptr [esi]
  512.             movzx edx, desired
  513.             lock cmpxchg word ptr [edi], dx
  514.             mov word ptr [esi], ax
  515.             sete success
  516.         };
  517.         // The success and failure fences are equivalent anyway
  518.         base_type::fence_after(success_order);
  519.         return success;
  520.     }
  521.  
  522.     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  523.     {
  524.         base_type::fence_before(order);
  525.         int backup;
  526.         __asm
  527.         {
  528.             mov backup, ebx
  529.             xor edx, edx
  530.             mov edi, storage
  531.             movzx ebx, v
  532.             movzx eax, word ptr [edi]
  533.             align 16
  534.         again:
  535.             mov dx, ax
  536.             and dx, bx
  537.             lock cmpxchg word ptr [edi], dx
  538.             jne again
  539.             mov v, ax
  540.             mov ebx, backup
  541.         };
  542.         base_type::fence_after(order);
  543.         return v;
  544.     }
  545.  
  546.     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  547.     {
  548.         base_type::fence_before(order);
  549.         int backup;
  550.         __asm
  551.         {
  552.             mov backup, ebx
  553.             xor edx, edx
  554.             mov edi, storage
  555.             movzx ebx, v
  556.             movzx eax, word ptr [edi]
  557.             align 16
  558.         again:
  559.             mov dx, ax
  560.             or dx, bx
  561.             lock cmpxchg word ptr [edi], dx
  562.             jne again
  563.             mov v, ax
  564.             mov ebx, backup
  565.         };
  566.         base_type::fence_after(order);
  567.         return v;
  568.     }
  569.  
  570.     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  571.     {
  572.         base_type::fence_before(order);
  573.         int backup;
  574.         __asm
  575.         {
  576.             mov backup, ebx
  577.             xor edx, edx
  578.             mov edi, storage
  579.             movzx ebx, v
  580.             movzx eax, word ptr [edi]
  581.             align 16
  582.         again:
  583.             mov dx, ax
  584.             xor dx, bx
  585.             lock cmpxchg word ptr [edi], dx
  586.             jne again
  587.             mov v, ax
  588.             mov ebx, backup
  589.         };
  590.         base_type::fence_after(order);
  591.         return v;
  592.     }
  593. };
  594.  
  595. #else
  596.  
  597. template< bool Signed >
  598. struct operations< 2u, Signed > :
  599.     public extending_cas_based_operations< operations< 4u, Signed >, 2u, Signed >
  600. {
  601. };
  602.  
  603. #endif
  604.  
  605.  
  606. #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B)
  607.  
  608. template< bool Signed >
  609. struct msvc_dcas_x86
  610. {
  611.     typedef typename make_storage_type< 8u, Signed >::type storage_type;
  612.     typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
  613.  
  614.     // Intel 64 and IA-32 Architectures Software Developer's Manual, Volume 3A, 8.1.1. Guaranteed Atomic Operations:
  615.     //
  616.     // The Pentium processor (and newer processors since) guarantees that the following additional memory operations will always be carried out atomically:
  617.     // * Reading or writing a quadword aligned on a 64-bit boundary
  618.     //
  619.     // Luckily, the memory is almost always 8-byte aligned in our case because atomic<> uses 64 bit native types for storage and dynamic memory allocations
  620.     // have at least 8 byte alignment. The only unfortunate case is when atomic is placed on the stack and it is not 8-byte aligned (like on 32 bit Windows).
  621.  
  622.     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  623.     {
  624.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  625.  
  626.         storage_type volatile* p = &storage;
  627.         if (((uint32_t)p & 0x00000007) == 0)
  628.         {
  629. #if defined(_M_IX86_FP) && _M_IX86_FP >= 2
  630. #if defined(__AVX__)
  631.             __asm
  632.             {
  633.                 mov edx, p
  634.                 vmovq xmm4, v
  635.                 vmovq qword ptr [edx], xmm4
  636.             };
  637. #else
  638.             __asm
  639.             {
  640.                 mov edx, p
  641.                 movq xmm4, v
  642.                 movq qword ptr [edx], xmm4
  643.             };
  644. #endif
  645. #else
  646.             __asm
  647.             {
  648.                 mov edx, p
  649.                 fild v
  650.                 fistp qword ptr [edx]
  651.             };
  652. #endif
  653.         }
  654.         else
  655.         {
  656.             int backup;
  657.             __asm
  658.             {
  659.                 mov backup, ebx
  660.                 mov edi, p
  661.                 mov ebx, dword ptr [v]
  662.                 mov ecx, dword ptr [v + 4]
  663.                 mov eax, dword ptr [edi]
  664.                 mov edx, dword ptr [edi + 4]
  665.                 align 16
  666.             again:
  667.                 lock cmpxchg8b qword ptr [edi]
  668.                 jne again
  669.                 mov ebx, backup
  670.             };
  671.         }
  672.  
  673.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  674.     }
  675.  
  676.     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT
  677.     {
  678.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  679.  
  680.         storage_type const volatile* p = &storage;
  681.         storage_type value;
  682.  
  683.         if (((uint32_t)p & 0x00000007) == 0)
  684.         {
  685. #if defined(_M_IX86_FP) && _M_IX86_FP >= 2
  686. #if defined(__AVX__)
  687.             __asm
  688.             {
  689.                 mov edx, p
  690.                 vmovq xmm4, qword ptr [edx]
  691.                 vmovq value, xmm4
  692.             };
  693. #else
  694.             __asm
  695.             {
  696.                 mov edx, p
  697.                 movq xmm4, qword ptr [edx]
  698.                 movq value, xmm4
  699.             };
  700. #endif
  701. #else
  702.             __asm
  703.             {
  704.                 mov edx, p
  705.                 fild qword ptr [edx]
  706.                 fistp value
  707.             };
  708. #endif
  709.         }
  710.         else
  711.         {
  712.             // We don't care for comparison result here; the previous value will be stored into value anyway.
  713.             // Also we don't care for ebx and ecx values, they just have to be equal to eax and edx before cmpxchg8b.
  714.             __asm
  715.             {
  716.                 mov edi, p
  717.                 mov eax, ebx
  718.                 mov edx, ecx
  719.                 lock cmpxchg8b qword ptr [edi]
  720.                 mov dword ptr [value], eax
  721.                 mov dword ptr [value + 4], edx
  722.             };
  723.         }
  724.  
  725.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  726.  
  727.         return value;
  728.     }
  729.  
  730.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  731.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
  732.     {
  733.         // MSVC-11 in 32-bit mode sometimes generates messed up code without compiler barriers,
  734.         // even though the _InterlockedCompareExchange64 intrinsic already provides one.
  735.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  736.  
  737.         storage_type volatile* p = &storage;
  738. #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
  739.         const storage_type old_val = (storage_type)BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(p, desired, expected);
  740.         const bool result = (old_val == expected);
  741.         expected = old_val;
  742. #else
  743.         bool result;
  744.         int backup;
  745.         __asm
  746.         {
  747.             mov backup, ebx
  748.             mov edi, p
  749.             mov esi, expected
  750.             mov ebx, dword ptr [desired]
  751.             mov ecx, dword ptr [desired + 4]
  752.             mov eax, dword ptr [esi]
  753.             mov edx, dword ptr [esi + 4]
  754.             lock cmpxchg8b qword ptr [edi]
  755.             mov dword ptr [esi], eax
  756.             mov dword ptr [esi + 4], edx
  757.             mov ebx, backup
  758.             sete result
  759.         };
  760. #endif
  761.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  762.  
  763.         return result;
  764.     }
  765.  
  766.     static BOOST_FORCEINLINE bool compare_exchange_weak(
  767.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
  768.     {
  769.         return compare_exchange_strong(storage, expected, desired, success_order, failure_order);
  770.     }
  771.  
  772.     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  773.     {
  774.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  775.  
  776.         storage_type volatile* p = &storage;
  777.         int backup;
  778.         __asm
  779.         {
  780.             mov backup, ebx
  781.             mov edi, p
  782.             mov ebx, dword ptr [v]
  783.             mov ecx, dword ptr [v + 4]
  784.             mov eax, dword ptr [edi]
  785.             mov edx, dword ptr [edi + 4]
  786.             align 16
  787.         again:
  788.             lock cmpxchg8b qword ptr [edi]
  789.             jne again
  790.             mov ebx, backup
  791.             mov dword ptr [v], eax
  792.             mov dword ptr [v + 4], edx
  793.         };
  794.  
  795.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  796.  
  797.         return v;
  798.     }
  799.  
  800.     static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
  801.     {
  802.         return true;
  803.     }
  804. };
  805.  
  806. template< bool Signed >
  807. struct operations< 8u, Signed > :
  808.     public cas_based_operations< msvc_dcas_x86< Signed > >
  809. {
  810. };
  811.  
  812. #elif defined(_M_AMD64)
  813.  
  814. template< bool Signed >
  815. struct operations< 8u, Signed > :
  816.     public msvc_x86_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > >
  817. {
  818.     typedef msvc_x86_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > > base_type;
  819.     typedef typename base_type::storage_type storage_type;
  820.     typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
  821.  
  822.     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  823.     {
  824.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&storage, v));
  825.     }
  826.  
  827.     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  828.     {
  829.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&storage, v));
  830.     }
  831.  
  832.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  833.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
  834.     {
  835.         storage_type previous = expected;
  836.         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&storage, desired, previous));
  837.         expected = old_val;
  838.         return (previous == old_val);
  839.     }
  840.  
  841.     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  842.     {
  843.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&storage, v));
  844.     }
  845.  
  846.     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  847.     {
  848.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&storage, v));
  849.     }
  850.  
  851.     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  852.     {
  853.         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&storage, v));
  854.     }
  855. };
  856.  
  857. #endif
  858.  
  859. #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
  860.  
  861. template< bool Signed >
  862. struct msvc_dcas_x86_64
  863. {
  864.     typedef typename make_storage_type< 16u, Signed >::type storage_type;
  865.     typedef typename make_storage_type< 16u, Signed >::aligned aligned_storage_type;
  866.  
  867.     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
  868.     {
  869.         storage_type value = const_cast< storage_type& >(storage);
  870.         while (!BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, v, &value)) {}
  871.     }
  872.  
  873.     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT
  874.     {
  875.         storage_type value = storage_type();
  876.         BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, value, &value);
  877.         return value;
  878.     }
  879.  
  880.     static BOOST_FORCEINLINE bool compare_exchange_strong(
  881.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
  882.     {
  883.         return !!BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, desired, &expected);
  884.     }
  885.  
  886.     static BOOST_FORCEINLINE bool compare_exchange_weak(
  887.         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
  888.     {
  889.         return compare_exchange_strong(storage, expected, desired, success_order, failure_order);
  890.     }
  891.  
  892.     static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT
  893.     {
  894.         return true;
  895.     }
  896. };
  897.  
  898. template< bool Signed >
  899. struct operations< 16u, Signed > :
  900.     public cas_based_operations< cas_based_exchange< msvc_dcas_x86_64< Signed > > >
  901. {
  902. };
  903.  
  904. #endif // defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
  905.  
  906. BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
  907. {
  908.     BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  909.     if (order == memory_order_seq_cst)
  910.         msvc_x86_operations_base::hardware_full_fence();
  911.     BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  912. }
  913.  
  914. BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
  915. {
  916.     if (order != memory_order_relaxed)
  917.         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  918. }
  919.  
  920. } // namespace detail
  921. } // namespace atomics
  922. } // namespace mars_boost
  923.  
  924. #if defined(BOOST_MSVC)
  925. #pragma warning(pop)
  926. #endif
  927.  
  928. #endif // BOOST_ATOMIC_DETAIL_OPS_MSVC_X86_HPP_INCLUDED_
  929.  
downloadops_msvc_x86.hpp Source code - Download mars Source code
Related Source Codes/Software:
Hero - Elegant transition library for iOS & tvOS 2017-06-09
deep-photo-styletransfer - Code and data for paper "Deep Photo Style Transfer... 2017-06-09
mastodon - A GNU Social-compatible microblogging server ... 2017-06-09
plyr - A simple HTML5, YouTube and Vimeo player ... 2017-06-08
prepack - Prepack is a partial evaluator for JavaScript. Pre... 2017-06-08
Public-APIs - 2017-06-09
lottie-ios - An iOS library to natively render After Effects ve... 2017-06-09
Awesome-Hacking - A collection of various awesome lists for hackers,... 2017-06-09
algorithms - Minimal examples of data structures and algorithms... 2017-06-10
lectures - Oxford Deep NLP 2017 course 2017-06-10
CRYENGINE - CRYENGINE is a powerful real-time game development... 2017-06-11
postal - 2017-06-11
reactide - Reactide is the first dedicated IDE for React web ... 2017-06-11
rkt - rkt is a pod-native container engine for Linux. It... 2017-06-11
uWebSockets - Tiny WebSockets https://for... 2017-06-11
realworld - TodoMVC for the RealWorld - Exemplary fullstack Me... 2017-06-11
goreplay - GoReplay is an open-source tool for capturing and ... 2017-06-10
pyenv - Simple Python version management 2017-06-10
redux-saga - An alternative side effect model for Redux apps ... 2017-06-10
angular-starter - 2017-06-10

 Back to top