mutex与spinlock

在写多线程的过程中，难免会遇到锁，一般使用的是mutex或者rwlock。

调试的过程中也发现，使用mutex的情况下，spinlock在perf会占有一定的比例。那么

1 mutex会在多大程度上影响程序的系能，什么情况下是值得使用的？

2 rwlock和mutex在实现上有什么不同？

3 spinlock和mutex有什么关系？

4 如何看待流行的非锁程序？性能会有提升吗？

从源码入手：mutex的接口使用，是pthread库中的pthread_mutex_lock，使用glibc-2.12.1，如下

/* Data structures for mutex handling.  The structure of the attribute   type is not exposed on purpose.  */typedef union{  struct <span style="background-color: rgb(255, 102, 102);">__pthread_mutex_s</span>  {    int <span style="color:#ff6666;">__lock</span>;    unsigned int __count;    int __owner;#if __WORDSIZE == 64    unsigned int __nusers;#endif    /* KIND must stay at this position in the structure to maintain       binary compatibility.  */    int <span style="color:#ff6666;">__kind</span>;#if __WORDSIZE == 64    int __spins;    __pthread_list_t __list;# define __PTHREAD_MUTEX_HAVE_PREV1#else    unsigned int __nusers;    __extension__ union    {      int __spins;      __pthread_slist_t __list;    };#endif  } __data;  char __size[__SIZEOF_PTHREAD_MUTEX_T];  long int __align;} pthread_mutex_t;int<span style="background-color: rgb(255, 102, 102);">__pthread_mutex_lock </span>(mutex)     pthread_mutex_t *mutex;{  assert (sizeof (mutex->__size) >= sizeof (mutex->__data));  unsigned int type = PTHREAD_MUTEX_TYPE (mutex);  if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))    return __pthread_mutex_lock_full (mutex);  pid_t id = THREAD_GETMEM (THREAD_SELF, tid);  if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)      == PTHREAD_MUTEX_TIMED_NP)    {    simple:      /* Normal mutex.  */      <span style="background-color: rgb(255, 102, 102);">LLL_MUTEX_LOCK </span>(mutex);      assert (mutex->__data.__owner == 0);    }  else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))    {      /* Recursive mutex.  */      /* Check whether we already hold the mutex.  */      if (mutex->__data.__owner == id){  /* Just bump the counter.  */  if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))    /* Overflow of the counter.  */    return EAGAIN;  ++mutex->__data.__count;  return 0;}      /* We have to get the mutex.  */      LLL_MUTEX_LOCK (mutex);      assert (mutex->__data.__owner == 0);      mutex->__data.__count = 1;    }  else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))    {      if (! __is_smp)goto simple;      if (LLL_MUTEX_TRYLOCK (mutex) != 0){  int cnt = 0;  int max_cnt = MIN (MAX_ADAPTIVE_COUNT,     mutex->__data.__spins * 2 + 10);  do    {      if (cnt++ >= max_cnt){  LLL_MUTEX_LOCK (mutex);  break;}#ifdef BUSY_WAIT_NOP      BUSY_WAIT_NOP;#endif    }  while (LLL_MUTEX_TRYLOCK (mutex) != 0);  mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;}      assert (mutex->__data.__owner == 0);    }  else    {      assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);      /* Check whether we already hold the mutex.  */      if (__builtin_expect (mutex->__data.__owner == id, 0))return EDEADLK;      goto simple;    }  /* Record the ownership.  */  mutex->__data.__owner = id;#ifndef NO_INCR  ++mutex->__data.__nusers;#endif  return 0;}#define LLL_MUTEX_LOCK(mutex) \  lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))#define lll_cond_lock(futex, private) \  (void)      \    ({ int ignore1, ignore2, ignore3;      \       __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t"      \ "jnz 1f\n\t"      \ ".subsection 1\n\t"      \ ".type _L_cond_lock_%=, @function\n"      \ "_L_cond_lock_%=:\n"      \ "1:\tleaq %2, %%rdi\n"      \ "2:\tsubq $128, %%rsp\n"      \ "3:\tcallq <span style="background-color: rgb(255, 102, 102);">__lll_lock_wait</span>\n"      \ "4:\taddq $128, %%rsp\n"      \ "5:\tjmp 24f\n"      \ "6:\t.size _L_cond_lock_%=, 6b-1b\n\t"      \ ".previous\n"      \ LLL_STUB_UNWIND_INFO_5      \ "24:"      \ : "=S" (ignore1), "=D" (ignore2), "=m" (futex),      \   "=a" (ignore3)      \ : "1" (2), "m" (futex), "3" (0), "0" (private)      \ : "cx", "r11", "cc", "memory");      \    })

void__lll_lock_wait (int *futex, int private){  if (*futex == 2)    lll_futex_wait (futex, 2, private);  while (atomic_exchange_acq (futex, 2) != 0)    lll_futex_wait (futex, 2, private);}#define lll_futex_wait(futex, val, private) \  lll_futex_timed_wait(futex, val, NULL, private)#define lll_futex_timed_wait(futex, val, timeout, private) \  ({      \    register const struct timespec *__to __asm ("r10") = timeout;      \    int __status;      \    register __typeof (val) _val __asm ("edx") = (val);      \    __asm __volatile ("syscall"      \      : "=a" (__status)      \      : "0" (SYS_futex), "D" (futex),      \"S" (__lll_private_flag (FUTEX_WAIT, private)),      \"d" (_val), "r" (__to)      \      : "memory", "cc", "r11", "cx");      \    __status;      \  })

glibc最终会调用系统调用futex，系统调用futex的源码在linux-2.6.38中如下：

SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,struct timespec __user *, utime, u32 __user *, uaddr2,u32, val3){struct timespec ts;ktime_t t, *tp = NULL;u32 val2 = 0;int cmd = op & FUTEX_CMD_MASK;if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||      cmd == FUTEX_WAIT_BITSET ||      cmd == FUTEX_WAIT_REQUEUE_PI)) {if (copy_from_user(&ts, utime, sizeof(ts)) != 0)return -EFAULT;if (!timespec_valid(&ts))return -EINVAL;t = timespec_to_ktime(ts);if (cmd == FUTEX_WAIT)t = ktime_add_safe(ktime_get(), t);tp = &t;}/* * requeue parameter in 'utime' if cmd == FUTEX_*_REQUEUE_*. * number of waiters to wake in 'utime' if cmd == FUTEX_WAKE_OP. */if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||    cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)val2 = (u32) (unsigned long) utime;return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);}

其中调用了do_futex

long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,u32 __user *uaddr2, u32 val2, u32 val3){int ret = -ENOSYS, cmd = op & FUTEX_CMD_MASK;unsigned int flags = 0;if (!(op & FUTEX_PRIVATE_FLAG))flags |= FLAGS_SHARED;if (op & FUTEX_CLOCK_REALTIME) {flags |= FLAGS_CLOCKRT;if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)return -ENOSYS;}switch (cmd) {case FUTEX_WAIT:val3 = FUTEX_BITSET_MATCH_ANY;case FUTEX_WAIT_BITSET:ret = futex_wait(uaddr, flags, val, timeout, val3);break;case FUTEX_WAKE:val3 = FUTEX_BITSET_MATCH_ANY;case FUTEX_WAKE_BITSET:ret = futex_wake(uaddr, flags, val, val3);break;case FUTEX_REQUEUE:ret = futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);break;case FUTEX_CMP_REQUEUE:ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);break;case FUTEX_WAKE_OP:ret = futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);break;case FUTEX_LOCK_PI:if (futex_cmpxchg_enabled)ret = futex_lock_pi(uaddr, flags, val, timeout, 0);break;case FUTEX_UNLOCK_PI:if (futex_cmpxchg_enabled)ret = futex_unlock_pi(uaddr, flags);break;case FUTEX_TRYLOCK_PI:if (futex_cmpxchg_enabled)ret = futex_lock_pi(uaddr, flags, 0, timeout, 1);break;case FUTEX_WAIT_REQUEUE_PI:val3 = FUTEX_BITSET_MATCH_ANY;ret = futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,    uaddr2);break;case FUTEX_CMP_REQUEUE_PI:ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);break;default:ret = -ENOSYS;}return ret;}

使用strace捕获系统调用时的一些参数如下：

 futex(0x7fbf0e41e0ac, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fbf0e41e0a8, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1} <unfinished ...>
12312 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12324 <... futex resumed> )             = 0
12307 <... futex resumed> )             = 1
12312 <... write resumed> )             = 8
12324 futex(0x7fbf0e41e080, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12312 futex(0x7fbf1a41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12307 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> )             = -1 EAGAIN (Resource temporarily unavailable)
12307 <... futex resumed> )             = 0
12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12307 futex(0x7fbf1341e16c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fbf1341e168, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1} <unfinished ...>
12324 <... futex resumed> )             = 0
12319 <... futex resumed> )             = 0
12307 <... futex resumed> )             = 1
12324 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12319 futex(0x7fbf1341e140, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12307 futex(0x7fbf1341e140, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... write resumed> )             = 8
12319 <... futex resumed> )             = -1 EAGAIN (Resource temporarily unavailable)
12307 <... futex resumed> )             = 0
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12319 futex(0x7fbf1341e140, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12307 futex(0x7fbf1e42128c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7fbf1e421288, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1} <unfinished ...>
12319 <... futex resumed> )             = 0
12308 <... futex resumed> )             = 0
12307 <... futex resumed> )             = 1
12319 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12308 futex(0x7fbf1e421260, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12307 futex(0x7fbf1e421260, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>

grep其中一个线程的

12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> )             = 0
12324 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12324 <... write resumed> )             = 8
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12324 <... futex resumed> )             = 0
12324 futex(0x7fbf0e41e080, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12324 <... futex resumed> )             = -1 EAGAIN (Resource temporarily unavailable)
12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> )             = 0
12324 write(8, "\1\0\0\0\0\0\0\0", 8)   = 8
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12324 <... futex resumed> )             = 0
12324 futex(0x7fbf0e41e080, FUTEX_WAIT_PRIVATE, 2, NULL <unfinished ...>
12324 <... futex resumed> )             = -1 EAGAIN (Resource temporarily unavailable)
12324 futex(0x7fbf0e41e080, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
12324 <... futex resumed> )             = 0
12324 write(8, "\1\0\0\0\0\0\0\0", 8 <unfinished ...>
12324 <... write resumed> )             = 8
12324 futex(0x7fbf0e41e0ac, FUTEX_WAIT_PRIVATE, 1, NULL <unfinished ...>
12324 <... futex resumed> )             = 0

其中futex的第三个参数是futex的具体cmd，定义如下：

#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_FD 2