lockrank_on.go

Documentation: runtime

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.staticlockranking
     6  
     7  package runtime
     8  
     9  import (
    10  	"internal/runtime/atomic"
    11  	"unsafe"
    12  )
    13  
    14  const staticLockRanking = true
    15  
    16  // worldIsStopped is accessed atomically to track world-stops. 1 == world
    17  // stopped.
    18  var worldIsStopped atomic.Uint32
    19  
    20  // lockRankStruct is embedded in mutex
    21  type lockRankStruct struct {
    22  	// static lock ranking of the lock
    23  	rank lockRank
    24  }
    25  
    26  // lockInit(l *mutex, rank int) sets the rank of lock before it is used.
    27  // If there is no clear place to initialize a lock, then the rank of a lock can be
    28  // specified during the lock call itself via lockWithRank(l *mutex, rank int).
    29  func lockInit(l *mutex, rank lockRank) {
    30  	l.rank = rank
    31  }
    32  
    33  func getLockRank(l *mutex) lockRank {
    34  	return l.rank
    35  }
    36  
    37  // lockWithRank is like lock(l), but allows the caller to specify a lock rank
    38  // when acquiring a non-static lock.
    39  //
    40  // Note that we need to be careful about stack splits:
    41  //
    42  // This function is not nosplit, thus it may split at function entry. This may
    43  // introduce a new edge in the lock order, but it is no different from any
    44  // other (nosplit) call before this call (including the call to lock() itself).
    45  //
    46  // However, we switch to the systemstack to record the lock held to ensure that
    47  // we record an accurate lock ordering. e.g., without systemstack, a stack
    48  // split on entry to lock2() would record stack split locks as taken after l,
    49  // even though l is not actually locked yet.
    50  func lockWithRank(l *mutex, rank lockRank) {
    51  	if l == &debuglock || l == &paniclk || l == &raceFiniLock {
    52  		// debuglock is only used for println/printlock(). Don't do lock
    53  		// rank recording for it, since print/println are used when
    54  		// printing out a lock ordering problem below.
    55  		//
    56  		// paniclk is only used for fatal throw/panic. Don't do lock
    57  		// ranking recording for it, since we throw after reporting a
    58  		// lock ordering problem. Additionally, paniclk may be taken
    59  		// after effectively any lock (anywhere we might panic), which
    60  		// the partial order doesn't cover.
    61  		//
    62  		// raceFiniLock is held while exiting when running
    63  		// the race detector. Don't do lock rank recording for it,
    64  		// since we are exiting.
    65  		lock2(l)
    66  		return
    67  	}
    68  	if rank == 0 {
    69  		rank = lockRankLeafRank
    70  	}
    71  	gp := getg()
    72  	// Log the new class.
    73  	systemstack(func() {
    74  		i := gp.m.locksHeldLen
    75  		if i >= len(gp.m.locksHeld) {
    76  			throw("too many locks held concurrently for rank checking")
    77  		}
    78  		gp.m.locksHeld[i].rank = rank
    79  		gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l))
    80  		gp.m.locksHeldLen++
    81  
    82  		// i is the index of the lock being acquired
    83  		if i > 0 {
    84  			checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
    85  		}
    86  		lock2(l)
    87  	})
    88  }
    89  
    90  // nosplit to ensure it can be called in as many contexts as possible.
    91  //
    92  //go:nosplit
    93  func printHeldLocks(gp *g) {
    94  	if gp.m.locksHeldLen == 0 {
    95  		println("<none>")
    96  		return
    97  	}
    98  
    99  	for j, held := range gp.m.locksHeld[:gp.m.locksHeldLen] {
   100  		println(j, ":", held.rank.String(), held.rank, unsafe.Pointer(gp.m.locksHeld[j].lockAddr))
   101  	}
   102  }
   103  
   104  // acquireLockRankAndM acquires a rank which is not associated with a mutex
   105  // lock. To maintain the invariant that an M which holds lock-like resources
   106  // will have m.locks/mutexMLocksDelta > 0, it also acquires the M and adjusts
   107  // m.locks by mutexMLocksDelta.
   108  //
   109  // This function may be called in nosplit context and thus must be nosplit.
   110  //
   111  //go:nosplit
   112  func acquireLockRankAndM(rank lockRank) {
   113  	mp := acquirem()
   114  	mp.locks += mutexMLocksDelta // not safe to profile right now
   115  
   116  	gp := getg()
   117  	// Log the new class. See comment on lockWithRank.
   118  	systemstack(func() {
   119  		i := gp.m.locksHeldLen
   120  		if i >= len(gp.m.locksHeld) {
   121  			throw("too many locks held concurrently for rank checking")
   122  		}
   123  		gp.m.locksHeld[i].rank = rank
   124  		gp.m.locksHeld[i].lockAddr = 0
   125  		gp.m.locksHeldLen++
   126  
   127  		// i is the index of the lock being acquired
   128  		if i > 0 {
   129  			checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
   130  		}
   131  	})
   132  }
   133  
   134  // checkRanks checks if goroutine g, which has mostly recently acquired a lock
   135  // with rank 'prevRank', can now acquire a lock with rank 'rank'.
   136  //
   137  //go:systemstack
   138  func checkRanks(gp *g, prevRank, rank lockRank) {
   139  	rankOK := false
   140  	if rank < prevRank {
   141  		// If rank < prevRank, then we definitely have a rank error
   142  		rankOK = false
   143  	} else if rank == lockRankLeafRank {
   144  		// If new lock is a leaf lock, then the preceding lock can
   145  		// be anything except another leaf lock.
   146  		rankOK = prevRank < lockRankLeafRank
   147  	} else {
   148  		// We've now verified the total lock ranking, but we
   149  		// also enforce the partial ordering specified by
   150  		// lockPartialOrder as well. Two locks with the same rank
   151  		// can only be acquired at the same time if explicitly
   152  		// listed in the lockPartialOrder table.
   153  		list := lockPartialOrder[rank]
   154  		for _, entry := range list {
   155  			if entry == prevRank {
   156  				rankOK = true
   157  				break
   158  			}
   159  		}
   160  	}
   161  	if !rankOK {
   162  		printlock()
   163  		println(gp.m.procid, " ======")
   164  		printHeldLocks(gp)
   165  		throw("lock ordering problem")
   166  	}
   167  }
   168  
   169  // See comment on lockWithRank regarding stack splitting.
   170  func unlockWithRank(l *mutex) {
   171  	if l == &debuglock || l == &paniclk || l == &raceFiniLock {
   172  		// See comment at beginning of lockWithRank.
   173  		unlock2(l)
   174  		return
   175  	}
   176  	gp := getg()
   177  	systemstack(func() {
   178  		found := false
   179  		for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
   180  			if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) {
   181  				found = true
   182  				copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen])
   183  				gp.m.locksHeldLen--
   184  				break
   185  			}
   186  		}
   187  		if !found {
   188  			println(gp.m.procid, ":", l.rank.String(), l.rank, l)
   189  			throw("unlock without matching lock acquire")
   190  		}
   191  		unlock2(l)
   192  	})
   193  }
   194  
   195  // releaseLockRankAndM releases a rank which is not associated with a mutex
   196  // lock. To maintain the invariant that an M with m.locks==0 does not hold any
   197  // lock-like resources, it also releases the M.
   198  //
   199  // This function may be called in nosplit context and thus must be nosplit.
   200  //
   201  //go:nosplit
   202  func releaseLockRankAndM(rank lockRank) {
   203  	gp := getg()
   204  	systemstack(func() {
   205  		found := false
   206  		for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
   207  			if gp.m.locksHeld[i].rank == rank && gp.m.locksHeld[i].lockAddr == 0 {
   208  				found = true
   209  				copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen])
   210  				gp.m.locksHeldLen--
   211  				break
   212  			}
   213  		}
   214  		if !found {
   215  			println(gp.m.procid, ":", rank.String(), rank)
   216  			throw("lockRank release without matching lockRank acquire")
   217  		}
   218  	})
   219  
   220  	mp := getg().m
   221  	mp.locks -= mutexMLocksDelta
   222  	releasem(mp)
   223  }
   224  
   225  // nosplit because it may be called from nosplit contexts.
   226  //
   227  //go:nosplit
   228  func lockWithRankMayAcquire(l *mutex, rank lockRank) {
   229  	gp := getg()
   230  	if gp.m.locksHeldLen == 0 {
   231  		// No possibility of lock ordering problem if no other locks held
   232  		return
   233  	}
   234  
   235  	systemstack(func() {
   236  		i := gp.m.locksHeldLen
   237  		if i >= len(gp.m.locksHeld) {
   238  			throw("too many locks held concurrently for rank checking")
   239  		}
   240  		// Temporarily add this lock to the locksHeld list, so
   241  		// checkRanks() will print out list, including this lock, if there
   242  		// is a lock ordering problem.
   243  		gp.m.locksHeld[i].rank = rank
   244  		gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l))
   245  		gp.m.locksHeldLen++
   246  		checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
   247  		gp.m.locksHeldLen--
   248  	})
   249  }
   250  
   251  // nosplit to ensure it can be called in as many contexts as possible.
   252  //
   253  //go:nosplit
   254  func checkLockHeld(gp *g, l *mutex) bool {
   255  	for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
   256  		if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) {
   257  			return true
   258  		}
   259  	}
   260  	return false
   261  }
   262  
   263  // assertLockHeld throws if l is not held by the caller.
   264  //
   265  // nosplit to ensure it can be called in as many contexts as possible.
   266  //
   267  //go:nosplit
   268  func assertLockHeld(l *mutex) {
   269  	gp := getg()
   270  
   271  	held := checkLockHeld(gp, l)
   272  	if held {
   273  		return
   274  	}
   275  
   276  	// Crash from system stack to avoid splits that may cause
   277  	// additional issues.
   278  	systemstack(func() {
   279  		printlock()
   280  		print("caller requires lock ", l, " (rank ", l.rank.String(), "), holding:\n")
   281  		printHeldLocks(gp)
   282  		throw("not holding required lock!")
   283  	})
   284  }
   285  
   286  // assertRankHeld throws if a mutex with rank r is not held by the caller.
   287  //
   288  // This is less precise than assertLockHeld, but can be used in places where a
   289  // pointer to the exact mutex is not available.
   290  //
   291  // nosplit to ensure it can be called in as many contexts as possible.
   292  //
   293  //go:nosplit
   294  func assertRankHeld(r lockRank) {
   295  	gp := getg()
   296  
   297  	for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
   298  		if gp.m.locksHeld[i].rank == r {
   299  			return
   300  		}
   301  	}
   302  
   303  	// Crash from system stack to avoid splits that may cause
   304  	// additional issues.
   305  	systemstack(func() {
   306  		printlock()
   307  		print("caller requires lock with rank ", r.String(), "), holding:\n")
   308  		printHeldLocks(gp)
   309  		throw("not holding required lock!")
   310  	})
   311  }
   312  
   313  // worldStopped notes that the world is stopped.
   314  //
   315  // Caller must hold worldsema.
   316  //
   317  // nosplit to ensure it can be called in as many contexts as possible.
   318  //
   319  //go:nosplit
   320  func worldStopped() {
   321  	if stopped := worldIsStopped.Add(1); stopped != 1 {
   322  		systemstack(func() {
   323  			print("world stop count=", stopped, "\n")
   324  			throw("recursive world stop")
   325  		})
   326  	}
   327  }
   328  
   329  // worldStarted that the world is starting.
   330  //
   331  // Caller must hold worldsema.
   332  //
   333  // nosplit to ensure it can be called in as many contexts as possible.
   334  //
   335  //go:nosplit
   336  func worldStarted() {
   337  	if stopped := worldIsStopped.Add(-1); stopped != 0 {
   338  		systemstack(func() {
   339  			print("world stop count=", stopped, "\n")
   340  			throw("released non-stopped world stop")
   341  		})
   342  	}
   343  }
   344  
   345  // nosplit to ensure it can be called in as many contexts as possible.
   346  //
   347  //go:nosplit
   348  func checkWorldStopped() bool {
   349  	stopped := worldIsStopped.Load()
   350  	if stopped > 1 {
   351  		systemstack(func() {
   352  			print("inconsistent world stop count=", stopped, "\n")
   353  			throw("inconsistent world stop count")
   354  		})
   355  	}
   356  
   357  	return stopped == 1
   358  }
   359  
   360  // assertWorldStopped throws if the world is not stopped. It does not check
   361  // which M stopped the world.
   362  //
   363  // nosplit to ensure it can be called in as many contexts as possible.
   364  //
   365  //go:nosplit
   366  func assertWorldStopped() {
   367  	if checkWorldStopped() {
   368  		return
   369  	}
   370  
   371  	throw("world not stopped")
   372  }
   373  
   374  // assertWorldStoppedOrLockHeld throws if the world is not stopped and the
   375  // passed lock is not held.
   376  //
   377  // nosplit to ensure it can be called in as many contexts as possible.
   378  //
   379  //go:nosplit
   380  func assertWorldStoppedOrLockHeld(l *mutex) {
   381  	if checkWorldStopped() {
   382  		return
   383  	}
   384  
   385  	gp := getg()
   386  	held := checkLockHeld(gp, l)
   387  	if held {
   388  		return
   389  	}
   390  
   391  	// Crash from system stack to avoid splits that may cause
   392  	// additional issues.
   393  	systemstack(func() {
   394  		printlock()
   395  		print("caller requires world stop or lock ", l, " (rank ", l.rank.String(), "), holding:\n")
   396  		println("<no world stop>")
   397  		printHeldLocks(gp)
   398  		throw("no world stop or required lock!")
   399  	})
   400  }
   401
View as plain text