github.com/cockroachdb/cockroach/pkg/storage: TestRaftRemoveRace failed under stress #11614

cockroach-teamcity · 2016-11-25T10:17:01Z

SHA: /~https://github.com/cockroachdb/cockroach/commits/0db6aea9cec5d83dd5c05d870b6edf4774d023ea

Parameters:

COCKROACH_PROPOSER_EVALUATED_KV=true
TAGS=deadlock stress
GOFLAGS=

Stress build found a failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=67518&tab=buildLog

	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3818891 [select, 21 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc430132540, 0x25851c0, 0xc428ec2160, 0xc432d04ca8, 0x20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x1992f40, 0xc430132540, 0x25810e0, 0xc4239bd7a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc423688320, 0x2585580, 0xc42e852870, 0xc428aeb3b0, 0xc42ff6bad0, 0x2555580, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:689 +0x702
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc423688320, 0x2585580, 0xc42e852870, 0xc428aeb3b0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:773 +0xc36
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42fe0f190, 0xc423688320, 0x2585580, 0xc42e852870, 0xc428aeb3b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:422 +0xab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:423 +0xa3

goroutine 3819577 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc430132540, 0xc400000006, 0xc42eba4d20, 0xc420530bc0, 0x2586600, 0xc428cb74e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc430132540, 0x6, 0x257f540, 0xc420014578, 0x0, 0x0, 0xc4239bdc20, 0xc42eba4d20, 0xc420530bc0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc430132540, 0x257f540, 0xc420014578, 0x6, 0xc42eba4d20, 0xc420530bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc4291a8630)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14308212 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4282e9e00, 0xc427172b10, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc429cee3c0, 0xc427172b10, 0x5, 0x5, 0x10, 0xc42fe54ca0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc429cee3c0, 0xc427172b10, 0x5, 0x5, 0x5, 0x4, 0x1841200, 0xc42d7215b8)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc429cee3c0, 0xc427172b10, 0x5, 0x5, 0x89389b, 0xc427813c40, 0x5d9947)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc427172b00, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc427172b00, 0x257d700, 0x2a78e08, 0xc429cee3c0, 0x0, 0x0, 0x190e9e0, 0xc43102dd10, 0x7fffffff, 0xc42776d000, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc422a2aa20, 0x190e9e0, 0xc43102dd10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42878bdb0, 0x3, 0x0, 0xc420026de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2586600, 0xc42878bdb0, 0xc42f962e40, 0xc42d808000, 0x257f540, 0xc420014578, 0xaf78cf, 0xc42bcbe788)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc43102dce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3827234 [select, 21 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42cb8cb40, 0xc42cf73170, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc422b47770, 0xc42cf73170, 0x5, 0x5, 0x10, 0xc430200b80, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc422b47770, 0xc42cf73170, 0x5, 0x5, 0x5, 0x1878460, 0x1841201, 0xc430200b80)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc422b47770, 0xc42cf73170, 0x5, 0x5, 0x89389b, 0xc427813c40, 0xc400000001)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42cf73160, 0x7fffffff, 0x2, 0x2, 0x2, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42cf73160, 0x257d700, 0x2a78e08, 0xc422b47770, 0x0, 0x0, 0x190e9e0, 0xc4280202d0, 0x7fffffff, 0xc420118000, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4298b8120, 0x190e9e0, 0xc4280202d0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc428662280, 0x6094bb, 0x2a3d750, 0xc420022de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2586600, 0xc428662280, 0xc425834f40, 0xc430132540, 0x257f540, 0xc420014578, 0xaf78cf, 0xc42ad94f88)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc428020210)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3814567 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42d9898c0, 0xc40000000a, 0xc430d2ba40, 0xc425021dc0, 0x2586600, 0xc4284a9b50, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42d9898c0, 0xa, 0x257f540, 0xc420014578, 0x0, 0x0, 0xc428c31290, 0xc430d2ba40, 0xc425021dc0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42d9898c0, 0x257f540, 0xc420014578, 0xa, 0xc430d2ba40, 0xc425021dc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42f76a090)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14310501 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc42f504400, 0xc426f491d0, 0xc42ce377a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83

goroutine 3819059 [select, 21 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc4299eed00, 0xc42a6b4f00, 0xc42d83b8c0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83

goroutine 3801143 [select, 21 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc4299eed00, 0xc42a6b40f0, 0xc42ac65680)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83

goroutine 3808445 [select, 21 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc430132540, 0x25851c0, 0xc431583090, 0xc4328444a8, 0x20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x1992f40, 0xc430132540, 0x25810e0, 0xc429a12fc0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc423688320, 0x2585580, 0xc42e852870, 0xc427450000, 0xc42ff6bad0, 0x2555580, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:689 +0x702
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc423688320, 0x2585580, 0xc42e852870, 0xc427450000, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:773 +0xc36
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc42fe0f190, 0xc423688320, 0x2585580, 0xc42e852870, 0xc427450000)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:422 +0xab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:423 +0xa3

goroutine 3818775 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4300d1ec0, 0xc40000000a, 0xc4252a1ce0, 0xc420590ac0, 0x2586600, 0xc4290bbb10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc4300d1ec0, 0xa, 0x257f540, 0xc420014578, 0x0, 0x0, 0xc429a134d0, 0xc4252a1ce0, 0xc420590ac0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4300d1ec0, 0x257f540, 0xc420014578, 0xa, 0xc4252a1ce0, 0xc420590ac0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc4330137d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 3823550 [select, 21 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc430421e60, 0x25851c0, 0xc4287f1710, 0xc42063a4a8, 0x20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x1992f40, 0xc430421e60, 0x25810e0, 0xc424d34d80, 0x5, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc42f990d20, 0x2585580, 0xc42e852630, 0xc428aeb950, 0xc4300cef90, 0x2555580, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:689 +0x702
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42f990d20, 0x2585580, 0xc42e852630, 0xc428aeb950, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:773 +0xc36
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4302010c0, 0xc42f990d20, 0x2585580, 0xc42e852630, 0xc428aeb950)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:422 +0xab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:423 +0xa3

goroutine 14326321 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42e4d97c0, 0xc4293ba570, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc429cef3b0, 0xc4293ba570, 0x5, 0x5, 0x10, 0xc425f6c460, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc429cef3b0, 0xc4293ba570, 0x5, 0x5, 0x5, 0x4, 0x1841200, 0xc42d7206b8)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc429cef3b0, 0xc4293ba570, 0x5, 0x5, 0x89389b, 0xc427813c40, 0x5d9947)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4293ba560, 0x7fffffff, 0x2571600, 0xc42e4b2780, 0xc42bca5590, 0x0, 0x2, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4293ba560, 0x257d700, 0x2a78e08, 0xc429cef3b0, 0x0, 0x0, 0x190e9e0, 0xc428918300, 0x7fffffff, 0xc426f5f000, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc42cbbcfc0, 0x190e9e0, 0xc428918300, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42ad329e0, 0x6094bb, 0xc42fc15974, 0xc4200257e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2586600, 0xc42ad329e0, 0xc426ccad40, 0xc4300d1ec0, 0x257f540, 0xc420014578, 0xaf78cf, 0xc42bca5788)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc422d5d980)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14326318 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc42d886a00, 0xc429cef3b0, 0xc42cbbcfc0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83

goroutine 14309201 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc42d886a00, 0xc42a12ef00, 0xc42c095200)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83

goroutine 14310196 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc427159580, 0xc426928190, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc429cee690, 0xc426928190, 0x5, 0x5, 0xc42e399d14, 0xc42eeb7c00, 0x8d19d3)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc429cee690, 0xc426928190, 0x5, 0x5, 0x5, 0xc42eeb7d78, 0x6e7c56, 0xc42d79a420)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc429cee690, 0xc426928190, 0x5, 0x5, 0xc42eeb7d08, 0x1, 0x0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc426928180, 0x7fffffff, 0x1, 0x18, 0x1bb7d01, 0x1bb7d00, 0xc41fbba864, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc426928180, 0x257d700, 0x2a78e08, 0xc429cee690, 0x0, 0x0, 0x190e900, 0xc4288af340, 0x7fffffff, 0xc4291d0001, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42ebefdd0, 0x190e900, 0xc4288af340, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4291d0000, 0x257f5c0, 0xc431a78990, 0xc42db2d800)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x25851c0, 0xc4291d0000, 0xc42e4b2780, 0xc432848788, 0xc432848790)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc431a79a40)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14310209 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42e4b2780, 0x4, 0xc42b74cf60, 0xc42de9c640, 0x2586600, 0xc429170aa0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42e4b2780, 0x4, 0x257f540, 0xc420014578, 0x1, 0x0, 0xc42c61a090, 0xc42b74cf60, 0xc42de9c640, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42e4b2780, 0x257f540, 0xc420014578, 0x4, 0xc42b74cf60, 0xc42de9c640)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc431a79ce0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14308193 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42d808000, 0x3, 0xc42e6426c0, 0xc42f962e40, 0x2586600, 0xc42878bdb0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42d808000, 0x3, 0x257f540, 0xc420014578, 0x1, 0x0, 0xc429a7d200, 0xc42e6426c0, 0xc42f962e40, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42d808000, 0x257f540, 0xc420014578, 0x3, 0xc42e6426c0, 0xc42f962e40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42a770ab0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14311100 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4267adf00, 0xc42b66cb10, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42c5ad0e0, 0xc42b66cb10, 0x5, 0x5, 0x10, 0x2, 0x7f0b6b4e0b38)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42c5ad0e0, 0xc42b66cb10, 0x5, 0x5, 0x5, 0x0, 0x0, 0x10)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42c5ad0e0, 0xc42b66cb10, 0x5, 0x5, 0x893800, 0xc43085fc68, 0x5d9947)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42b66cb00, 0x7fffffff, 0x2571600, 0xc42c0f5ec0, 0xc43085fd90, 0xaaa7b2, 0x1992f40, 0xc42c0f5ec0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42b66cb00, 0x257d700, 0x2a78e08, 0xc42c5ad0e0, 0x0, 0x0, 0x190e9e0, 0xc431119d10, 0x7fffffff, 0xc42776d000, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc429e43200, 0x190e9e0, 0xc431119d10, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc428ec2fd0, 0x6094bb, 0xc43020e784, 0xc420022de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2586600, 0xc428ec2fd0, 0xc42ff24fc0, 0xc42e4b2780, 0x257f540, 0xc420014578, 0xaf78cf, 0xc43085ff88)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc429f2cd20)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14373197 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42fe3e880, 0xc430b40e30, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42f507c20, 0xc430b40e30, 0x5, 0x5, 0x10, 0xc42a2cc260, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42f507c20, 0xc430b40e30, 0x5, 0x5, 0x5, 0x1878460, 0x1841201, 0xc42a2cc260)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42f507c20, 0xc430b40e30, 0x5, 0x5, 0x89389b, 0xc427813c40, 0x5d9947)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc430b40e20, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc430b40e20, 0x257d700, 0x2a78e08, 0xc42f507c20, 0x0, 0x0, 0x190e9e0, 0xc4301d98c0, 0x7fffffff, 0xc42050a000, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc42e02d560, 0x190e9e0, 0xc4301d98c0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42e6071e0, 0x6094bb, 0xc42b566b24, 0xc4200242e0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2586600, 0xc42e6071e0, 0xc42b60aa40, 0xc42f8cde60, 0x257f540, 0xc420014578, 0xaf78cf, 0xc42af77f88)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc4335c98c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14320756 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4304b6fc0, 0xc42633b790, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42c5ad1d0, 0xc42633b790, 0x5, 0x5, 0x10, 0xc431a4d4c0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42c5ad1d0, 0xc42633b790, 0x5, 0x5, 0x5, 0x1878460, 0x1841201, 0xc431a4d4c0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42c5ad1d0, 0xc42633b790, 0x5, 0x5, 0x89389b, 0xc427813c40, 0xc428673400)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42633b780, 0x7fffffff, 0xd94f67, 0xd93e28, 0xc428673400, 0x0, 0x4, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42633b780, 0x257d700, 0x2a78e08, 0xc42c5ad1d0, 0x0, 0x0, 0x190e9e0, 0xc431909b60, 0x7fffffff, 0xc420034800, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc43209e900, 0x190e9e0, 0xc431909b60, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4330bea10, 0x6094bb, 0xc4244e5e1c, 0xc420022de0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2586600, 0xc4330bea10, 0xc42f477c00, 0xc42e4b2780, 0x257f540, 0xc420014578, 0xaf78cf, 0xc421412788)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc428e22450)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14299269 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42e4b2780, 0x25851c0, 0xc431af5d00, 0xc42039b4a8, 0x20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x1992f40, 0xc42e4b2780, 0x25810e0, 0xc42c013cb0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc423689180, 0x2585580, 0xc42d676e10, 0xc42ead04b0, 0xc42d23b620, 0x2555580, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:689 +0x702
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc423689180, 0x2585580, 0xc42d676e10, 0xc42ead04b0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:773 +0xc36
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc425681610, 0xc423689180, 0x2585580, 0xc42d676e10, 0xc42ead04b0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:422 +0xab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:423 +0xa3

goroutine 14329894 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42f8cde60, 0x25851c0, 0xc424d11fa0, 0xc4308584a8, 0x20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x1992f40, 0xc42f8cde60, 0x25810e0, 0xc42b0b5170, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc42f7a20a0, 0x2585580, 0xc42f6de240, 0xc424ce4c30, 0xc42f9de960, 0x2555580, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:689 +0x702
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42f7a20a0, 0x2585580, 0xc42f6de240, 0xc424ce4c30, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:773 +0xc36
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4218af270, 0xc42f7a20a0, 0x2585580, 0xc42f6de240, 0xc424ce4c30)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:422 +0xab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:423 +0xa3

goroutine 14308452 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42fb5cb40, 0xc4299bd4d0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc426f490e0, 0xc4299bd4d0, 0x5, 0x5, 0xc42e399d14, 0xc4330e1c00, 0x8d19d3)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc426f490e0, 0xc4299bd4d0, 0x5, 0x5, 0x5, 0xc4330e1d78, 0x6e7c56, 0xc42d79a420)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc426f490e0, 0xc4299bd4d0, 0x5, 0x5, 0xc4330e1d08, 0x1, 0x0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4299bd4c0, 0x7fffffff, 0x1, 0x18, 0x1bb7d01, 0x1bb7d00, 0xc41f81fa74, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4299bd4c0, 0x257d700, 0x2a78e08, 0xc426f490e0, 0x0, 0x0, 0x190e900, 0xc42fc0b140, 0x7fffffff, 0xc4239af901, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42b56b7a0, 0x190e900, 0xc42fc0b140, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4239af9e0, 0x257f5c0, 0xc42a770cf0, 0xc4322e2000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x25851c0, 0xc4239af9e0, 0xc42e4b2780, 0xc420e39788, 0xc420e39790)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42eada450)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14311474 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc427aa1b40, 0xc4263c49d0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42d28fa40, 0xc4263c49d0, 0x5, 0x5, 0xc42fc88a14, 0xc432e4dc00, 0x8d19d3)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42d28fa40, 0xc4263c49d0, 0x5, 0x5, 0x5, 0xc432e4dd78, 0x6e7c56, 0xc42f636dc0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42d28fa40, 0xc4263c49d0, 0x5, 0x5, 0xc432e4dd08, 0x1, 0x0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4263c49c0, 0x7fffffff, 0x1, 0x18, 0x1bb7d01, 0x1bb7d00, 0xc41f9740e9, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4263c49c0, 0x257d700, 0x2a78e08, 0xc42d28fa40, 0x0, 0x0, 0x190e900, 0xc42d17e2a0, 0x7fffffff, 0xc428f24f01, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc429a7c1b0, 0x190e900, 0xc42d17e2a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc428f24f20, 0x257f5c0, 0xc42c85a210, 0xc42f5bb380)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x25851c0, 0xc428f24f20, 0xc42fc15920, 0xc427ac2f88, 0xc427ac2f90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc4315291d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14310706 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42873d940, 0xc42898bb90, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42498e4b0, 0xc42898bb90, 0x5, 0x5, 0xc42f58fc20, 0x6d6a40, 0xc42b30bb98)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42498e4b0, 0xc42898bb90, 0x5, 0x5, 0x5, 0xc42f58fd78, 0x6e7c56, 0xc42b30bb80)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42498e4b0, 0xc42898bb90, 0x5, 0x5, 0xc42f58fd08, 0x1, 0x0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42898bb80, 0x7fffffff, 0x4, 0x18, 0x1bb7d01, 0x1bb7d00, 0xc41fe53c4f, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42898bb80, 0x257d700, 0x2a78e08, 0xc42498e4b0, 0x0, 0x0, 0x190e900, 0xc4235875e0, 0x7fffffff, 0xc432a72201, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4299e7290, 0x190e900, 0xc4235875e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc432a72230, 0x257f5c0, 0xc42f642480, 0xc4322e4c00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x25851c0, 0xc432a72230, 0xc42d9898c0, 0xc432844f88, 0xc432844f90)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc429f9a810)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14373498 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42e4b2780, 0x25851c0, 0xc42e5ac260, 0xc429c79ca8, 0x20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x1992f40, 0xc42e4b2780, 0x25810e0, 0xc42a5dc3f0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc423689180, 0x2585580, 0xc42d676e10, 0xc42dd92a50, 0xc42d23b620, 0x2555580, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:689 +0x702
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc423689180, 0x2585580, 0xc42d676e10, 0xc42dd92a50, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:773 +0xc36
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc425681610, 0xc423689180, 0x2585580, 0xc42d676e10, 0xc42dd92a50)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:422 +0xab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:423 +0xa3

goroutine 14335248 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc424061f00, 0xc429dd94b0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42a12f770, 0xc429dd94b0, 0x5, 0x5, 0x10, 0xc43333aaa0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42a12f770, 0xc429dd94b0, 0x5, 0x5, 0x5, 0x1878460, 0x1841201, 0xc43333aaa0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42a12f770, 0xc429dd94b0, 0x5, 0x5, 0x89389b, 0xc427813c40, 0x38)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc429dd94a0, 0x7fffffff, 0xc429f741c0, 0xc42bc9f538, 0xd0a852, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc429dd94a0, 0x257d700, 0x2a78e08, 0xc42a12f770, 0x0, 0x0, 0x190e9e0, 0xc42d829aa0, 0x7fffffff, 0xc420230000, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc431c450e0, 0x190e9e0, 0xc42d829aa0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:305 +0xc4
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42865bc20, 0x6094bb, 0xc429c28404, 0xc42001ede0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:278 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x2586600, 0xc42865bc20, 0xc430904bc0, 0xc42e4b2780, 0x257f540, 0xc420014578, 0xaf78cf, 0xc42bc9f788)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:468 +0x56
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:485 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42d829a70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14311421 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42f761080, 0x3, 0xc4324d26c0, 0xc430910bc0, 0x2586600, 0xc428fed6a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42f761080, 0x3, 0x257f540, 0xc420014578, 0x1, 0x0, 0xc429046f30, 0xc4324d26c0, 0xc430910bc0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42f761080, 0x257f540, 0xc420014578, 0x3, 0xc4324d26c0, 0xc430910bc0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc431529170)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14328724 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4300d1ec0, 0x3, 0xc431b609c0, 0xc426ccad40, 0x2586600, 0xc42ad329e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc4300d1ec0, 0x3, 0x257f540, 0xc420014578, 0x1, 0x0, 0xc42b35bdd0, 0xc431b609c0, 0xc426ccad40, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4300d1ec0, 0x257f540, 0xc420014578, 0x3, 0xc431b609c0, 0xc426ccad40)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42d6a4870)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14319945 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42c189740, 0xc4252964f0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42d28fb30, 0xc4252964f0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42d28fb30, 0xc4252964f0, 0x5, 0x5, 0x5, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42d28fb30, 0xc4252964f0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4252964e0, 0x7fffffff, 0x0, 0x18, 0x1bb7d01, 0x1bb7d00, 0xc41fea898d, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4252964e0, 0x257d700, 0x2a78e08, 0xc42d28fb30, 0x0, 0x0, 0x190e900, 0xc422aece20, 0x7fffffff, 0xc427f77601, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc429a7c630, 0x190e900, 0xc422aece20, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc427f77630, 0x257f5c0, 0xc42c85bd40, 0xc43167c000)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x25851c0, 0xc427f77630, 0xc42e4b2780, 0xc420507788, 0xc420507790)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42c85bd70)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14309366 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42fc15920, 0x3, 0xc428c28ea0, 0xc430aab580, 0x2586600, 0xc42950d720, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42fc15920, 0x3, 0x257f540, 0xc420014578, 0x1, 0x0, 0xc42bdb41b0, 0xc428c28ea0, 0xc430aab580, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42fc15920, 0x257f540, 0xc420014578, 0x3, 0xc428c28ea0, 0xc430aab580)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42da1ef30)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14304569 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc422d53280, 0xc4294efed0, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc429cee2d0, 0xc4294efed0, 0x5, 0x5, 0xc42e399d14, 0xc432abfc00, 0x8d19d3)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc429cee2d0, 0xc4294efed0, 0x5, 0x5, 0x5, 0xc432abfd78, 0x6e7c56, 0xc42d79a420)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc429cee2d0, 0xc4294efed0, 0x5, 0x5, 0xc432abfd08, 0x1, 0x0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4294efec0, 0x7fffffff, 0x1, 0x18, 0x1bb7d01, 0x1bb7d00, 0xc41fa43ef9, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4294efec0, 0x257d700, 0x2a78e08, 0xc429cee2d0, 0x0, 0x0, 0x190e900, 0xc42b7820a0, 0x7fffffff, 0xc425a32b01, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42ebef560, 0x190e900, 0xc42b7820a0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc425a32b30, 0x257f5c0, 0xc431ea1560, 0xc433c08a80)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x25851c0, 0xc425a32b30, 0xc42e4b2780, 0xc4251d2788, 0xc4251d2790)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc431ea15f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14592479 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc429927800, 0x257f5c0, 0xc42c7bdad0, 0xc42b09f2d0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1397 +0x10da
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc429927800, 0x257f5c0, 0xc42c7bdad0, 0x7b, 0x9da1, 0x100000001, 0x1, 0x1, 0x0, 0xc42ae746c0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1802 +0xbfd
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc429927800, 0x257f5c0, 0xc42c7bdad0, 0x7b, 0x9da1, 0x100000001, 0x1, 0x1, 0x0, 0xc42ae746c0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1715 +0x90
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc429927800, 0x257f5c0, 0xc42c7bdad0, 0x7b, 0x9da1, 0x100000001, 0x1, 0x1, 0x0, 0xc42ae746c0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1183 +0x222
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc422733880, 0x257f5c0, 0xc42c7bda70, 0x7b, 0x9da1, 0x100000001, 0x1, 0x1, 0x0, 0xc42ae746c0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2386 +0x75e
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42befd3e0, 0x257f500, 0xc42f3cbf00, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc42ae74600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202
github.com/cockroachdb/cockroach/pkg/storage_test.(*multiTestContextKVTransport).SendNext.func1(0x257f500, 0xc42f3cbf00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/client_test.go:459 +0x136
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc42b02e120, 0x226c6f0, 0x16, 0x1e9, 0x0, 0x0, 0xc42f159020, 0x257f500, 0xc42f3cbf00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234

goroutine 14303989 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc42d9898c0, 0x3, 0xc42c1c4960, 0xc42f962940, 0x2586600, 0xc42971cf50, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:496 +0x65d
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc42d9898c0, 0x3, 0x257f540, 0xc420014578, 0x2, 0x0, 0xc429f86750, 0xc42c1c4960, 0xc42f962940, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:440 +0x388
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc42d9898c0, 0x257f540, 0xc420014578, 0x3, 0xc42c1c4960, 0xc42f962940)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:441 +0x105
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func2.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:576 +0xad
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc43171a090)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14310489 [select, 1 minutes]:
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc42d808000, 0x25851c0, 0xc432818b80, 0xc428f66ca8, 0x20)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x1b0
github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x1992f40, 0xc42d808000, 0x25810e0, 0xc424d35c20, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:327 +0xbb
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc42e6af7c0, 0x2585580, 0xc42d860d80, 0xc42ead05a0, 0xc42d327200, 0x2555580, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:689 +0x702
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42e6af7c0, 0x2585580, 0xc42d860d80, 0xc42ead05a0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:773 +0xc36
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc424087f30, 0xc42e6af7c0, 0x2585580, 0xc42d860d80, 0xc42ead05a0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:422 +0xab
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:423 +0xa3

goroutine 14308177 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc42d886a00, 0xc429cee3c0, 0xc422a2aa20)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83

goroutine 14373501 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc43013c200, 0xc42eaec970, 0x5, 0x5, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:141 +0x69a
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc42dd92a50, 0xc42eaec970, 0x5, 0x5, 0xc431647c20, 0x6d6a40, 0xc42d79a438)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:314 +0x59
io.ReadAtLeast(0x256b9c0, 0xc42dd92a50, 0xc42eaec970, 0x5, 0x5, 0x5, 0xc431647d78, 0x6e7c56, 0xc42d79a420)
	/usr/local/go/src/io/io.go:307 +0xa4
io.ReadFull(0x256b9c0, 0xc42dd92a50, 0xc42eaec970, 0x5, 0x5, 0xc431647d08, 0x1, 0x0)
	/usr/local/go/src/io/io.go:325 +0x58
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc42eaec960, 0x7fffffff, 0x4, 0x18, 0x1bb7d01, 0x1bb7d00, 0xc41fbba8df, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:231 +0x6f
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc42eaec960, 0x257d700, 0x2a78e08, 0xc42dd92a50, 0x0, 0x0, 0x190e900, 0xc4288ae3e0, 0x7fffffff, 0xc42e5ac201, ...)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:315 +0x4d
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc42a5dc3f0, 0x190e900, 0xc4288ae3e0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:492 +0xec
github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc42e5ac260, 0x257f5c0, 0xc42fda3b30, 0xc430cb7500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:346 +0x62
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x25851c0, 0xc42e5ac260, 0xc42e4b2780, 0xc4294d7788, 0xc4294d7790)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:301 +0x45
github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1()
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:324 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42b02e000, 0xc42fda3b60)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0x7d
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:197 +0x66

goroutine 14634995 [select]:
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc429927800, 0x257f5c0, 0xc428918b10, 0xc4293e0bd0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1397 +0x10da
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryAddWriteCmd(0xc429927800, 0x257f5c0, 0xc428918b10, 0x7b, 0x9e20, 0x100000001, 0x1, 0x1, 0x0, 0xc428114600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1802 +0xbfd
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).addWriteCmd(0xc429927800, 0x257f5c0, 0xc428918b10, 0x7b, 0x9e20, 0x100000001, 0x1, 0x1, 0x0, 0xc428114600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1715 +0x90
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc429927800, 0x257f5c0, 0xc428918b10, 0x7b, 0x9e20, 0x100000001, 0x1, 0x1, 0x0, 0xc428114600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1183 +0x222
github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc422733880, 0x257f5c0, 0xc428918ab0, 0x7b, 0x9e20, 0x100000001, 0x1, 0x1, 0x0, 0xc428114600, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2386 +0x75e
github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc42befd3e0, 0x257f500, 0xc43155fa00, 0x0, 0x0, 0x100000001, 0x1, 0x1, 0x0, 0xc428114540, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:187 +0x202
github.com/cockroachdb/cockroach/pkg/storage_test.(*multiTestContextKVTransport).SendNext.func1(0x257f500, 0xc43155fa00)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/client_test.go:459 +0x136
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc42b02e120, 0x226c6f0, 0x16, 0x1e9, 0x0, 0x0, 0xc42d1b12e0, 0x257f500, 0xc43155fa00)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:264 +0xdf
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:265 +0x234

goroutine 14373194 [select]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc42d886a00, 0xc42f507c20, 0xc42e02d560)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83

goroutine 14335245 [select, 1 minutes]:
github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream.func2(0x2585520, 0xc42f504d00, 0xc42a12f770, 0xc431c450e0)
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:197 +0x426
created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.NewClientStream
	/go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:217 +0xb83


ERROR: exit status 2

make: *** [stress] Error 1
1 runs completed, 1 failures, over 30m12s
Makefile:138: recipe for target 'stress' failed

The text was updated successfully, but these errors were encountered:

tamird · 2016-11-28T15:48:04Z

Something went off the rails here.

[10:16:52][Step 2/2] W161125 10:16:39.195729 1185207 vendor/github.com/coreos/etcd/raft/raft.go:794  [s9,r1/9:/M{in-ax}] 9 stepped down to follower since quorum is not active
[10:16:52][Step 2/2] panic: test timed out after 30m0s

petermattis · 2016-11-28T18:34:45Z

Simpler repro:

cd pkg/storage; go test -v -run TestRaftRemoveRace -tags deadlock

The deadlock tag makes everything horrendously slow. If I change the number of initial replicas from 10 to 3, the test finishes in 25s. I'm not sure this is worth investigating further. Perhaps we should using testing.Short to skip certain tests when running with the deadlock tag.

tamird · 2016-11-28T18:59:21Z

Wouldn't we expect this to fail routinely, then? I don't think skipping a test is the way to go; another options is to jack up the timeout when we run with -deadlock.

petermattis · 2016-11-28T19:09:34Z

With -tags deadlock, the test routinely takes 90+s (without stress). Add in stress and I imagine we could be tickling excessive raft elections or other fun stuff. Tracking down what is going on here is low priority in my opinion, hence the suggestion. Another option is to only look for deadlock errors when running -tags deadlock. Or perhaps run with stress -p 1.

tamird · 2016-11-28T20:18:53Z

On Mon, Nov 28, 2016 at 2:09 PM, Peter Mattis ***@***.***> wrote: Another option is to only look for deadlock errors when running -tags deadlock. Or perhaps run with stress -p 1.

This sounds good to me (looking for deadlock errors only).

tamird · 2016-12-01T19:58:30Z

Instead, I've gone and made go-deadlock 1000% faster: sasha-s/go-deadlock#2.

cockroach-teamcity added O-robot Originated from a bot. C-test-failure Broken test (automatically or manually discovered). labels Nov 25, 2016

tamird assigned petermattis Nov 28, 2016

petermattis assigned tamird and unassigned petermattis Dec 1, 2016

tamird mentioned this issue Dec 1, 2016

github.com/cockroachdb/cockroach/pkg/storage: TestRefreshPendingCommands failed under stress #11643

Closed

tamird closed this as completed Dec 1, 2016

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

github.com/cockroachdb/cockroach/pkg/storage: TestRaftRemoveRace failed under stress #11614

github.com/cockroachdb/cockroach/pkg/storage: TestRaftRemoveRace failed under stress #11614

cockroach-teamcity commented Nov 25, 2016

tamird commented Nov 28, 2016

petermattis commented Nov 28, 2016

tamird commented Nov 28, 2016

petermattis commented Nov 28, 2016

tamird commented Nov 28, 2016 via email

tamird commented Dec 1, 2016

github.com/cockroachdb/cockroach/pkg/storage: TestRaftRemoveRace failed under stress #11614

github.com/cockroachdb/cockroach/pkg/storage: TestRaftRemoveRace failed under stress #11614

Comments

cockroach-teamcity commented Nov 25, 2016

tamird commented Nov 28, 2016

petermattis commented Nov 28, 2016

tamird commented Nov 28, 2016

petermattis commented Nov 28, 2016

tamird commented Nov 28, 2016 via email

tamird commented Dec 1, 2016