1/* $NetBSD: lockstat.c,v 1.24 2015/08/20 14:40:17 christos Exp $ */
2
3/*-
4 * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Lock statistics driver, providing kernel support for the lockstat(8)
34 * command.
35 *
36 * We use a global lock word (lockstat_lock) to track device opens.
37 * Only one thread can hold the device at a time, providing a global lock.
38 *
39 * XXX Timings for contention on sleep locks are currently incorrect.
40 */
41
42#include <sys/cdefs.h>
43__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.24 2015/08/20 14:40:17 christos Exp $");
44
45#include <sys/types.h>
46#include <sys/param.h>
47#include <sys/proc.h>
48#include <sys/resourcevar.h>
49#include <sys/systm.h>
50#include <sys/kernel.h>
51#include <sys/kmem.h>
52#include <sys/conf.h>
53#include <sys/cpu.h>
54#include <sys/syslog.h>
55#include <sys/atomic.h>
56
57#include <dev/lockstat.h>
58
59#include <machine/lock.h>
60
61#include "ioconf.h"
62
63#ifndef __HAVE_CPU_COUNTER
64#error CPU counters not available
65#endif
66
67#if LONG_BIT == 64
68#define LOCKSTAT_HASH_SHIFT 3
69#elif LONG_BIT == 32
70#define LOCKSTAT_HASH_SHIFT 2
71#endif
72
73#define LOCKSTAT_MINBUFS 1000
74#define LOCKSTAT_DEFBUFS 10000
75#define LOCKSTAT_MAXBUFS 1000000
76
77#define LOCKSTAT_HASH_SIZE 128
78#define LOCKSTAT_HASH_MASK (LOCKSTAT_HASH_SIZE - 1)
79#define LOCKSTAT_HASH(key) \
80 ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
81
82typedef struct lscpu {
83 SLIST_HEAD(, lsbuf) lc_free;
84 u_int lc_overflow;
85 LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
86} lscpu_t;
87
88typedef struct lslist lslist_t;
89
90void lockstat_start(lsenable_t *);
91int lockstat_alloc(lsenable_t *);
92void lockstat_init_tables(lsenable_t *);
93int lockstat_stop(lsdisable_t *);
94void lockstat_free(void);
95
96dev_type_open(lockstat_open);
97dev_type_close(lockstat_close);
98dev_type_read(lockstat_read);
99dev_type_ioctl(lockstat_ioctl);
100
101volatile u_int lockstat_enabled;
102volatile u_int lockstat_dev_enabled;
103uintptr_t lockstat_csstart;
104uintptr_t lockstat_csend;
105uintptr_t lockstat_csmask;
106uintptr_t lockstat_lamask;
107uintptr_t lockstat_lockstart;
108uintptr_t lockstat_lockend;
109__cpu_simple_lock_t lockstat_lock;
110lwp_t *lockstat_lwp;
111lsbuf_t *lockstat_baseb;
112size_t lockstat_sizeb;
113int lockstat_busy;
114struct timespec lockstat_stime;
115
116#ifdef KDTRACE_HOOKS
117volatile u_int lockstat_dtrace_enabled;
118CTASSERT(LB_NEVENT <= 3);
119CTASSERT(LB_NLOCK <= (7 << LB_LOCK_SHIFT));
120void
121lockstat_probe_stub(uint32_t id, uintptr_t lock, uintptr_t callsite,
122 uintptr_t flags, uintptr_t count, uintptr_t cycles)
123{
124}
125
126uint32_t lockstat_probemap[LS_NPROBES];
127void (*lockstat_probe_func)(uint32_t, uintptr_t, uintptr_t,
128 uintptr_t, uintptr_t, uintptr_t) = &lockstat_probe_stub;
129#endif
130
131const struct cdevsw lockstat_cdevsw = {
132 .d_open = lockstat_open,
133 .d_close = lockstat_close,
134 .d_read = lockstat_read,
135 .d_write = nowrite,
136 .d_ioctl = lockstat_ioctl,
137 .d_stop = nostop,
138 .d_tty = notty,
139 .d_poll = nopoll,
140 .d_mmap = nommap,
141 .d_kqfilter = nokqfilter,
142 .d_discard = nodiscard,
143 .d_flag = D_OTHER | D_MPSAFE
144};
145
146/*
147 * Called when the pseudo-driver is attached.
148 */
149void
150lockstatattach(int nunits)
151{
152
153 (void)nunits;
154
155 __cpu_simple_lock_init(&lockstat_lock);
156}
157
158/*
159 * Prepare the per-CPU tables for use, or clear down tables when tracing is
160 * stopped.
161 */
162void
163lockstat_init_tables(lsenable_t *le)
164{
165 int i, per, slop, cpuno;
166 CPU_INFO_ITERATOR cii;
167 struct cpu_info *ci;
168 lscpu_t *lc;
169 lsbuf_t *lb;
170
171 /* coverity[assert_side_effect] */
172 KASSERT(!lockstat_dev_enabled);
173
174 for (CPU_INFO_FOREACH(cii, ci)) {
175 if (ci->ci_lockstat != NULL) {
176 kmem_free(ci->ci_lockstat, sizeof(lscpu_t));
177 ci->ci_lockstat = NULL;
178 }
179 }
180
181 if (le == NULL)
182 return;
183
184 lb = lockstat_baseb;
185 per = le->le_nbufs / ncpu;
186 slop = le->le_nbufs - (per * ncpu);
187 cpuno = 0;
188 for (CPU_INFO_FOREACH(cii, ci)) {
189 lc = kmem_alloc(sizeof(*lc), KM_SLEEP);
190 lc->lc_overflow = 0;
191 ci->ci_lockstat = lc;
192
193 SLIST_INIT(&lc->lc_free);
194 for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
195 LIST_INIT(&lc->lc_hash[i]);
196
197 for (i = per; i != 0; i--, lb++) {
198 lb->lb_cpu = (uint16_t)cpuno;
199 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
200 }
201 if (--slop > 0) {
202 lb->lb_cpu = (uint16_t)cpuno;
203 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
204 lb++;
205 }
206 cpuno++;
207 }
208}
209
210/*
211 * Start collecting lock statistics.
212 */
213void
214lockstat_start(lsenable_t *le)
215{
216
217 /* coverity[assert_side_effect] */
218 KASSERT(!lockstat_dev_enabled);
219
220 lockstat_init_tables(le);
221
222 if ((le->le_flags & LE_CALLSITE) != 0)
223 lockstat_csmask = (uintptr_t)-1LL;
224 else
225 lockstat_csmask = 0;
226
227 if ((le->le_flags & LE_LOCK) != 0)
228 lockstat_lamask = (uintptr_t)-1LL;
229 else
230 lockstat_lamask = 0;
231
232 lockstat_csstart = le->le_csstart;
233 lockstat_csend = le->le_csend;
234 lockstat_lockstart = le->le_lockstart;
235 lockstat_lockstart = le->le_lockstart;
236 lockstat_lockend = le->le_lockend;
237 membar_sync();
238 getnanotime(&lockstat_stime);
239 lockstat_dev_enabled = le->le_mask;
240 LOCKSTAT_ENABLED_UPDATE();
241}
242
243/*
244 * Stop collecting lock statistics.
245 */
246int
247lockstat_stop(lsdisable_t *ld)
248{
249 CPU_INFO_ITERATOR cii;
250 struct cpu_info *ci;
251 u_int cpuno, overflow;
252 struct timespec ts;
253 int error;
254 lwp_t *l;
255
256 /* coverity[assert_side_effect] */
257 KASSERT(lockstat_dev_enabled);
258
259 /*
260 * Set enabled false, force a write barrier, and wait for other CPUs
261 * to exit lockstat_event().
262 */
263 lockstat_dev_enabled = 0;
264 LOCKSTAT_ENABLED_UPDATE();
265 getnanotime(&ts);
266 tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
267
268 /*
269 * Did we run out of buffers while tracing?
270 */
271 overflow = 0;
272 for (CPU_INFO_FOREACH(cii, ci))
273 overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
274
275 if (overflow != 0) {
276 error = EOVERFLOW;
277 log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
278 overflow);
279 } else
280 error = 0;
281
282 lockstat_init_tables(NULL);
283
284 /* Run through all LWPs and clear the slate for the next run. */
285 mutex_enter(proc_lock);
286 LIST_FOREACH(l, &alllwp, l_list) {
287 l->l_pfailaddr = 0;
288 l->l_pfailtime = 0;
289 l->l_pfaillock = 0;
290 }
291 mutex_exit(proc_lock);
292
293 if (ld == NULL)
294 return error;
295
296 /*
297 * Fill out the disable struct for the caller.
298 */
299 timespecsub(&ts, &lockstat_stime, &ld->ld_time);
300 ld->ld_size = lockstat_sizeb;
301
302 cpuno = 0;
303 for (CPU_INFO_FOREACH(cii, ci)) {
304 if (cpuno >= sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
305 log(LOG_WARNING, "lockstat: too many CPUs\n");
306 break;
307 }
308 ld->ld_freq[cpuno++] = cpu_frequency(ci);
309 }
310
311 return error;
312}
313
314/*
315 * Allocate buffers for lockstat_start().
316 */
317int
318lockstat_alloc(lsenable_t *le)
319{
320 lsbuf_t *lb;
321 size_t sz;
322
323 /* coverity[assert_side_effect] */
324 KASSERT(!lockstat_dev_enabled);
325 lockstat_free();
326
327 sz = sizeof(*lb) * le->le_nbufs;
328
329 lb = kmem_zalloc(sz, KM_SLEEP);
330 if (lb == NULL)
331 return (ENOMEM);
332
333 /* coverity[assert_side_effect] */
334 KASSERT(!lockstat_dev_enabled);
335 KASSERT(lockstat_baseb == NULL);
336 lockstat_sizeb = sz;
337 lockstat_baseb = lb;
338
339 return (0);
340}
341
342/*
343 * Free allocated buffers after tracing has stopped.
344 */
345void
346lockstat_free(void)
347{
348
349 /* coverity[assert_side_effect] */
350 KASSERT(!lockstat_dev_enabled);
351
352 if (lockstat_baseb != NULL) {
353 kmem_free(lockstat_baseb, lockstat_sizeb);
354 lockstat_baseb = NULL;
355 }
356}
357
358/*
359 * Main entry point from lock primatives.
360 */
361void
362lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
363 uint64_t cycles)
364{
365 lslist_t *ll;
366 lscpu_t *lc;
367 lsbuf_t *lb;
368 u_int event;
369 int s;
370
371#ifdef KDTRACE_HOOKS
372 uint32_t id;
373 CTASSERT((LS_NPROBES & (LS_NPROBES - 1)) == 0);
374 if ((id = lockstat_probemap[LS_COMPRESS(flags)]) != 0)
375 (*lockstat_probe_func)(id, lock, callsite, flags, count,
376 cycles);
377#endif
378
379 if ((flags & lockstat_dev_enabled) != flags || count == 0)
380 return;
381 if (lock < lockstat_lockstart || lock > lockstat_lockend)
382 return;
383 if (callsite < lockstat_csstart || callsite > lockstat_csend)
384 return;
385
386 callsite &= lockstat_csmask;
387 lock &= lockstat_lamask;
388
389 /*
390 * Find the table for this lock+callsite pair, and try to locate a
391 * buffer with the same key.
392 */
393 s = splhigh();
394 lc = curcpu()->ci_lockstat;
395 ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
396 event = (flags & LB_EVENT_MASK) - 1;
397
398 LIST_FOREACH(lb, ll, lb_chain.list) {
399 if (lb->lb_lock == lock && lb->lb_callsite == callsite)
400 break;
401 }
402
403 if (lb != NULL) {
404 /*
405 * We found a record. Move it to the front of the list, as
406 * we're likely to hit it again soon.
407 */
408 if (lb != LIST_FIRST(ll)) {
409 LIST_REMOVE(lb, lb_chain.list);
410 LIST_INSERT_HEAD(ll, lb, lb_chain.list);
411 }
412 lb->lb_counts[event] += count;
413 lb->lb_times[event] += cycles;
414 } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
415 /*
416 * Pinch a new buffer and fill it out.
417 */
418 SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
419 LIST_INSERT_HEAD(ll, lb, lb_chain.list);
420 lb->lb_flags = (uint16_t)flags;
421 lb->lb_lock = lock;
422 lb->lb_callsite = callsite;
423 lb->lb_counts[event] = count;
424 lb->lb_times[event] = cycles;
425 } else {
426 /*
427 * We didn't find a buffer and there were none free.
428 * lockstat_stop() will notice later on and report the
429 * error.
430 */
431 lc->lc_overflow++;
432 }
433
434 splx(s);
435}
436
437/*
438 * Accept an open() on /dev/lockstat.
439 */
440int
441lockstat_open(dev_t dev, int flag, int mode, lwp_t *l)
442{
443
444 if (!__cpu_simple_lock_try(&lockstat_lock))
445 return EBUSY;
446 lockstat_lwp = curlwp;
447 return 0;
448}
449
450/*
451 * Accept the last close() on /dev/lockstat.
452 */
453int
454lockstat_close(dev_t dev, int flag, int mode, lwp_t *l)
455{
456
457 lockstat_lwp = NULL;
458 __cpu_simple_unlock(&lockstat_lock);
459 return 0;
460}
461
462/*
463 * Handle control operations.
464 */
465int
466lockstat_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
467{
468 lsenable_t *le;
469 int error;
470
471 if (lockstat_lwp != curlwp)
472 return EBUSY;
473
474 switch (cmd) {
475 case IOC_LOCKSTAT_GVERSION:
476 *(int *)data = LS_VERSION;
477 error = 0;
478 break;
479
480 case IOC_LOCKSTAT_ENABLE:
481 le = (lsenable_t *)data;
482
483 if (!cpu_hascounter()) {
484 error = ENODEV;
485 break;
486 }
487 if (lockstat_dev_enabled) {
488 error = EBUSY;
489 break;
490 }
491
492 /*
493 * Sanitize the arguments passed in and set up filtering.
494 */
495 if (le->le_nbufs == 0)
496 le->le_nbufs = LOCKSTAT_DEFBUFS;
497 else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
498 le->le_nbufs < LOCKSTAT_MINBUFS) {
499 error = EINVAL;
500 break;
501 }
502 if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
503 le->le_csstart = 0;
504 le->le_csend = le->le_csstart - 1;
505 }
506 if ((le->le_flags & LE_ONE_LOCK) == 0) {
507 le->le_lockstart = 0;
508 le->le_lockend = le->le_lockstart - 1;
509 }
510 if ((le->le_mask & LB_EVENT_MASK) == 0)
511 return EINVAL;
512 if ((le->le_mask & LB_LOCK_MASK) == 0)
513 return EINVAL;
514
515 /*
516 * Start tracing.
517 */
518 if ((error = lockstat_alloc(le)) == 0)
519 lockstat_start(le);
520 break;
521
522 case IOC_LOCKSTAT_DISABLE:
523 if (!lockstat_dev_enabled)
524 error = EINVAL;
525 else
526 error = lockstat_stop((lsdisable_t *)data);
527 break;
528
529 default:
530 error = ENOTTY;
531 break;
532 }
533
534 return error;
535}
536
537/*
538 * Copy buffers out to user-space.
539 */
540int
541lockstat_read(dev_t dev, struct uio *uio, int flag)
542{
543
544 if (curlwp != lockstat_lwp || lockstat_dev_enabled)
545 return EBUSY;
546 return uiomove(lockstat_baseb, lockstat_sizeb, uio);
547}
548