root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 72018 events in 2.696 seconds (26715 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 24062 33% 33% 0.00 10001 page pv 0xc000000003142640 19472 27% 60% 0.00 11308 page pv moea64_enter+0x86c 19079 26% 87% 0.00 10430 page pv (usermode) 1739 2% 89% 0.00 7886 vm active pagequeue 0xc00000000310e854 1719 2% 92% 0.00 9171 vm page vm_page_change_lock+0x160 679 1% 93% 0.00 6555 vnode interlock vget+0x2f8 413 1% 93% 0.00 13109 vm page vm_fault_hold+0x2724 392 1% 94% 0.00 198971 zio_write_issue 0xc000000002d7c86c 377 1% 94% 0.00 6330 vnode interlock (usermode) 337 0% 95% 0.00 7635 vm active pagequeue (usermode) 314 0% 95% 0.00 282966 zio_write_issue (usermode) 242 0% 96% 0.00 17934 dp_sync_taskq (usermode) 228 0% 96% 0.00 8213 vnode interlock 0xc000000002e224a0 226 0% 96% 0.00 223235 zio_write_issue taskqueue_enqueue+0x158 212 0% 96% 0.00 8244 struct mount mtx vfs_unbusy+0x15c 208 0% 97% 0.00 17414 dp_sync_taskq 0xc000000002cc30c0 199 0% 97% 0.00 27458 dp_sync_taskq 0xc000000002d7c86c 198 0% 97% 0.00 106661 zio_write_issue 0xc000000002cc30c0 156 0% 98% 0.00 22154 zio_write_intr (usermode) 144 0% 98% 0.00 52798 kernelpmap moea64_kextract+0x174 140 0% 98% 0.00 5906 struct mount mtx vfs_busy+0x434 123 0% 98% 0.00 16287 dp_sync_taskq taskqueue_enqueue+0x158 116 0% 98% 0.00 20914 zio_write_intr 0xc000000002d7c86c 111 0% 98% 0.00 9649 pmap moea64_enter+0x880 89 0% 99% 0.00 9223 vm page (usermode) 87 0% 99% 0.00 13139 zio_write_intr 0xc000000002cc30c0 79 0% 99% 0.00 5278 struct mount mtx (usermode) 73 0% 99% 0.00 10759 vnode interlock 0xc000000002e22484 71 0% 99% 0.00 563464 process lock ast+0x7e0 63 0% 99% 0.00 7453 page pv moea64_enter+0x910 59 0% 99% 0.00 20172 zio_write_intr taskqueue_enqueue+0x158 49 0% 99% 0.00 32758 kernelpmap moea64_enter+0x880 39 0% 99% 0.00 7096 vnode interlock 0xc0000000030ce5b8 37 0% 99% 0.00 9162 pmap (usermode) 36 0% 99% 0.00 11506 vm active pagequeue vm_page_dequeue+0x1b4 32 0% 99% 0.00 1107 struct mount vlist mtx _vdrop+0x6f4 29 0% 99% 0.00 222020 process lock kern_thr_exit+0x12c 23 0% 99% 0.00 5658 umtxql (usermode) 21 0% 100% 0.00 7765 select mtxpool 0xc000000002d93038 19 0% 100% 0.00 146626 process lock kern_thr_exit+0x24c 18 0% 100% 0.00 29456 umtxql 0xc000000002d245d4 18 0% 100% 0.00 40403 select mtxpool selrecord+0x1dc 18 0% 100% 0.00 133988 kernelpmap (usermode) 16 0% 100% 0.00 6375 umtxql 0xc000000002d228cc 16 0% 100% 0.00 900 UPVO entry uma_zfree_arg+0x5c8 14 0% 100% 0.00 5147 struct mount vlist mtx _vhold+0x474 12 0% 100% 0.00 13804 vm page 0xc0000000030ec28c 11 0% 100% 0.00 5650 ttymtx (usermode) 11 0% 100% 0.00 799 struct mount vlist mtx (usermode) 11 0% 100% 0.00 39895 pipe mutex 0xc000000002d9456c 10 0% 100% 0.00 713 UPVO entry (usermode) 9 0% 100% 0.00 8345 vm page free queue 0xc000000003111274 9 0% 100% 0.00 10988 ttymtx ttyoutq_read_uio+0x34c 9 0% 100% 0.00 55191 kernelpmap moea64_remove+0x200 9 0% 100% 0.00 735 UPVO entry uma_zalloc_arg+0x98c 8 0% 100% 0.00 6597 umtxql 0xc000000002d253f8 7 0% 100% 0.00 2755 pipe mutex 0xc000000002d93a60 7 0% 100% 0.00 4577 CAM device lock 0xc000000002524148 6 0% 100% 0.00 5456 umtxql 0xc000000002d25428 6 0% 100% 0.00 214 pipe mutex 0xc000000002d95074 6 0% 100% 0.00 184430 nvme qpair lock 0xc000000002a101b0 5 0% 100% 0.00 1770 ttymtx 0xc000000002db84b4 5 0% 100% 0.00 466 ttymtx 0xc000000002db7520 5 0% 100% 0.00 260192 process lock (usermode) 4 0% 100% 0.00 20292 vm active pagequeue 0xc000000003114894 4 0% 100% 0.00 16837 umtxql 0xc000000002d22b78 4 0% 100% 0.00 3364 umtxql 0xc000000002cc30c0 4 0% 100% 0.00 212 pipe mutex 0xc000000002d95914 4 0% 100% 0.00 67432 pipe mutex (usermode) 3 0% 100% 0.00 7988 vnode interlock vinactive+0x25c 3 0% 100% 0.00 5165 vm page free queue (usermode) 3 0% 100% 0.00 6119 pipe mutex 0xc000000002d93828 3 0% 100% 0.00 11790 kernelpmap moea64_kenter_attr+0x3d4 3 0% 100% 0.00 9374 CAM device lock (usermode) 2 0% 100% 0.00 721 vm page vm_imgact_unmap_page+0x184 2 0% 100% 0.00 5493 ttymtx ttydisc_write+0x388 2 0% 100% 0.00 942 ttymtx 0xc000000002db18d8 2 0% 100% 0.00 176243 select mtxpool (usermode) 2 0% 100% 0.00 222 ncneglh cache_lookup+0x1638 2 0% 100% 0.00 12283 dp_sync_taskq taskqueue_quiesce+0x2e8 2 0% 100% 0.00 925 CAM device lock 0xc000000002524ac0 1 0% 100% 0.00 184 zio_free_issue_3 0xc000000002d7c86c 1 0% 100% 0.00 306 zio_free_issue_3 taskqueue_enqueue+0x158 1 0% 100% 0.00 778 zio_free_issue_3 0xc000000002cc30c0 1 0% 100% 0.00 248 zio_free_issue_4 taskqueue_enqueue+0x158 1 0% 100% 0.00 154 zio_free_issue_3 (usermode) 1 0% 100% 0.00 459 zio_free_issue_2 0xc000000002cc30c0 1 0% 100% 0.00 175 vnode_free_list getnewvnode_reserve+0x480 1 0% 100% 0.00 5609 vm page free queue 0xc000000003110fb4 1 0% 100% 0.00 668 vnode interlock 0xc000000002ce0a70 1 0% 100% 0.00 1041 vnode interlock 0xc000000002e09c84 1 0% 100% 0.00 4078 vm page 0xc000000002df9ce0 1 0% 100% 0.00 1146 vm active pagequeue 0xc000000003114bd8 1 0% 100% 0.00 1471 vm page vm_page_rename+0x224 1 0% 100% 0.00 10612 struct mount mtx vn_finished_write+0x174 1 0% 100% 0.00 1047 struct mount mtx vfs_ref+0x114 1 0% 100% 0.00 5816 struct mount mtx 0xc000000002e23bdc 1 0% 100% 0.00 203 struct mount mtx vfs_notify_upper+0x460 1 0% 100% 0.00 5921 umtxql 0xc000000002d2546c 1 0% 100% 0.00 1042 ncvn cache_lookup+0xa9c 1 0% 100% 0.00 3619 metaslab_group_taskq (usermode) 1 0% 100% 0.00 176 Global Softdep Lock (usermode) 1 0% 100% 0.00 260 FFS (usermode) 1 0% 100% 0.00 1123 CAM device lock 0xc000000002523518 1 0% 100% 0.00 201 SLB table moea64_release_vsid+0x144 1 0% 100% 0.00 5558 Global Softdep Lock 0xc0000000030958c8 1 0% 100% 0.00 371 process lock kern_sigprocmask+0x378 1 0% 100% 0.00 10815 pmap moea64_is_prefaultable+0x13c 1 0% 100% 0.00 207 process lock kthread_suspend_check+0x184 1 0% 100% 0.00 180 select mtxpool 0xc000000002d92a68 1 0% 100% 0.00 10241 sellck 0xc000000002d92870 1 0% 100% 0.00 382 sleep mtxpool g_io_request+0x414 ------------------------------------------------------------------------------- Adaptive mutex block: 16 events in 2.696 seconds (6 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 3 19% 19% 0.00 413269 vm page vm_page_change_lock+0x160 3 19% 38% 0.00 39336 kernelpmap moea64_enter+0x880 2 12% 50% 0.00 26337 zio_write_intr (usermode) 2 12% 62% 0.00 32719 page pv 0xc000000003142640 2 12% 75% 0.00 49695 page pv moea64_enter+0x910 2 12% 88% 0.00 1081951 page pv moea64_enter+0x86c 1 6% 94% 0.00 32401 CAM device lock 0xc000000002524ac0 1 6% 100% 0.00 54953 kernelpmap moea64_kextract+0x174 ------------------------------------------------------------------------------- Spin lock spin: 28104 events in 2.696 seconds (10425 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 13522 48% 48% 0.00 425486 turnstile chain (usermode) 12564 45% 93% 0.00 367679 turnstile chain turnstile_trywait+0x208 842 3% 96% 0.00 12276 sleepq chain sleepq_lock+0xe4 407 1% 97% 0.00 17517 sleepq chain (usermode) 304 1% 98% 0.00 133984 turnstile chain turnstile_chain_lock+0xdc 168 1% 99% 0.00 545 turnstile lock turnstile_trywait+0x150 161 1% 100% 0.00 571 turnstile lock (usermode) 13 0% 100% 0.00 3731 sched lock 25 0xc000000002d3cccc 10 0% 100% 0.00 4735 sched lock 0 0xc000000002d3cccc 9 0% 100% 0.00 2503 sched lock 30 0xc000000002d3cccc 4 0% 100% 0.00 11873 sched lock 30 0xc000000002d3cce4 4 0% 100% 0.00 16929 sched lock 25 (usermode) 4 0% 100% 0.00 43134 sched lock 15 0xc000000002d3cccc 3 0% 100% 0.00 327 sched lock 30 sched_add+0x48c 3 0% 100% 0.00 14586 sched lock 30 (usermode) 2 0% 100% 0.00 2667 turnstile lock turnstile_lookup+0xd4 2 0% 100% 0.00 3699 sched lock 70 0xc000000002d3cce4 2 0% 100% 0.00 2579 sched lock 70 sched_add+0x48c 2 0% 100% 0.00 3856 sched lock 6 (usermode) 2 0% 100% 0.00 761 sched lock 52 sched_add+0x48c 2 0% 100% 0.00 934 sched lock 50 0xc000000002d3cce4 2 0% 100% 0.00 517 sched lock 5 sched_switch+0x81c 2 0% 100% 0.00 300 sched lock 44 (usermode) 2 0% 100% 0.00 7883 sched lock 4 0xc000000002d3cccc 2 0% 100% 0.00 577 sched lock 35 sched_add+0x48c 2 0% 100% 0.00 2855 sched lock 31 (usermode) 2 0% 100% 0.00 430 sched lock 23 sched_add+0x48c 2 0% 100% 0.00 1900 sched lock 15 sched_switch+0x81c 2 0% 100% 0.00 3313 sched lock 15 (usermode) 2 0% 100% 0.00 74220 sched lock 14 0xc000000002d3cccc 2 0% 100% 0.00 22155 sched lock 10 0xc000000002d3cccc 2 0% 100% 0.00 5916 sched lock 10 (usermode) 2 0% 100% 0.00 5987 callout (usermode) 1 0% 100% 0.00 853 sched lock 9 0xc000000002d3cccc 1 0% 100% 0.00 1947 sched lock 8 sched_add+0x48c 1 0% 100% 0.00 23135 sched lock 8 0xc000000002d3cccc 1 0% 100% 0.00 46563 sched lock 8 0xc000000002d3cce4 1 0% 100% 0.00 4035 sched lock 9 sched_add+0x48c 1 0% 100% 0.00 280 sched lock 60 sched_add+0x48c 1 0% 100% 0.00 682 sched lock 60 (usermode) 1 0% 100% 0.00 3675 sched lock 6 sched_add+0x48c 1 0% 100% 0.00 2019 sched lock 56 sched_add+0x48c 1 0% 100% 0.00 822 sched lock 57 (usermode) 1 0% 100% 0.00 9866 sched lock 65 sched_switch+0x81c 1 0% 100% 0.00 1914 sched lock 64 sched_add+0x48c 1 0% 100% 0.00 1341 sched lock 63 0xc000000002d3cce4 1 0% 100% 0.00 279 sched lock 67 sched_add+0x48c 1 0% 100% 0.00 289 sched lock 66 0xc000000002d3cce4 1 0% 100% 0.00 1363 sched lock 69 0xc000000002d3cce4 1 0% 100% 0.00 972 sched lock 70 (usermode) 1 0% 100% 0.00 443 sched lock 4 sched_add+0x48c 1 0% 100% 0.00 1041 sched lock 36 sched_add+0x48c 1 0% 100% 0.00 3132 sched lock 35 0xc000000002d3cce4 1 0% 100% 0.00 324 sched lock 35 (usermode) 1 0% 100% 0.00 1385 sched lock 35 0xc000000002d3cccc 1 0% 100% 0.00 388 sched lock 32 0xc000000002d3cce4 1 0% 100% 0.00 724 sched lock 33 sched_add+0x48c 1 0% 100% 0.00 251 sched lock 34 sched_switch+0x81c 1 0% 100% 0.00 1109 sched lock 34 sched_add+0x48c 1 0% 100% 0.00 19513 sched lock 46 0xc000000002d3cce4 1 0% 100% 0.00 647 sched lock 46 sched_add+0x48c 1 0% 100% 0.00 690 sched lock 45 0xc000000002d3cce4 1 0% 100% 0.00 1102 sched lock 41 0xc000000002d3cccc 1 0% 100% 0.00 1037 sched lock 5 0xc000000002d3cccc 1 0% 100% 0.00 1010 sched lock 52 (usermode) 1 0% 100% 0.00 687 sched lock 53 0xc000000002d3cccc 1 0% 100% 0.00 5103 sched lock 55 0xc000000002d3cce4 1 0% 100% 0.00 228 callout 0xc000000002d1b164 1 0% 100% 0.00 3498 sched lock 0 sched_switch+0x81c 1 0% 100% 0.00 3407 sched lock 2 sched_add+0x48c 1 0% 100% 0.00 3548 sched lock 16 (usermode) 1 0% 100% 0.00 484 sched lock 15 0xc000000002d3cce4 1 0% 100% 0.00 3072 sched lock 15 sched_add+0x48c 1 0% 100% 0.00 1930 sched lock 11 0xc000000002d3cccc 1 0% 100% 0.00 1279 sched lock 11 (usermode) 1 0% 100% 0.00 2007 sched lock 12 (usermode) 1 0% 100% 0.00 3039 sched lock 25 sched_add+0x48c 1 0% 100% 0.00 80172 sched lock 25 sched_switch+0xca0 1 0% 100% 0.00 1142 sched lock 20 (usermode) 1 0% 100% 0.00 193 sched lock 22 sched_add+0x48c 1 0% 100% 0.00 3377 sched lock 29 0xc000000002d3cccc 1 0% 100% 0.00 1031 sched lock 28 (usermode) 1 0% 100% 0.00 398 sched lock 3 (usermode) ------------------------------------------------------------------------------- Thread lock spin: 520 events in 2.696 seconds (193 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 307 59% 59% 0.00 3934 sleepq chain _thread_lock+0xf4 119 23% 82% 0.00 3718 sleepq chain (usermode) 18 3% 85% 0.00 5354 turnstile lock _thread_lock+0xf4 6 1% 87% 0.00 5569 sched lock 30 _thread_lock+0xf4 4 1% 87% 0.00 5733 sched lock 55 _thread_lock+0xf4 3 1% 88% 0.00 16955 sched lock 5 _thread_lock+0xf4 3 1% 88% 0.00 736 sched lock 44 _thread_lock+0xf4 3 1% 89% 0.00 10408 sched lock 30 (usermode) 3 1% 90% 0.00 3209 sched lock 25 _thread_lock+0xf4 3 1% 90% 0.00 4660 sched lock 15 (usermode) 3 1% 91% 0.00 112525 sched lock 14 _thread_lock+0xf4 2 0% 91% 0.00 1804 sched lock 35 _thread_lock+0xf4 2 0% 92% 0.00 2703 sched lock 35 (usermode) 2 0% 92% 0.00 432 sched lock 32 (usermode) 2 0% 92% 0.00 912 sched lock 31 (usermode) 2 0% 93% 0.00 4043850 sched lock 25 (usermode) 2 0% 93% 0.00 6205 sched lock 11 _thread_lock+0xf4 2 0% 93% 0.00 12449 sched lock 0 _thread_lock+0xf4 1 0% 94% 0.00 2882 turnstile lock (usermode) 1 0% 94% 0.00 15777 sched lock 8 (usermode) 1 0% 94% 0.00 5501 sched lock 70 _thread_lock+0xf4 1 0% 94% 0.00 359 sched lock 67 _thread_lock+0xf4 1 0% 94% 0.00 508 sched lock 70 (usermode) 1 0% 95% 0.00 547 sched lock 60 (usermode) 1 0% 95% 0.00 248 sched lock 64 _thread_lock+0xf4 1 0% 95% 0.00 11943 sched lock 67 (usermode) 1 0% 95% 0.00 512 sched lock 55 (usermode) 1 0% 95% 0.00 1260 sched lock 53 (usermode) 1 0% 96% 0.00 612 sched lock 56 (usermode) 1 0% 96% 0.00 3310 sched lock 6 (usermode) 1 0% 96% 0.00 77394 sched lock 6 _thread_lock+0xf4 1 0% 96% 0.00 570 sched lock 44 (usermode) 1 0% 96% 0.00 14244 sched lock 36 (usermode) 1 0% 97% 0.00 365 sched lock 32 _thread_lock+0xf4 1 0% 97% 0.00 39568 sched lock 33 (usermode) 1 0% 97% 0.00 419 sched lock 49 (usermode) 1 0% 97% 0.00 3132 sched lock 46 (usermode) 1 0% 97% 0.00 3128 sched lock 50 _thread_lock+0xf4 1 0% 98% 0.00 3222 sched lock 52 (usermode) 1 0% 98% 0.00 432 sched lock 18 _thread_lock+0xf4 1 0% 98% 0.00 1301 sched lock 17 _thread_lock+0xf4 1 0% 98% 0.00 414 sched lock 16 _thread_lock+0xf4 1 0% 98% 0.00 10745 sched lock 15 _thread_lock+0xf4 1 0% 98% 0.00 7749 sched lock 16 (usermode) 1 0% 99% 0.00 523 sched lock 11 (usermode) 1 0% 99% 0.00 852 sched lock 26 (usermode) 1 0% 99% 0.00 20044 sched lock 19 (usermode) 1 0% 99% 0.00 769 sched lock 19 _thread_lock+0xf4 1 0% 99% 0.00 1080 sched lock 22 _thread_lock+0xf4 1 0% 100% 0.00 529 sched lock 29 (usermode) 1 0% 100% 0.00 2109 sched lock 27 _thread_lock+0xf4 1 0% 100% 0.00 6890 sched lock 31 _thread_lock+0xf4 ------------------------------------------------------------------------------- R/W writer blocked by writer: 894 events in 2.696 seconds (332 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 197 22% 22% 0.00 64240854 vm object vm_fault_hold+0x1b28 151 17% 39% 0.00 62592083 vm object vm_object_deallocate+0x800 135 15% 54% 0.00 55282920 vm object vm_map_insert+0x598 95 11% 65% 0.00 64436289 vm object vm_object_reference+0x134 79 9% 73% 0.00 73471835 vm object (usermode) 66 7% 81% 0.00 75332474 vm object 0xc0000000030f8558 62 7% 88% 0.00 77355507 vm object vm_object_shadow+0x390 54 6% 94% 0.00 64459177 vm object vm_object_shadow+0x378 25 3% 97% 0.00 55100176 vm object vm_imgact_map_page+0x3f0 23 3% 99% 0.00 60687006 vm object exec_map_first_page+0x780 3 0% 100% 0.00 58903387 vm object vm_object_deallocate+0x814 2 0% 100% 0.00 29121972 vm object vnode_create_vobject+0x18c 1 0% 100% 0.00 2498097 kernel vm object kmem_back_domain+0x340 1 0% 100% 0.00 2520713 kernel vm object 0xc0000000030f09c0 ------------------------------------------------------------------------------- R/W writer blocked by readers: 1501 events in 2.696 seconds (557 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 620 41% 41% 0.00 57579178 vm object vm_fault_hold+0x1b28 160 11% 52% 0.00 46677364 vm object (usermode) 156 10% 62% 0.00 50217212 vm object vm_object_deallocate+0x800 124 8% 71% 0.00 51981309 vm object vnode_create_vobject+0x18c 124 8% 79% 0.00 67387571 vm object vm_object_deallocate+0x814 124 8% 87% 0.00 64890289 vm object 0xc0000000030f8558 74 5% 92% 0.00 44336294 vm object vm_object_reference+0x134 39 3% 95% 0.00 36147942 vm object vm_map_insert+0x598 37 2% 97% 0.00 37718377 vm object vm_imgact_map_page+0x3f0 27 2% 99% 0.00 55013975 vm object vm_object_shadow+0x378 11 1% 100% 0.00 43236490 vm object vm_object_shadow+0x390 5 0% 100% 0.00 24315923 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader blocked by writer: 303 events in 2.696 seconds (112 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 191 63% 63% 0.00 32673254 vm object __rw_rlock_int+0x9c 112 37% 100% 0.00 16428365 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 58 events in 2.696 seconds (22 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 43 74% 74% 0.00 42100638 vm object __rw_rlock_int+0x9c 15 26% 100% 0.00 21803674 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 1565 events in 2.696 seconds (581 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 296 19% 19% 0.00 4850051 vm object vm_fault_hold+0x1b28 246 16% 35% 0.00 4763758 vm object vm_object_deallocate+0x800 199 13% 47% 0.00 3999784 vm object vm_map_insert+0x598 160 10% 58% 0.00 4133141 vm object vm_object_reference+0x134 149 10% 67% 0.00 4744520 vm object (usermode) 92 6% 73% 0.00 6909337 vm object 0xc0000000030f8558 91 6% 79% 0.00 13710 vm object vm_fault_hold+0x2700 84 5% 84% 0.00 6537844 vm object vm_object_shadow+0x390 79 5% 89% 0.00 5099164 vm object vm_object_shadow+0x378 61 4% 93% 0.00 2613039 vm object vm_imgact_map_page+0x3f0 29 2% 95% 0.00 4610199 vm object exec_map_first_page+0x780 14 1% 96% 0.00 152461 kernel vm object kmem_back_domain+0x340 12 1% 97% 0.00 264907 kernel vm object 0xc0000000030f09c0 9 1% 97% 0.00 22365 Per-Filesystem Softdep Lock (usermode) 5 0% 98% 0.00 18847 vm object 0xc00000000311f65c 5 0% 98% 0.00 3562938 vm object vm_object_deallocate+0x814 5 0% 98% 0.00 12066 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 4 0% 98% 0.00 23865 vm object vm_object_collapse+0x710 4 0% 99% 0.00 79379 kernel vm object (usermode) 3 0% 99% 0.00 2289133 vm object vnode_create_vobject+0x18c 3 0% 99% 0.00 18706 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 3 0% 99% 0.00 7681 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 3 0% 99% 0.00 19407 Per-Filesystem Softdep Lock 0xc000000003095904 2 0% 100% 0.00 54655 vm object vm_fault_hold+0x1298 2 0% 100% 0.00 51207 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 2 0% 100% 0.00 8357 Per-Filesystem Softdep Lock 0xc00000000309f2b4 1 0% 100% 0.00 19911 bufobj interlock (usermode) 1 0% 100% 0.00 744 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c 1 0% 100% 0.00 29774 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c ------------------------------------------------------------------------------- R/W writer spin on readers: 1600 events in 2.696 seconds (594 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 626 39% 39% 0.00 6084336 vm object vm_fault_hold+0x1b28 168 10% 50% 0.00 7478029 vm object (usermode) 165 10% 60% 0.00 5257995 vm object vm_object_deallocate+0x800 129 8% 68% 0.00 7792777 vm object vm_object_deallocate+0x814 126 8% 76% 0.00 7730503 vm object 0xc0000000030f8558 124 8% 84% 0.00 5194940 vm object vnode_create_vobject+0x18c 76 5% 88% 0.00 4220377 vm object vm_object_reference+0x134 44 3% 91% 0.00 2511180 vm object vm_map_insert+0x598 38 2% 94% 0.00 3101251 vm object vm_imgact_map_page+0x3f0 29 2% 95% 0.00 18063 vm object vm_fault_hold+0x2700 27 2% 97% 0.00 6946689 vm object vm_object_shadow+0x378 26 2% 99% 0.00 9782 vm object vm_fault_hold+0x1298 14 1% 100% 0.00 3050279 vm object vm_object_shadow+0x390 6 0% 100% 0.00 1617495 vm object exec_map_first_page+0x780 1 0% 100% 0.00 6151 vm object 0xc00000000311f65c 1 0% 100% 0.00 237 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W reader spin on writer: 503 events in 2.696 seconds (187 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 335 67% 67% 0.00 1184277 vm object __rw_rlock_int+0x9c 168 33% 100% 0.00 990629 vm object (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 54 events in 2.696 seconds (20 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 39 72% 72% 0.00 1619641 vm object __rw_rlock_int+0x9c 15 28% 100% 0.00 1292441 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 2 events in 2.696 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2 100% 100% 0.00 51767 vm map (user) (usermode) ------------------------------------------------------------------------------- SX exclusive spin: 2220 events in 2.696 seconds (824 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 427 19% 19% 0.00 3680 rrl->rr_lock 0xc00000000260abbc 329 15% 34% 0.00 3865 rrl->rr_lock rrw_exit+0x310 214 10% 44% 0.00 31980 vm map (user) (usermode) 126 6% 49% 0.00 39352 proctree kern_wait6+0x3e8 109 5% 54% 0.00 77174 vq->vq_lock vdev_queue_io+0x268 105 5% 59% 0.00 2177 zp->z_acl_lock 0xc0000000026790e0 86 4% 63% 0.00 26856 tm->tm_lock trim_map_free+0x19c 61 3% 66% 0.00 668715 os->os_userused_lock (usermode) 59 3% 68% 0.00 63839 vq->vq_lock (usermode) 53 2% 71% 0.00 26211 ms->ms_lock metaslab_alloc_dva+0x1138 48 2% 73% 0.00 3600 rrl->rr_lock (usermode) 47 2% 75% 0.00 21451 vm map (user) _vm_map_lock+0x138 41 2% 77% 0.00 589481 os->os_userused_lock 0xc0000000025b7cdc 41 2% 79% 0.00 15172 db->db_mtx dmu_buf_will_dirty+0x2e8 31 1% 80% 0.00 5280 zp->z_acl_lock (usermode) 27 1% 81% 0.00 7760 dd->dd_lock dsl_dir_tempreserve_clear+0x88 26 1% 82% 0.00 7602 zio->io_lock 0xc0000000026bfad8 24 1% 84% 0.00 516920 os->os_userused_lock 0xc0000000025b7e74 22 1% 85% 0.00 19174 zio->io_lock 0xc0000000026c1ac0 22 1% 85% 0.00 4165 zio->io_lock (usermode) 21 1% 86% 0.00 17723 scl->scl_lock spa_config_enter+0x140 19 1% 87% 0.00 6586 zio->io_lock zio_add_child+0x274 19 1% 88% 0.00 5903 mg->mg_lock metaslab_alloc_dva+0x3a8 18 1% 89% 0.00 7038 mc->mc_lock metaslab_class_throttle_unreserve+0x154 17 1% 90% 0.00 13284 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x268 17 1% 90% 0.00 10803 scl->scl_lock spa_config_exit+0x84 16 1% 91% 0.00 4338 mg->mg_lock (usermode) 15 1% 92% 0.00 67127 vq->vq_lock 0xc0000000026563c8 14 1% 93% 0.00 13299 ms->ms_lock (usermode) 14 1% 93% 0.00 13022 mg->mg_lock metaslab_alloc_dva+0x7d4 12 1% 94% 0.00 18503 vm map (user) _vm_map_lock_upgrade+0x180 12 1% 94% 0.00 9395 spa->spa_alloc_locks[i] (usermode) 12 1% 95% 0.00 21908 dp->dp_lock dsl_pool_undirty_space+0x18c 12 1% 95% 0.00 7167 dd->dd_lock (usermode) 11 0% 96% 0.00 6328 vq->vq_lock vdev_queue_io_done+0x278 9 0% 96% 0.00 3159 dd->dd_lock 0xc0000000025e7c24 8 0% 97% 0.00 5916 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 7 0% 97% 0.00 4378 vq->vq_lock vdev_queue_io_done+0x12c 7 0% 97% 0.00 7959 spa->spa_alloc_locks[i] 0xc0000000026bd584 6 0% 97% 0.00 2397 scl->scl_lock (usermode) 5 0% 98% 0.00 158658 proctree (usermode) 5 0% 98% 0.00 28416 db->db_mtx (usermode) 4 0% 98% 0.00 3266 hdl->sa_lock sa_lookup_uio+0x198 4 0% 98% 0.00 5622 dd->dd_lock dsl_dir_diduse_space+0x270 4 0% 98% 0.00 7247 db->db_mtx dbuf_rele+0xbc 3 0% 99% 0.00 2531 tm->tm_lock (usermode) 3 0% 99% 0.00 33563 proctree exit1+0x1aec 3 0% 99% 0.00 6945 hdl->sa_lock (usermode) 3 0% 99% 0.00 10301 h->hash_mutexes[i] dbuf_find+0x114 3 0% 99% 0.00 2139 dp->dp_lock dsl_pool_need_dirty_delay+0x1ac 2 0% 99% 0.00 3602 hdl->sa_lock sa_bulk_lookup+0x154 2 0% 99% 0.00 1680 mc->mc_lock (usermode) 2 0% 99% 0.00 10514 h->hash_mutexes[i] (usermode) 2 0% 100% 0.00 6987 dp->dp_lock (usermode) 2 0% 100% 0.00 520 dd->dd_lock dsl_dir_willuse_space+0x204 2 0% 100% 0.00 11052 db->db_mtx dbuf_read+0xa44 1 0% 100% 0.00 393 zio->io_lock 0xc0000000026bbef0 1 0% 100% 0.00 249 tm->tm_lock trim_map_write_done+0x188 1 0% 100% 0.00 41184 tm->tm_lock trim_map_write_start+0x3f8 1 0% 100% 0.00 402 ms->ms_lock metaslab_free_concrete+0x2e8 1 0% 100% 0.00 2558 dp->dp_lock dsl_pool_dirty_space+0x17c 1 0% 100% 0.00 1162 dn->dn_mtx (usermode) 1 0% 100% 0.00 283 allproc fork1+0x26bc ------------------------------------------------------------------------------- SX shared spin: 293 events in 2.696 seconds (109 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 237 81% 81% 0.00 23662 vm map (user) _sx_slock_int+0xa0 52 18% 99% 0.00 30529 vm map (user) (usermode) 3 1% 100% 0.00 652 zf->zf_rwlock (usermode) 1 0% 100% 0.00 598 zf->zf_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- root@fledgling:~ #