root@fledgling:~ # root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 19584 events in 1.203 seconds (16281 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 6858 35% 35% 0.00 17069 page pv moea64_enter+0x86c 5620 29% 64% 0.00 14940 page pv 0xc000000003142640 5243 27% 90% 0.00 16470 page pv (usermode) 543 3% 93% 0.00 8011 vm active pagequeue 0xc00000000310e854 421 2% 95% 0.00 10739 vm page vm_page_change_lock+0x160 143 1% 96% 0.00 13790 vm page vm_fault_hold+0x2724 138 1% 97% 0.00 1959 vnode interlock vget+0x2f8 105 1% 97% 0.00 11954 vm active pagequeue (usermode) 98 1% 98% 0.00 12420848 process lock (usermode) 85 0% 98% 0.00 8038 vnode interlock (usermode) 52 0% 99% 0.00 10626 vm page (usermode) 43 0% 99% 0.00 10596 vnode interlock 0xc000000002e224a0 36 0% 99% 0.00 5481 page pv moea64_enter+0x910 22 0% 99% 0.00 7850 struct mount mtx vfs_unbusy+0x15c 18 0% 99% 0.00 11420 vnode interlock 0xc000000002e22484 17 0% 99% 0.00 10947 vnode interlock 0xc0000000030ce5b8 16 0% 99% 0.00 3773162 process lock ast+0x7e0 15 0% 99% 0.00 6326 select mtxpool selrecord+0x1dc 13 0% 99% 0.00 241 struct mount mtx vfs_busy+0x434 10 0% 100% 0.00 5160 struct mount mtx (usermode) 8 0% 100% 0.00 6068 vm active pagequeue vm_page_dequeue+0x1b4 8 0% 100% 0.00 4317407 process lock kern_thr_exit+0x12c 6 0% 100% 0.00 4838 vm page 0xc0000000030ec28c 5 0% 100% 0.00 4049 struct mount vlist mtx _vdrop+0x6f4 5 0% 100% 0.00 5002 select mtxpool 0xc000000002d93038 4 0% 100% 0.00 3816 struct mount vlist mtx _vhold+0x474 4 0% 100% 0.00 1222957 process lock kern_thr_exit+0x24c 4 0% 100% 0.00 588 UPVO entry uma_zalloc_arg+0x98c 3 0% 100% 0.00 290 vm page vm_page_rename+0x224 3 0% 100% 0.00 4461 vm page exec_unmap_first_page+0x188 3 0% 100% 0.00 4573 ttymtx (usermode) 3 0% 100% 0.00 295 pipe mutex 0xc000000002d95914 3 0% 100% 0.00 5643 pipe mutex 0xc000000002d9456c 3 0% 100% 0.00 2120 pipe mutex 0xc000000002d95074 3 0% 100% 0.00 3755 pipe mutex 0xc000000002d93a60 2 0% 100% 0.00 19961 vm page free queue 0xc000000003111274 2 0% 100% 0.00 5772 ttymtx ttyoutq_read_uio+0x34c 2 0% 100% 0.00 5718 sellck 0xc000000002d92870 2 0% 100% 0.00 15775 pipe mutex (usermode) 2 0% 100% 0.00 1065 UPVO entry uma_zfree_arg+0x5c8 2 0% 100% 0.00 944 UPVO entry (usermode) 1 0% 100% 0.00 184 vnode interlock _vhold+0x4a4 1 0% 100% 0.00 281 vm page vm_imgact_unmap_page+0x184 1 0% 100% 0.00 3203 vm page vm_imgact_map_page+0x43c 1 0% 100% 0.00 851 vm page exec_map_first_page+0x7b8 1 0% 100% 0.00 243 ttymtx 0xc000000002db84b4 1 0% 100% 0.00 185 vm active pagequeue 0xc000000003114894 1 0% 100% 0.00 19122 struct mount mtx vfs_rel+0x128 1 0% 100% 0.00 219 struct mount vlist mtx (usermode) 1 0% 100% 0.00 11405 bge0 0xc00000000282746c 1 0% 100% 0.00 11997 select mtxpool (usermode) 1 0% 100% 0.00 164 select mtxpool 0xc000000002d92a68 ------------------------------------------------------------------------------- Adaptive mutex block: 1 events in 1.203 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 5997662 process lock (usermode) ------------------------------------------------------------------------------- Spin lock spin: 7904 events in 1.203 seconds (6571 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 4666 59% 59% 0.00 349845 turnstile chain (usermode) 2898 37% 96% 0.00 292011 turnstile chain turnstile_trywait+0x208 106 1% 97% 0.00 66282 turnstile chain turnstile_chain_lock+0xdc 93 1% 98% 0.00 3645 sleepq chain sleepq_lock+0xe4 46 1% 99% 0.00 337 turnstile lock turnstile_trywait+0x150 43 1% 99% 0.00 416 turnstile lock (usermode) 34 0% 100% 0.00 7382 sleepq chain (usermode) 2 0% 100% 0.00 1359 sched lock 59 0xc000000002d3cce4 2 0% 100% 0.00 1262 sched lock 37 (usermode) 1 0% 100% 0.00 189 turnstile lock turnstile_lookup+0xd4 1 0% 100% 0.00 1613 sched lock 52 sched_add+0x48c 1 0% 100% 0.00 403 sched lock 51 (usermode) 1 0% 100% 0.00 2103 sched lock 57 0xc000000002d3cccc 1 0% 100% 0.00 387 sched lock 62 sched_add+0x48c 1 0% 100% 0.00 990 sched lock 30 sched_add+0x48c 1 0% 100% 0.00 13157 sched lock 3 0xc000000002d3cce4 1 0% 100% 0.00 1157 sched lock 22 0xc000000002d3cccc 1 0% 100% 0.00 443 sched lock 23 sched_add+0x48c 1 0% 100% 0.00 838 sched lock 3 sched_add+0x48c 1 0% 100% 0.00 216 callout 0xc000000002d1b164 1 0% 100% 0.00 1794 sched lock 0 0xc000000002d3cccc 1 0% 100% 0.00 2900 sched lock 15 sched_switch+0x81c 1 0% 100% 0.00 148 sched lock 20 0xc000000002d3cce4 ------------------------------------------------------------------------------- Thread lock spin: 60 events in 1.203 seconds (50 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 26 43% 43% 0.00 4003 sleepq chain _thread_lock+0xf4 10 17% 60% 0.00 4038 sleepq chain (usermode) 9 15% 75% 0.00 6940 turnstile lock _thread_lock+0xf4 2 3% 78% 0.00 6742 sched lock 55 _thread_lock+0xf4 2 3% 82% 0.00 6147 sched lock 13 _thread_lock+0xf4 2 3% 85% 0.00 4988 sched lock 1 _thread_lock+0xf4 1 2% 87% 0.00 1924 sched lock 71 (usermode) 1 2% 88% 0.00 421 sched lock 65 (usermode) 1 2% 90% 0.00 6130 sched lock 62 (usermode) 1 2% 92% 0.00 10571 sched lock 52 (usermode) 1 2% 93% 0.00 12145 sched lock 46 _thread_lock+0xf4 1 2% 95% 0.00 576 sched lock 46 (usermode) 1 2% 97% 0.00 22498 sched lock 20 _thread_lock+0xf4 1 2% 98% 0.00 6263 sched lock 23 (usermode) 1 2% 100% 0.00 6102 sched lock 0 _thread_lock+0xf4 ------------------------------------------------------------------------------- R/W writer blocked by writer: 211 events in 1.203 seconds (175 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 45 21% 21% 0.00 61636031 vm object vm_fault_hold+0x1b28 44 21% 42% 0.00 100960821 vm object vm_map_insert+0x598 30 14% 56% 0.00 96953132 vm object vm_object_reference+0x134 25 12% 68% 0.00 89289766 vm object vm_object_deallocate+0x800 18 9% 77% 0.00 66497549 vm object (usermode) 12 6% 82% 0.00 77905630 vm object 0xc0000000030f8558 11 5% 88% 0.00 150563833 vm object vm_object_shadow+0x390 11 5% 93% 0.00 76747244 vm object vm_object_shadow+0x378 8 4% 97% 0.00 87621650 vm object vm_imgact_map_page+0x3f0 5 2% 99% 0.00 189347759 vm object exec_map_first_page+0x780 1 0% 100% 0.00 45128850 vm object vnode_create_vobject+0x18c 1 0% 100% 0.00 5480863 vm object vm_object_deallocate+0x814 ------------------------------------------------------------------------------- R/W writer blocked by readers: 466 events in 1.203 seconds (387 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 206 44% 44% 0.00 76012096 vm object vm_fault_hold+0x1b28 53 11% 56% 0.00 88277476 vm object vm_object_deallocate+0x800 40 9% 64% 0.00 79145894 vm object (usermode) 37 8% 72% 0.00 74172409 vm object vnode_create_vobject+0x18c 33 7% 79% 0.00 51685015 vm object vm_object_deallocate+0x814 32 7% 86% 0.00 77721590 vm object 0xc0000000030f8558 22 5% 91% 0.00 95525734 vm object vm_object_reference+0x134 17 4% 94% 0.00 52461867 vm object vm_imgact_map_page+0x3f0 11 2% 97% 0.00 108224746 vm object vm_object_shadow+0x378 9 2% 99% 0.00 36045477 vm object vm_map_insert+0x598 5 1% 100% 0.00 120677498 vm object vm_object_shadow+0x390 1 0% 100% 0.00 932328 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader blocked by writer: 90 events in 1.203 seconds (75 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 60 67% 67% 0.00 34824261 vm object __rw_rlock_int+0x9c 30 33% 100% 0.00 23807183 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 13 events in 1.203 seconds (11 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 10 77% 77% 0.00 56894431 vm object __rw_rlock_int+0x9c 3 23% 100% 0.00 7219389 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 423 events in 1.203 seconds (352 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 79 19% 19% 0.00 2749790 vm object vm_fault_hold+0x1b28 75 18% 36% 0.00 4160048 vm object vm_map_insert+0x598 58 14% 50% 0.00 1642241 vm object (usermode) 55 13% 63% 0.00 3551986 vm object vm_object_reference+0x134 43 10% 73% 0.00 4474229 vm object vm_object_deallocate+0x800 29 7% 80% 0.00 2743270 vm object vm_object_shadow+0x378 17 4% 84% 0.00 5956976 vm object vm_object_shadow+0x390 17 4% 88% 0.00 3173839 vm object vm_imgact_map_page+0x3f0 15 4% 92% 0.00 5823608 vm object 0xc0000000030f8558 11 3% 94% 0.00 26638 vm object 0xc00000000311f65c 7 2% 96% 0.00 9847840 vm object exec_map_first_page+0x780 7 2% 98% 0.00 16926 Per-Filesystem Softdep Lock (usermode) 3 1% 98% 0.00 8580 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 1 0% 99% 0.00 3107257 vm object vnode_create_vobject+0x18c 1 0% 99% 0.00 1170739 vm object vm_object_deallocate+0x814 1 0% 99% 0.00 1227 tidhash tidhash_remove+0x13c 1 0% 99% 0.00 10245 Per-Filesystem Softdep Lock 0xc0000000030adb58 1 0% 100% 0.00 1050 Per-Filesystem Softdep Lock softdep_slowdown+0x374 1 0% 100% 0.00 2031 Per-Filesystem Softdep Lock 0xc00000000309a308 1 0% 100% 0.00 1591 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 ------------------------------------------------------------------------------- R/W writer spin on readers: 490 events in 1.203 seconds (407 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 212 43% 43% 0.00 4541578 vm object vm_fault_hold+0x1b28 56 11% 55% 0.00 5827918 vm object vm_object_deallocate+0x800 43 9% 63% 0.00 3445263 vm object vnode_create_vobject+0x18c 42 9% 72% 0.00 4942978 vm object (usermode) 33 7% 79% 0.00 4120575 vm object vm_object_deallocate+0x814 33 7% 86% 0.00 5445732 vm object 0xc0000000030f8558 23 5% 90% 0.00 5316167 vm object vm_object_reference+0x134 18 4% 94% 0.00 2923979 vm object vm_imgact_map_page+0x3f0 11 2% 96% 0.00 7542363 vm object vm_object_shadow+0x378 11 2% 98% 0.00 2371517 vm object vm_map_insert+0x598 5 1% 99% 0.00 9774809 vm object vm_object_shadow+0x390 2 0% 100% 0.00 501892 vm object exec_map_first_page+0x780 1 0% 100% 0.00 62679 vm object 0xc00000000311f65c ------------------------------------------------------------------------------- R/W reader spin on writer: 134 events in 1.203 seconds (111 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 94 70% 70% 0.00 987394 vm object __rw_rlock_int+0x9c 40 30% 100% 0.00 981806 vm object (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 9 events in 1.203 seconds (7 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 7 78% 78% 0.00 1996522 vm object __rw_rlock_int+0x9c 2 22% 100% 0.00 290898 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive spin: 135 events in 1.203 seconds (112 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 40 30% 30% 0.00 6216 rrl->rr_lock 0xc00000000260abbc 25 19% 48% 0.00 2927 rrl->rr_lock rrw_exit+0x310 18 13% 61% 0.00 12762 proctree kern_wait6+0x3e8 16 12% 73% 0.00 1037 zp->z_acl_lock 0xc0000000026790e0 13 10% 83% 0.00 1422 zp->z_acl_lock (usermode) 7 5% 88% 0.00 9121 rrl->rr_lock (usermode) 5 4% 92% 0.00 6645 dd->dd_lock dsl_dir_tempreserve_clear+0x88 3 2% 94% 0.00 18334 proctree exit1+0x1aec 3 2% 96% 0.00 4866 proctree (usermode) 2 1% 98% 0.00 1126 dd->dd_lock 0xc0000000025e7c24 2 1% 99% 0.00 11063 dd->dd_lock (usermode) 1 1% 100% 0.00 520 proctree fork1+0x28bc ------------------------------------------------------------------------------- SX shared spin: 1 events in 1.203 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 555 zf->zf_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- ^[[Aroot@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 55898 events in 2.335 seconds (23939 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 17243 31% 31% 0.00 14472 page pv 0xc000000003142640 17233 31% 62% 0.00 13665 page pv (usermode) 15855 28% 90% 0.00 14099 page pv moea64_enter+0x86c 1536 3% 93% 0.00 7870 vm active pagequeue 0xc00000000310e854 1208 2% 95% 0.00 9185 vm page vm_page_change_lock+0x160 459 1% 96% 0.00 4269 vnode interlock vget+0x2f8 371 1% 96% 0.00 11912 vm page vm_fault_hold+0x2724 348 1% 97% 0.00 9551 vm active pagequeue (usermode) 285 1% 98% 0.00 7586 vnode interlock (usermode) 179 0% 98% 0.00 10792 vm page (usermode) 156 0% 98% 0.00 8557 vnode interlock 0xc000000002e224a0 119 0% 98% 0.00 6964 struct mount mtx vfs_unbusy+0x15c 98 0% 99% 0.00 9657 pmap moea64_enter+0x880 89 0% 99% 0.00 12460 page pv moea64_enter+0x910 71 0% 99% 0.00 583469 process lock ast+0x7e0 66 0% 99% 0.00 7561 struct mount mtx vfs_busy+0x434 49 0% 99% 0.00 8889 pmap (usermode) 47 0% 99% 0.00 13115 vnode interlock 0xc000000002e22484 46 0% 99% 0.00 2779 struct mount mtx (usermode) 34 0% 99% 0.00 10085 vnode interlock 0xc0000000030ce5b8 34 0% 99% 0.00 8621 umtxql (usermode) 33 0% 99% 0.00 206880 process lock kern_thr_exit+0x12c 30 0% 99% 0.00 3378 vm active pagequeue vm_page_dequeue+0x1b4 25 0% 99% 0.00 120609 process lock kern_thr_exit+0x24c 24 0% 100% 0.00 4117 struct mount vlist mtx _vdrop+0x6f4 24 0% 100% 0.00 1869 struct mount vlist mtx _vhold+0x474 17 0% 100% 0.00 9519 umtxql 0xc000000002d245d4 17 0% 100% 0.00 7769 umtxql 0xc000000002d228cc 15 0% 100% 0.00 5928 select mtxpool selrecord+0x1dc 14 0% 100% 0.00 3150 select mtxpool 0xc000000002d93038 11 0% 100% 0.00 2691 vm page 0xc0000000030ec28c 11 0% 100% 0.00 6743 ttymtx (usermode) 10 0% 100% 0.00 4409 umtxql 0xc000000002d22b78 9 0% 100% 0.00 783 struct mount vlist mtx (usermode) 8 0% 100% 0.00 12700 vm page free queue 0xc000000003111274 7 0% 100% 0.00 4040 vnode interlock _vhold+0x4a4 7 0% 100% 0.00 4713 select mtxpool (usermode) 7 0% 100% 0.00 1074 UPVO entry uma_zfree_arg+0x5c8 6 0% 100% 0.00 2600 vm page vm_page_rename+0x224 6 0% 100% 0.00 5820 ttymtx ttyoutq_read_uio+0x34c 6 0% 100% 0.00 997 UPVO entry (usermode) 5 0% 100% 0.00 2648 pipe mutex 0xc000000002d95914 5 0% 100% 0.00 1492 UPVO entry uma_zalloc_arg+0x98c 4 0% 100% 0.00 19492 umtxql 0xc000000002d253f8 4 0% 100% 0.00 1980 ttymtx 0xc000000002db84b4 4 0% 100% 0.00 116693 process lock (usermode) 4 0% 100% 0.00 1169 pipe mutex 0xc000000002d95074 4 0% 100% 0.00 5822 pipe mutex 0xc000000002d9456c 4 0% 100% 0.00 4694 pipe mutex 0xc000000002d93828 4 0% 100% 0.00 4574 pipe mutex (usermode) 3 0% 100% 0.00 4001 vm page exec_map_first_page+0x7b8 3 0% 100% 0.00 6179 umtxql 0xc000000002d25428 3 0% 100% 0.00 943 ttymtx 0xc000000002db7520 3 0% 100% 0.00 5798 pipe mutex 0xc000000002d93a60 2 0% 100% 0.00 12545 vm page free queue 0xc000000003110fb4 2 0% 100% 0.00 5991 vm page 0xc0000000030ee9a0 2 0% 100% 0.00 3462 vm page vm_imgact_unmap_page+0x184 2 0% 100% 0.00 992 vm page vm_imgact_map_page+0x43c 2 0% 100% 0.00 325 ttymtx 0xc000000002db18d8 2 0% 100% 0.00 3193 struct mount mtx vfs_ref+0x114 2 0% 100% 0.00 13576 sellck 0xc000000002d92870 2 0% 100% 0.00 2993 select mtxpool 0xc000000002d92a68 2 0% 100% 0.00 3686 process_exit (usermode) 1 0% 100% 0.00 1003 vnode interlock 0xc000000002ce0a70 1 0% 100% 0.00 2892 vm page free queue (usermode) 1 0% 100% 0.00 6493 vm page exec_unmap_first_page+0x188 1 0% 100% 0.00 20726 vm active pagequeue 0xc000000003114894 1 0% 100% 0.00 6058 umtxql 0xc000000002d22e2c 1 0% 100% 0.00 11420 struct mount mtx vfs_rel+0x128 1 0% 100% 0.00 10493 struct mount mtx vn_finished_secondary_write+0x174 1 0% 100% 0.00 19292 struct mount mtx vfs_notify_upper+0x460 1 0% 100% 0.00 1195 page pv moea64_page_wired_mappings+0x1b8 1 0% 100% 0.00 22484 SLB table moea64_release_vsid+0x144 1 0% 100% 0.00 211 SLB table (usermode) 1 0% 100% 0.00 1670 ncneglh cache_lookup+0x1638 1 0% 100% 0.00 6522 nvme qpair lock 0xc000000002a101b0 1 0% 100% 0.00 12180 process lock exit1+0x1a54 1 0% 100% 0.00 223 process lock ast+0x6e0 1 0% 100% 0.00 1647 process lock 0xc000000002c8e430 1 0% 100% 0.00 24947 process_exit exit1+0x564 ------------------------------------------------------------------------------- Adaptive mutex block: 1 events in 2.335 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 39970 vm page vm_fault_hold+0x2724 ------------------------------------------------------------------------------- Spin lock spin: 18040 events in 2.335 seconds (7726 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 9665 54% 54% 0.00 424555 turnstile chain (usermode) 7383 41% 95% 0.00 340787 turnstile chain turnstile_trywait+0x208 347 2% 96% 0.00 2681 sleepq chain sleepq_lock+0xe4 233 1% 98% 0.00 161374 turnstile chain turnstile_chain_lock+0xdc 168 1% 99% 0.00 3987 sleepq chain (usermode) 106 1% 99% 0.00 425 turnstile lock (usermode) 101 1% 100% 0.00 451 turnstile lock turnstile_trywait+0x150 6 0% 100% 0.00 7634 sched lock 0 0xc000000002d3cccc 2 0% 100% 0.00 10323 turnstile lock turnstile_lookup+0xd4 2 0% 100% 0.00 2711 sched lock 32 0xc000000002d3cce4 2 0% 100% 0.00 994 sched lock 20 0xc000000002d3cccc 1 0% 100% 0.00 3546 sched lock 9 0xc000000002d3cccc 1 0% 100% 0.00 406 sched lock 7 sched_add+0x48c 1 0% 100% 0.00 981 sched lock 66 sched_switch+0x81c 1 0% 100% 0.00 787 sched lock 6 0xc000000002d3cccc 1 0% 100% 0.00 617 sched lock 55 (usermode) 1 0% 100% 0.00 422 sched lock 57 sched_add+0x48c 1 0% 100% 0.00 5722 sched lock 50 (usermode) 1 0% 100% 0.00 554 sched lock 49 sched_add+0x48c 1 0% 100% 0.00 1111 sched lock 50 0xc000000002d3cce4 1 0% 100% 0.00 236 sched lock 52 0xc000000002d3cce4 1 0% 100% 0.00 533 sched lock 31 sched_add+0x48c 1 0% 100% 0.00 461 sched lock 23 sched_add+0x48c 1 0% 100% 0.00 1738 sched lock 34 0xc000000002d3cccc 1 0% 100% 0.00 4304 sched lock 34 sched_add+0x48c 1 0% 100% 0.00 278 sched lock 35 sched_add+0x48c 1 0% 100% 0.00 761 sched lock 37 0xc000000002d3cce4 1 0% 100% 0.00 276 sched lock 21 (usermode) 1 0% 100% 0.00 2043 sched lock 11 0xc000000002d3cce4 1 0% 100% 0.00 938 sched lock 17 sched_add+0x48c 1 0% 100% 0.00 1107 sched lock 10 (usermode) 1 0% 100% 0.00 762 sched lock 11 (usermode) 1 0% 100% 0.00 2061 sched lock 11 0xc000000002d3cccc 1 0% 100% 0.00 375 callout (usermode) 1 0% 100% 0.00 11001 sched lock 0 (usermode) 1 0% 100% 0.00 978 sched lock 0 sched_switch+0x81c ------------------------------------------------------------------------------- Thread lock spin: 317 events in 2.335 seconds (136 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 187 59% 59% 0.00 3589 sleepq chain _thread_lock+0xf4 81 26% 85% 0.00 3231 sleepq chain (usermode) 13 4% 89% 0.00 6971 turnstile lock _thread_lock+0xf4 5 2% 90% 0.00 8814 sched lock 2 _thread_lock+0xf4 3 1% 91% 0.00 4026 sched lock 56 _thread_lock+0xf4 3 1% 92% 0.00 6124 sched lock 52 _thread_lock+0xf4 2 1% 93% 0.00 5615 sched lock 3 _thread_lock+0xf4 2 1% 93% 0.00 458 sched lock 0 _thread_lock+0xf4 1 0% 94% 0.00 336 sched lock 71 (usermode) 1 0% 94% 0.00 464 sched lock 71 _thread_lock+0xf4 1 0% 94% 0.00 334 sched lock 59 _thread_lock+0xf4 1 0% 95% 0.00 1213 sched lock 68 (usermode) 1 0% 95% 0.00 13703 sched lock 7 (usermode) 1 0% 95% 0.00 11954 sched lock 57 (usermode) 1 0% 96% 0.00 656 sched lock 54 (usermode) 1 0% 96% 0.00 12722 sched lock 49 (usermode) 1 0% 96% 0.00 1893 sched lock 48 _thread_lock+0xf4 1 0% 97% 0.00 729 sched lock 5 _thread_lock+0xf4 1 0% 97% 0.00 24247 sched lock 50 (usermode) 1 0% 97% 0.00 11846 sched lock 31 (usermode) 1 0% 97% 0.00 320 sched lock 29 _thread_lock+0xf4 1 0% 98% 0.00 6290 sched lock 37 _thread_lock+0xf4 1 0% 98% 0.00 1990 sched lock 33 _thread_lock+0xf4 1 0% 98% 0.00 416 sched lock 45 _thread_lock+0xf4 1 0% 99% 0.00 73060 sched lock 47 (usermode) 1 0% 99% 0.00 279 sched lock 28 _thread_lock+0xf4 1 0% 99% 0.00 648 sched lock 20 _thread_lock+0xf4 1 0% 100% 0.00 387 sched lock 19 _thread_lock+0xf4 1 0% 100% 0.00 553 sched lock 12 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 626 events in 2.335 seconds (268 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 140 22% 22% 0.00 67025014 vm object vm_fault_hold+0x1b28 102 16% 39% 0.00 74170009 vm object vm_map_insert+0x598 99 16% 54% 0.00 63255241 vm object vm_object_deallocate+0x800 82 13% 68% 0.00 77650780 vm object vm_object_reference+0x134 44 7% 75% 0.00 51441526 vm object (usermode) 40 6% 81% 0.00 74801180 vm object vm_object_shadow+0x390 39 6% 87% 0.00 73805400 vm object vm_object_shadow+0x378 39 6% 93% 0.00 82283036 vm object 0xc0000000030f8558 26 4% 98% 0.00 84719314 vm object vm_imgact_map_page+0x3f0 14 2% 100% 0.00 120364510 vm object exec_map_first_page+0x780 1 0% 100% 0.00 76172256 vm object vm_object_deallocate+0x814 ------------------------------------------------------------------------------- R/W writer blocked by readers: 1071 events in 2.335 seconds (459 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 435 41% 41% 0.00 56841005 vm object vm_fault_hold+0x1b28 131 12% 53% 0.00 62686463 vm object vm_object_deallocate+0x800 106 10% 63% 0.00 41388601 vm object (usermode) 93 9% 71% 0.00 75077261 vm object vnode_create_vobject+0x18c 90 8% 80% 0.00 40599679 vm object 0xc0000000030f8558 86 8% 88% 0.00 56264811 vm object vm_object_deallocate+0x814 38 4% 91% 0.00 76030521 vm object vm_object_reference+0x134 31 3% 94% 0.00 54067445 vm object vm_imgact_map_page+0x3f0 27 3% 97% 0.00 53373310 vm object vm_map_insert+0x598 20 2% 99% 0.00 81282845 vm object vm_object_shadow+0x378 9 1% 100% 0.00 38599245 vm object vm_object_shadow+0x390 4 0% 100% 0.00 25731437 vm object exec_map_first_page+0x780 1 0% 100% 0.00 61960 vm object 0xc00000000311f65c ------------------------------------------------------------------------------- R/W reader blocked by writer: 199 events in 2.335 seconds (85 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 127 64% 64% 0.00 37548006 vm object __rw_rlock_int+0x9c 72 36% 100% 0.00 29727254 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 37 events in 2.335 seconds (16 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 25 68% 68% 0.00 32726682 vm object __rw_rlock_int+0x9c 12 32% 100% 0.00 36517025 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 1163 events in 2.335 seconds (498 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 216 19% 19% 0.00 4137980 vm object vm_fault_hold+0x1b28 157 13% 32% 0.00 3811091 vm object vm_object_deallocate+0x800 156 13% 45% 0.00 5411930 vm object vm_map_insert+0x598 137 12% 57% 0.00 4722766 vm object vm_object_reference+0x134 111 10% 67% 0.00 4039719 vm object (usermode) 101 9% 75% 0.00 12051 vm object vm_fault_hold+0x2700 66 6% 81% 0.00 4884516 vm object vm_object_shadow+0x378 63 5% 87% 0.00 4869745 vm object vm_object_shadow+0x390 58 5% 92% 0.00 4616600 vm object 0xc0000000030f8558 40 3% 95% 0.00 5472567 vm object vm_imgact_map_page+0x3f0 18 2% 97% 0.00 8026783 vm object exec_map_first_page+0x780 15 1% 98% 0.00 20628 vm object 0xc00000000311f65c 7 1% 98% 0.00 8426 Per-Filesystem Softdep Lock (usermode) 2 0% 99% 0.00 5500915 vm object vm_object_deallocate+0x814 2 0% 99% 0.00 1674 Per-Filesystem Softdep Lock softdep_slowdown+0x374 2 0% 99% 0.00 18252 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 2 0% 99% 0.00 25432 Per-Filesystem Softdep Lock 0xc00000000309c6f4 2 0% 99% 0.00 9024 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 2 0% 99% 0.00 91028 Per-Filesystem Softdep Lock 0xc000000003095904 1 0% 100% 0.00 462 vm object vm_object_collapse+0x710 1 0% 100% 0.00 5886 Per-Filesystem Softdep Lock 0xc0000000030adb58 1 0% 100% 0.00 11371 Per-Filesystem Softdep Lock 0xc0000000030a7868 1 0% 100% 0.00 5882 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c 1 0% 100% 0.00 11325 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c 1 0% 100% 0.00 13246 Per-Filesystem Softdep Lock 0xc00000000309f2b4 ------------------------------------------------------------------------------- R/W writer spin on readers: 1167 events in 2.335 seconds (500 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 444 38% 38% 0.00 4744711 vm object vm_fault_hold+0x1b28 137 12% 50% 0.00 5715800 vm object vm_object_deallocate+0x800 114 10% 60% 0.00 4527970 vm object (usermode) 98 8% 68% 0.00 6155942 vm object vnode_create_vobject+0x18c 94 8% 76% 0.00 4614880 vm object 0xc0000000030f8558 86 7% 83% 0.00 5869826 vm object vm_object_deallocate+0x814 39 3% 87% 0.00 6747396 vm object vm_object_reference+0x134 33 3% 90% 0.00 22034 vm object vm_fault_hold+0x2700 32 3% 92% 0.00 5002336 vm object vm_imgact_map_page+0x3f0 31 3% 95% 0.00 3661851 vm object vm_map_insert+0x598 20 2% 97% 0.00 9875753 vm object vm_object_shadow+0x378 19 2% 98% 0.00 20859 vm object vm_fault_hold+0x1298 9 1% 99% 0.00 6089649 vm object vm_object_shadow+0x390 5 0% 99% 0.00 98870 vm object 0xc00000000311f65c 5 0% 100% 0.00 1915044 vm object exec_map_first_page+0x780 1 0% 100% 0.00 428 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W reader spin on writer: 383 events in 2.335 seconds (164 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 258 67% 67% 0.00 852320 vm object __rw_rlock_int+0x9c 125 33% 100% 0.00 871573 vm object (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 35 events in 2.335 seconds (15 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 24 69% 69% 0.00 1210492 vm object __rw_rlock_int+0x9c 11 31% 100% 0.00 1533777 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 1 events in 2.335 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 36128 vm map (user) (usermode) ------------------------------------------------------------------------------- SX shared block: 1 events in 2.335 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 30124 vm map (user) _sx_slock_int+0xa0 ------------------------------------------------------------------------------- SX exclusive spin: 790 events in 2.335 seconds (338 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 227 29% 29% 0.00 28241 vm map (user) (usermode) 160 20% 49% 0.00 5355 rrl->rr_lock 0xc00000000260abbc 105 13% 62% 0.00 5965 rrl->rr_lock rrw_exit+0x310 74 9% 72% 0.00 12039 zp->z_acl_lock 0xc0000000026790e0 44 6% 77% 0.00 4593 zp->z_acl_lock (usermode) 44 6% 83% 0.00 25412 vm map (user) _vm_map_lock+0x138 36 5% 87% 0.00 11387 proctree kern_wait6+0x3e8 27 3% 91% 0.00 12114 proctree (usermode) 26 3% 94% 0.00 5782 rrl->rr_lock (usermode) 14 2% 96% 0.00 13418 dd->dd_lock dsl_dir_tempreserve_clear+0x88 14 2% 98% 0.00 9210 dd->dd_lock (usermode) 7 1% 98% 0.00 412 dd->dd_lock 0xc0000000025e7c24 2 0% 99% 0.00 39787 vm map (user) _vm_map_lock_upgrade+0x180 2 0% 99% 0.00 6818 db->db_mtx dbuf_rele+0xbc 2 0% 99% 0.00 12256 db->db_mtx (usermode) 1 0% 99% 0.00 13990 proctree exit1+0x1aec 1 0% 99% 0.00 113886 proctree fork1+0x28bc 1 0% 100% 0.00 309 hdl->sa_lock sa_lookup_uio+0x198 1 0% 100% 0.00 281 hdl->sa_lock (usermode) 1 0% 100% 0.00 267 dp->dp_lock (usermode) 1 0% 100% 0.00 1012 h->hash_mutexes[i] dbuf_find+0x114 ------------------------------------------------------------------------------- SX shared spin: 295 events in 2.335 seconds (126 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 225 76% 76% 0.00 26013 vm map (user) _sx_slock_int+0xa0 65 22% 98% 0.00 35930 vm map (user) (usermode) 4 1% 100% 0.00 371 zf->zf_rwlock _sx_slock_int+0xa0 1 0% 100% 0.00 391 zf->zf_rwlock (usermode) ------------------------------------------------------------------------------- ^[[Aroot@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 39047 events in 1.626 seconds (24014 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 14551 37% 37% 0.00 13393 page pv 0xc000000003142640 10102 26% 63% 0.00 14413 page pv moea64_enter+0x86c 9240 24% 87% 0.00 13915 page pv (usermode) 1209 3% 90% 0.00 9086 vm active pagequeue 0xc00000000310e854 1039 3% 93% 0.00 10707 vm page vm_page_change_lock+0x160 276 1% 93% 0.00 4017 vnode interlock vget+0x2f8 264 1% 94% 0.00 12726 vm page vm_fault_hold+0x2724 219 1% 95% 0.00 10487 vm active pagequeue (usermode) 180 0% 95% 0.00 20032 dp_sync_taskq 0xc000000002cc30c0 177 0% 95% 0.00 64471 dp_sync_taskq 0xc000000002d7c86c 176 0% 96% 0.00 15571 vm page (usermode) 160 0% 96% 0.00 17239 dp_sync_taskq (usermode) 145 0% 97% 0.00 69242 zio_write_intr (usermode) 143 0% 97% 0.00 6383 vnode interlock (usermode) 135 0% 97% 0.00 18740 dp_sync_taskq taskqueue_enqueue+0x158 102 0% 98% 0.00 62918 zio_write_intr 0xc000000002d7c86c 91 0% 98% 0.00 8871 vnode interlock 0xc000000002e224a0 79 0% 98% 0.00 55575 kernelpmap moea64_kextract+0x174 63 0% 98% 0.00 48521 zio_write_intr 0xc000000002cc30c0 62 0% 98% 0.00 2010984 zio_write_issue (usermode) 53 0% 99% 0.00 7242 struct mount mtx vfs_unbusy+0x15c 47 0% 99% 0.00 2790 page pv moea64_enter+0x910 44 0% 99% 0.00 12967 struct mount mtx (usermode) 43 0% 99% 0.00 1533031 zio_write_issue 0xc000000002d7c86c 41 0% 99% 0.00 118664 zio_write_intr taskqueue_enqueue+0x158 41 0% 99% 0.00 8285 vm active pagequeue vm_page_dequeue+0x1b4 38 0% 99% 0.00 615311 zio_write_issue 0xc000000002cc30c0 35 0% 99% 0.00 16865 select mtxpool selrecord+0x1dc 34 0% 99% 0.00 2264 struct mount mtx vfs_busy+0x434 29 0% 99% 0.00 11351 vnode interlock 0xc000000002e22484 23 0% 99% 0.00 10191 vnode interlock 0xc0000000030ce5b8 20 0% 100% 0.00 45661 kernelpmap moea64_enter+0x880 18 0% 100% 0.00 5160 select mtxpool 0xc000000002d93038 16 0% 100% 0.00 945 UPVO entry (usermode) 15 0% 100% 0.00 897444 zio_write_issue taskqueue_enqueue+0x158 10 0% 100% 0.00 4202 vm page 0xc0000000030ec28c 10 0% 100% 0.00 3470 struct mount vlist mtx _vdrop+0x6f4 10 0% 100% 0.00 474 struct mount vlist mtx _vhold+0x474 8 0% 100% 0.00 7410 ttymtx ttyoutq_read_uio+0x34c 6 0% 100% 0.00 13259 vm page free queue 0xc000000003111274 6 0% 100% 0.00 86351 kernelpmap (usermode) 5 0% 100% 0.00 4553 ttymtx (usermode) 5 0% 100% 0.00 1073 struct mount vlist mtx (usermode) 5 0% 100% 0.00 1830 struct mount mtx vfs_ref+0x114 4 0% 100% 0.00 8053 process_exit (usermode) 3 0% 100% 0.00 601 vm page exec_map_first_page+0x7b8 3 0% 100% 0.00 2380 ttymtx 0xc000000002db7520 3 0% 100% 0.00 3977 struct mount mtx vn_finished_write+0x174 3 0% 100% 0.00 10100 struct mount mtx vfs_rel+0x128 3 0% 100% 0.00 1872 pipe mutex (usermode) 3 0% 100% 0.00 1263 UPVO entry uma_zalloc_arg+0x98c 3 0% 100% 0.00 10608 CAM device lock (usermode) 2 0% 100% 0.00 5452 zio_free_issue_3 (usermode) 2 0% 100% 0.00 5478 zio_free_issue_2 taskqueue_enqueue+0x158 2 0% 100% 0.00 2684 zio_free_issue_2 0xc000000002cc30c0 2 0% 100% 0.00 286 zio_free_issue_1 taskqueue_enqueue+0x158 2 0% 100% 0.00 17416 vm page free queue (usermode) 2 0% 100% 0.00 30246 vm page vm_object_madvise+0x54c 2 0% 100% 0.00 2583 vm page vm_imgact_unmap_page+0x184 2 0% 100% 0.00 1042 ttymtx 0xc000000002db84b4 2 0% 100% 0.00 5428 select mtxpool (usermode) 2 0% 100% 0.00 3219 reseed mutex 0xc000000002a42484 2 0% 100% 0.00 636 pipe mutex 0xc000000002d95914 2 0% 100% 0.00 67348 nvme qpair lock 0xc000000002a101b0 2 0% 100% 0.00 1873 UPVO entry uma_zfree_arg+0x5c8 2 0% 100% 0.00 2545 CAM device lock 0xc000000002524ac0 1 0% 100% 0.00 388362 vnode interlock _vhold+0x4a4 1 0% 100% 0.00 385 vm page 0xc000000002df9ce0 1 0% 100% 0.00 1076 vm page vm_object_unwire+0x2d0 1 0% 100% 0.00 6470 vm page free queue 0xc000000003110fb4 1 0% 100% 0.00 203 struct mount mtx vn_finished_secondary_write+0x174 1 0% 100% 0.00 133 ttymtx 0xc000000002db18d8 1 0% 100% 0.00 1013 ncvn (usermode) 1 0% 100% 0.00 3043 metaslab_group_taskq taskqueue_enqueue+0x158 1 0% 100% 0.00 10236 metaslab_group_taskq (usermode) 1 0% 100% 0.00 5860 kernelpmap moea64_remove+0x200 1 0% 100% 0.00 11243 dp_sync_taskq taskqueue_quiesce+0x2e8 1 0% 100% 0.00 418 CAM device lock 0xc000000002524148 1 0% 100% 0.00 1047 bge0 (usermode) 1 0% 100% 0.00 176 pipe mutex 0xc000000002d93a60 1 0% 100% 0.00 953 pipe mutex 0xc000000002d93828 1 0% 100% 0.00 981 page pv moea64_page_wired_mappings+0x1b8 1 0% 100% 0.00 1992 process lock 0xc000000002c8e430 1 0% 100% 0.00 12067 process lock (usermode) 1 0% 100% 0.00 1684 sellck (usermode) ------------------------------------------------------------------------------- Adaptive mutex block: 3 events in 1.626 seconds (2 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 33% 33% 0.00 74650 kernelpmap moea64_kextract+0x174 1 33% 67% 0.00 42501 dp_sync_taskq 0xc000000002cc30c0 1 33% 100% 0.00 31776 CAM device lock 0xc000000002524ac0 ------------------------------------------------------------------------------- Spin lock spin: 7648 events in 1.626 seconds (4703 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 3706 48% 48% 0.00 322606 turnstile chain (usermode) 3147 41% 90% 0.00 284653 turnstile chain turnstile_trywait+0x208 376 5% 95% 0.00 4884 sleepq chain sleepq_lock+0xe4 141 2% 96% 0.00 7919 sleepq chain (usermode) 111 1% 98% 0.00 58579 turnstile chain turnstile_chain_lock+0xdc 51 1% 98% 0.00 473 turnstile lock turnstile_trywait+0x150 38 0% 99% 0.00 353 turnstile lock (usermode) 6 0% 99% 0.00 4309 sched lock 35 0xc000000002d3cce4 4 0% 99% 0.00 4800 sched lock 35 0xc000000002d3cccc 4 0% 99% 0.00 5741 sched lock 35 (usermode) 3 0% 99% 0.00 1303 sched lock 70 0xc000000002d3cce4 3 0% 99% 0.00 4461 sched lock 55 0xc000000002d3cce4 3 0% 99% 0.00 2266 sched lock 30 (usermode) 2 0% 99% 0.00 3605 sched lock 68 (usermode) 2 0% 99% 0.00 19714 sched lock 59 0xc000000002d3cce4 2 0% 99% 0.00 1577 sched lock 59 (usermode) 2 0% 99% 0.00 12091 sched lock 5 0xc000000002d3cccc 2 0% 99% 0.00 558 sched lock 38 (usermode) 2 0% 99% 0.00 24885 sched lock 30 0xc000000002d3cce4 2 0% 99% 0.00 25993 sched lock 30 0xc000000002d3cccc 2 0% 99% 0.00 1026 sched lock 27 0xc000000002d3cce4 2 0% 100% 0.00 11073 sched lock 20 sched_switch+0x81c 2 0% 100% 0.00 1117 sched lock 0 0xc000000002d3cccc 1 0% 100% 0.00 10352 turnstile lock turnstile_lookup+0xd4 1 0% 100% 0.00 2000 sched lock 71 0xc000000002d3cce4 1 0% 100% 0.00 13790 sched lock 68 sched_switch+0xca0 1 0% 100% 0.00 1488 sched lock 68 0xc000000002d3cccc 1 0% 100% 0.00 11962 sched lock 60 0xc000000002d3cccc 1 0% 100% 0.00 2589 sched lock 60 sched_switch+0x81c 1 0% 100% 0.00 2639 sched lock 56 0xc000000002d3cce4 1 0% 100% 0.00 370 sched lock 53 sched_switch+0x81c 1 0% 100% 0.00 1574 sched lock 52 (usermode) 1 0% 100% 0.00 1754 sched lock 55 sched_add+0x48c 1 0% 100% 0.00 943 sched lock 56 (usermode) 1 0% 100% 0.00 7167 sched lock 43 (usermode) 1 0% 100% 0.00 5576 sched lock 40 0xc000000002d3cce4 1 0% 100% 0.00 1082 sched lock 39 sched_add+0x48c 1 0% 100% 0.00 5898 sched lock 40 sched_add+0x48c 1 0% 100% 0.00 3150 sched lock 45 sched_switch+0x81c 1 0% 100% 0.00 390 sched lock 44 sched_add+0x48c 1 0% 100% 0.00 1045 sched lock 45 0xc000000002d3cccc 1 0% 100% 0.00 209 sched lock 46 0xc000000002d3cce4 1 0% 100% 0.00 5872 sched lock 37 sched_add+0x48c 1 0% 100% 0.00 13045 sched lock 36 0xc000000002d3cce4 1 0% 100% 0.00 2722 sched lock 36 sched_add+0x48c 1 0% 100% 0.00 3486 sched lock 31 0xc000000002d3cccc 1 0% 100% 0.00 1004 sched lock 33 sched_add+0x48c 1 0% 100% 0.00 3451 sched lock 27 0xc000000002d3cccc 1 0% 100% 0.00 1718 sched lock 23 0xc000000002d3cce4 1 0% 100% 0.00 18633 sched lock 21 (usermode) 1 0% 100% 0.00 3408 sched lock 22 (usermode) 1 0% 100% 0.00 1165 sched lock 29 0xc000000002d3cce4 1 0% 100% 0.00 207 sched lock 3 sched_add+0x48c 1 0% 100% 0.00 653 callout (usermode) 1 0% 100% 0.00 11440 sched lock 10 0xc000000002d3cccc 1 0% 100% 0.00 2794 sched lock 10 sched_switch+0x81c 1 0% 100% 0.00 6261 sched lock 17 (usermode) 1 0% 100% 0.00 447 sched lock 2 sched_add+0x48c ------------------------------------------------------------------------------- Thread lock spin: 194 events in 1.626 seconds (119 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 95 49% 49% 0.00 3926 sleepq chain _thread_lock+0xf4 34 18% 66% 0.00 3585 sleepq chain (usermode) 11 6% 72% 0.00 4712 turnstile lock _thread_lock+0xf4 4 2% 74% 0.00 8155 sched lock 22 _thread_lock+0xf4 2 1% 75% 0.00 6328 sched lock 7 _thread_lock+0xf4 2 1% 76% 0.00 10466 sched lock 62 _thread_lock+0xf4 2 1% 77% 0.00 4610 sched lock 60 _thread_lock+0xf4 2 1% 78% 0.00 11307 sched lock 41 _thread_lock+0xf4 2 1% 79% 0.00 6819 sched lock 40 _thread_lock+0xf4 2 1% 80% 0.00 742 sched lock 4 _thread_lock+0xf4 2 1% 81% 0.00 1776 sched lock 38 (usermode) 2 1% 82% 0.00 8894 sched lock 35 _thread_lock+0xf4 2 1% 84% 0.00 3558 sched lock 20 _thread_lock+0xf4 2 1% 85% 0.00 10003 sched lock 18 _thread_lock+0xf4 2 1% 86% 0.00 1905 sched lock 15 _thread_lock+0xf4 2 1% 87% 0.00 5787 sched lock 12 _thread_lock+0xf4 2 1% 88% 0.00 602 sched lock 0 _thread_lock+0xf4 1 1% 88% 0.00 5577 sched lock 70 _thread_lock+0xf4 1 1% 89% 0.00 700 sched lock 70 (usermode) 1 1% 89% 0.00 509 sched lock 7 (usermode) 1 1% 90% 0.00 570 sched lock 64 (usermode) 1 1% 90% 0.00 10679 sched lock 59 (usermode) 1 1% 91% 0.00 10706 sched lock 59 _thread_lock+0xf4 1 1% 91% 0.00 5619 sched lock 55 (usermode) 1 1% 92% 0.00 3022 sched lock 51 (usermode) 1 1% 92% 0.00 5603 sched lock 55 _thread_lock+0xf4 1 1% 93% 0.00 5390 sched lock 56 (usermode) 1 1% 93% 0.00 1098 sched lock 5 _thread_lock+0xf4 1 1% 94% 0.00 12579 sched lock 46 _thread_lock+0xf4 1 1% 94% 0.00 916 sched lock 42 (usermode) 1 1% 95% 0.00 674 sched lock 37 _thread_lock+0xf4 1 1% 95% 0.00 1129 sched lock 36 _thread_lock+0xf4 1 1% 96% 0.00 761 sched lock 38 _thread_lock+0xf4 1 1% 96% 0.00 4968128 sched lock 2 _thread_lock+0xf4 1 1% 97% 0.00 836 sched lock 30 (usermode) 1 1% 97% 0.00 23441 sched lock 3 (usermode) 1 1% 98% 0.00 315 sched lock 33 _thread_lock+0xf4 1 1% 98% 0.00 5628 sched lock 35 (usermode) 1 1% 99% 0.00 469 sched lock 17 _thread_lock+0xf4 1 1% 99% 0.00 5933 sched lock 10 _thread_lock+0xf4 1 1% 100% 0.00 10425 sched lock 0 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 278 events in 1.626 seconds (171 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 54 19% 19% 0.00 48538518 vm object vm_object_deallocate+0x800 53 19% 38% 0.00 55113739 vm object vm_fault_hold+0x1b28 52 19% 57% 0.00 59499778 vm object vm_map_insert+0x598 39 14% 71% 0.00 72043747 vm object vm_object_reference+0x134 20 7% 78% 0.00 68564827 vm object vm_object_shadow+0x378 18 6% 85% 0.00 60571439 vm object vm_object_shadow+0x390 16 6% 91% 0.00 51145439 vm object 0xc0000000030f8558 12 4% 95% 0.00 62898576 vm object vm_imgact_map_page+0x3f0 7 3% 97% 0.00 55734097 vm object (usermode) 5 2% 99% 0.00 50286081 vm object exec_map_first_page+0x780 1 0% 100% 0.00 2542073 vm object vm_object_deallocate+0x814 1 0% 100% 0.00 1014475 kernel vm object (usermode) ------------------------------------------------------------------------------- R/W writer blocked by readers: 508 events in 1.626 seconds (312 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 238 47% 47% 0.00 56030064 vm object vm_fault_hold+0x1b28 58 11% 58% 0.00 61835125 vm object 0xc0000000030f8558 51 10% 68% 0.00 45359202 vm object vm_object_deallocate+0x814 49 10% 78% 0.00 66107714 vm object vm_object_deallocate+0x800 38 7% 85% 0.00 62320850 vm object vnode_create_vobject+0x18c 37 7% 93% 0.00 15665027 vm object (usermode) 13 3% 95% 0.00 67973180 vm object vm_object_reference+0x134 9 2% 97% 0.00 84453685 vm object vm_imgact_map_page+0x3f0 7 1% 98% 0.00 45116881 vm object vm_object_shadow+0x378 6 1% 100% 0.00 32071192 vm object vm_map_insert+0x598 1 0% 100% 0.00 159552619 vm object vm_object_shadow+0x390 1 0% 100% 0.00 96887685 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader blocked by writer: 82 events in 1.626 seconds (50 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 52 63% 63% 0.00 43226184 vm object __rw_rlock_int+0x9c 30 37% 100% 0.00 25634244 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 20 events in 1.626 seconds (12 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 13 65% 65% 0.00 29884968 vm object __rw_rlock_int+0x9c 7 35% 100% 0.00 15493267 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 500 events in 1.626 seconds (307 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 85 17% 17% 0.00 2654478 vm object vm_object_deallocate+0x800 81 16% 33% 0.00 3196554 vm object vm_fault_hold+0x1b28 80 16% 49% 0.00 3422867 vm object vm_map_insert+0x598 55 11% 60% 0.00 3970465 vm object vm_object_reference+0x134 42 8% 69% 0.00 946013 vm object (usermode) 32 6% 75% 0.00 3244041 vm object vm_object_shadow+0x378 27 5% 80% 0.00 3032019 vm object 0xc0000000030f8558 24 5% 85% 0.00 4379913 vm object vm_object_shadow+0x390 20 4% 89% 0.00 3148269 vm object vm_imgact_map_page+0x3f0 11 2% 91% 0.00 3007387 kernel vm object 0xc0000000030f09c0 9 2% 93% 0.00 26819 vm object 0xc00000000311f65c 7 1% 95% 0.00 3124854 vm object exec_map_first_page+0x780 6 1% 96% 0.00 112109 kernel vm object kmem_back_domain+0x340 4 1% 97% 0.00 320112 vm object vm_object_deallocate+0x814 3 1% 97% 0.00 22865 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 3 1% 98% 0.00 8996 Per-Filesystem Softdep Lock 0xc000000003095904 3 1% 98% 0.00 12999 Per-Filesystem Softdep Lock (usermode) 2 0% 99% 0.00 12992 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 2 0% 99% 0.00 359 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 1 0% 99% 0.00 92800 vm object vm_object_collapse+0x710 1 0% 100% 0.00 796 vm object bdwrite+0x2f4 1 0% 100% 0.00 12364 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c 1 0% 100% 0.00 352730 kernel vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on readers: 549 events in 1.626 seconds (338 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 247 45% 45% 0.00 3636275 vm object vm_fault_hold+0x1b28 60 11% 56% 0.00 4636292 vm object 0xc0000000030f8558 55 10% 66% 0.00 4226761 vm object vm_object_deallocate+0x800 52 9% 75% 0.00 3240328 vm object vm_object_deallocate+0x814 43 8% 83% 0.00 909942 vm object (usermode) 41 7% 91% 0.00 4143027 vm object vnode_create_vobject+0x18c 14 3% 93% 0.00 5859666 vm object vm_object_reference+0x134 12 2% 95% 0.00 4831331 vm object vm_imgact_map_page+0x3f0 9 2% 97% 0.00 3354502 vm object vm_object_shadow+0x378 9 2% 99% 0.00 816996 vm object vm_map_insert+0x598 3 1% 99% 0.00 13306 vm object vmspace_fork+0x2d8 2 0% 100% 0.00 7032858 vm object vm_object_shadow+0x390 1 0% 100% 0.00 53170 vm object 0xc00000000311f65c 1 0% 100% 0.00 7536297 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader spin on writer: 148 events in 1.626 seconds (91 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 98 66% 66% 0.00 1070385 vm object __rw_rlock_int+0x9c 49 33% 99% 0.00 905346 vm object (usermode) 1 1% 100% 0.00 405 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 18 events in 1.626 seconds (11 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 11 61% 61% 0.00 1037756 vm object __rw_rlock_int+0x9c 7 39% 100% 0.00 764455 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 1 events in 1.626 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 28430 rrl->rr_lock 0xc00000000260abbc ------------------------------------------------------------------------------- SX exclusive spin: 1238 events in 1.626 seconds (761 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 277 22% 22% 0.00 5653 rrl->rr_lock 0xc00000000260abbc 185 15% 37% 0.00 5331 rrl->rr_lock rrw_exit+0x310 135 11% 48% 0.00 29766 tm->tm_lock trim_map_free+0x19c 128 10% 59% 0.00 692216 os->os_userused_lock 0xc0000000025b7cdc 63 5% 64% 0.00 336282 vq->vq_lock (usermode) 57 5% 68% 0.00 484557 os->os_userused_lock 0xc0000000025b7e74 44 4% 72% 0.00 7450 rrl->rr_lock (usermode) 37 3% 75% 0.00 2997 zp->z_acl_lock 0xc0000000026790e0 29 2% 77% 0.00 7633 zio->io_lock (usermode) 27 2% 79% 0.00 5693 zio->io_lock zio_add_child+0x274 25 2% 81% 0.00 17251 zio->io_lock 0xc0000000026bfad8 24 2% 83% 0.00 10835 proctree kern_wait6+0x3e8 19 2% 85% 0.00 3274 zp->z_acl_lock (usermode) 16 1% 86% 0.00 39626 zio->io_lock 0xc0000000026c1ac0 16 1% 87% 0.00 235046 vq->vq_lock vdev_queue_io+0x268 15 1% 89% 0.00 92951 db->db_mtx dmu_buf_will_dirty+0x2e8 14 1% 90% 0.00 9014 proctree (usermode) 14 1% 91% 0.00 605908 os->os_userused_lock (usermode) 12 1% 92% 0.00 25282 dd->dd_lock dsl_dir_diduse_space+0x270 10 1% 93% 0.00 215166 vq->vq_lock 0xc0000000026563c8 10 1% 93% 0.00 11504 dd->dd_lock dsl_dir_tempreserve_clear+0x88 8 1% 94% 0.00 12611 dp->dp_lock (usermode) 8 1% 95% 0.00 6529 dd->dd_lock (usermode) 8 1% 95% 0.00 492496 db->db_mtx (usermode) 7 1% 96% 0.00 8697 vq->vq_lock vdev_queue_io_done+0x278 5 0% 96% 0.00 1745 vq->vq_lock vdev_queue_io_done+0x12c 5 0% 97% 0.00 192751 mg->mg_lock metaslab_alloc_dva+0x7d4 5 0% 97% 0.00 14785 dp->dp_lock dsl_pool_undirty_space+0x18c 5 0% 98% 0.00 709 dd->dd_lock 0xc0000000025e7c24 4 0% 98% 0.00 133631 mg->mg_lock (usermode) 3 0% 98% 0.00 2564 ms->ms_lock metaslab_alloc_dva+0x1138 3 0% 98% 0.00 16975 mg->mg_lock metaslab_alloc_dva+0x3a8 3 0% 99% 0.00 2984 h->hash_mutexes[i] dbuf_find+0x114 2 0% 99% 0.00 81488 tm->tm_lock (usermode) 2 0% 99% 0.00 2775 scl->scl_lock spa_config_exit+0x84 2 0% 99% 0.00 5595 ms->ms_lock (usermode) 2 0% 99% 0.00 2750 ds->ds_lock dsl_dataset_block_born+0x3d4 1 0% 99% 0.00 217 scl->scl_lock spa_config_enter+0x140 1 0% 99% 0.00 48057 proctree exit1+0x1aec 1 0% 100% 0.00 991 ms->ms_lock metaslab_free_concrete+0x2e8 1 0% 100% 0.00 307 dp->dp_lock dsl_pool_need_dirty_delay+0x1ac 1 0% 100% 0.00 1031 h->hash_mutexes[i] (usermode) 1 0% 100% 0.00 199 ds->ds_lock dsl_dataset_block_kill+0x614 1 0% 100% 0.00 1240 hdl->sa_lock (usermode) 1 0% 100% 0.00 355 dd->dd_lock dsl_dir_willuse_space+0x204 1 0% 100% 0.00 1216 db->db_mtx dbuf_dirty+0xcd0 ------------------------------------------------------------------------------- SX shared spin: 3 events in 1.626 seconds (2 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 3 100% 100% 0.00 385 zf->zf_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 81453 events in 2.312 seconds (35227 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 27415 34% 34% 0.00 14852 page pv 0xc000000003142640 22185 27% 61% 0.00 14887 page pv (usermode) 19935 24% 85% 0.00 15037 page pv moea64_enter+0x86c 3333 4% 89% 0.00 6820 vm active pagequeue 0xc00000000310e854 2642 3% 93% 0.00 10102 vm page vm_page_change_lock+0x160 517 1% 93% 0.00 10795 vm active pagequeue (usermode) 471 1% 94% 0.00 14803 vm page (usermode) 466 1% 94% 0.00 15777 pmap moea64_enter+0x880 395 0% 95% 0.00 12146 vm page vm_fault_hold+0x2724 387 0% 95% 0.00 2674 vnode interlock vget+0x2f8 244 0% 96% 0.00 5746 vnode interlock (usermode) 236 0% 96% 0.00 17006 dp_sync_taskq 0xc000000002cc30c0 203 0% 96% 0.00 14495 dp_sync_taskq (usermode) 193 0% 97% 0.00 14551 umtxql (usermode) 189 0% 97% 0.00 25240 dp_sync_taskq 0xc000000002d7c86c 187 0% 97% 0.00 38538 kernelpmap moea64_kextract+0x174 153 0% 97% 0.00 22137 zio_write_intr (usermode) 137 0% 97% 0.00 603197 process lock ast+0x7e0 114 0% 97% 0.00 14508 pmap (usermode) 114 0% 98% 0.00 7896 page pv moea64_enter+0x910 113 0% 98% 0.00 16166 dp_sync_taskq taskqueue_enqueue+0x158 110 0% 98% 0.00 7847 vnode interlock 0xc000000002e224a0 100 0% 98% 0.00 23219 umtxql 0xc000000002d245d4 89 0% 98% 0.00 8145 umtxql 0xc000000002d228cc 88 0% 98% 0.00 7275 struct mount mtx vfs_unbusy+0x15c 81 0% 98% 0.00 64163 zio_write_issue 0xc000000002cc30c0 80 0% 98% 0.00 20188 zio_write_intr 0xc000000002d7c86c 67 0% 99% 0.00 20181 zio_write_intr 0xc000000002cc30c0 66 0% 99% 0.00 63966 zio_write_issue (usermode) 62 0% 99% 0.00 69230 zio_write_issue 0xc000000002d7c86c 58 0% 99% 0.00 65684 kernelpmap (usermode) 55 0% 99% 0.00 127346 process lock kern_thr_exit+0x12c 54 0% 99% 0.00 1916 struct mount mtx vfs_busy+0x434 54 0% 99% 0.00 5889 select mtxpool selrecord+0x1dc 53 0% 99% 0.00 7574 vm active pagequeue vm_page_dequeue+0x1b4 53 0% 99% 0.00 36827 kernelpmap moea64_enter+0x880 51 0% 99% 0.00 18312 zio_write_intr taskqueue_enqueue+0x158 51 0% 99% 0.00 42869 pmap moea64_is_prefaultable+0x13c 50 0% 99% 0.00 15294 vnode interlock 0xc000000002e22484 45 0% 99% 0.00 90304 zio_write_issue taskqueue_enqueue+0x158 42 0% 99% 0.00 4130 struct mount mtx (usermode) 41 0% 99% 0.00 10084 umtxql 0xc000000002d22b78 37 0% 99% 0.00 15886 umtxql 0xc000000002d253f8 37 0% 100% 0.00 31649 kernelpmap moea64_kenter_attr+0x3d4 36 0% 100% 0.00 116664 process lock kern_thr_exit+0x24c 34 0% 100% 0.00 5847 select mtxpool 0xc000000002d93038 30 0% 100% 0.00 15469 vm page free queue 0xc000000003111274 29 0% 100% 0.00 248949 process lock (usermode) 24 0% 100% 0.00 8160 vnode interlock 0xc0000000030ce5b8 21 0% 100% 0.00 16392 umtxql 0xc000000002cc30c0 14 0% 100% 0.00 5582 umtxql 0xc000000002d25428 14 0% 100% 0.00 2021 UPVO entry (usermode) 13 0% 100% 0.00 28459 kernelpmap moea64_remove+0x200 12 0% 100% 0.00 4324 vm page vm_page_rename+0x224 12 0% 100% 0.00 3437 vm page 0xc0000000030ec28c 11 0% 100% 0.00 609 struct mount vlist mtx _vhold+0x474 10 0% 100% 0.00 5097 ttymtx ttydisc_write+0x388 10 0% 100% 0.00 918 UPVO entry uma_zfree_arg+0x5c8 9 0% 100% 0.00 8815 select mtxpool (usermode) 8 0% 100% 0.00 1184 UPVO entry uma_zalloc_arg+0x98c 7 0% 100% 0.00 10134 ttymtx ttyoutq_read_uio+0x34c 7 0% 100% 0.00 3020 struct mount vlist mtx _vdrop+0x6f4 7 0% 100% 0.00 729 struct mount vlist mtx (usermode) 6 0% 100% 0.00 11947 umtxql 0xc000000002d2546c 5 0% 100% 0.00 7551 vm page free queue (usermode) 4 0% 100% 0.00 399 ttymtx 0xc000000002db7520 4 0% 100% 0.00 3154 sellck 0xc000000002d92870 4 0% 100% 0.00 279 pipe mutex 0xc000000002d95074 4 0% 100% 0.00 72696 nvme qpair lock 0xc000000002a101b0 4 0% 100% 0.00 5158 CAM device lock (usermode) 3 0% 100% 0.00 9988 vnode interlock vinactive+0x25c 3 0% 100% 0.00 14151 vm page free queue 0xc000000003110fb4 3 0% 100% 0.00 500 ttymtx 0xc000000002db84b4 3 0% 100% 0.00 5342 ttymtx 0xc000000002db18d8 3 0% 100% 0.00 7196 select mtxpool 0xc000000002d92a68 3 0% 100% 0.00 198 pipe mutex 0xc000000002d9456c 3 0% 100% 0.00 13272 metaslab_group_taskq (usermode) 3 0% 100% 0.00 1347 CAM device lock 0xc000000002524ac0 3 0% 100% 0.00 9866 CAM device lock 0xc000000002524148 2 0% 100% 0.00 12430 zio_free_issue_1 0xc000000002d7c86c 2 0% 100% 0.00 217 zio_free_issue_1 taskqueue_enqueue+0x158 2 0% 100% 0.00 1683 umtxql 0xc000000002d24294 2 0% 100% 0.00 8190 umtxql 0xc000000002d22e2c 2 0% 100% 0.00 5081 thread_ctor 0xc000000002cc9b40 2 0% 100% 0.00 402 struct mount mtx vfs_ref+0x114 1 0% 100% 0.00 146 zio_free_issue_5 taskqueue_enqueue+0x158 1 0% 100% 0.00 207 zio_free_issue_6 0xc000000002cc30c0 1 0% 100% 0.00 178 zio_free_issue_0 0xc000000002d7c86c 1 0% 100% 0.00 955 zio_free_issue_3 0xc000000002cc30c0 1 0% 100% 0.00 10220 vm page vm_object_madvise+0x54c 1 0% 100% 0.00 1918 vm page vm_imgact_unmap_page+0x184 1 0% 100% 0.00 6187 vm page 0xc0000000030ee9a0 1 0% 100% 0.00 160 umtxql kern_umtx_wake+0x324 1 0% 100% 0.00 1215 vm page exec_unmap_first_page+0x188 1 0% 100% 0.00 496 vm page exec_map_first_page+0x7b8 1 0% 100% 0.00 114134 vm page 0xc000000002df9ce0 1 0% 100% 0.00 4078 page pv moea64_page_wired_mappings+0x1b8 1 0% 100% 0.00 496 metaslab_group_taskq taskqueue_enqueue+0x158 1 0% 100% 0.00 557 FFS 0xc0000000030c7e3c 1 0% 100% 0.00 19951 dp_sync_taskq taskqueue_quiesce+0x2e8 1 0% 100% 0.00 171 g_disk_start 0xc000000002c27478 1 0% 100% 0.00 1182 reseed mutex 0xc000000002a42484 1 0% 100% 0.00 1371 process_exit (usermode) 1 0% 100% 0.00 1211 pipe mutex 0xc000000002d95914 1 0% 100% 0.00 22328 sellck 0xc000000002cc30c0 1 0% 100% 0.00 3183 struct mount mtx vfs_rel+0x128 1 0% 100% 0.00 11358 struct mount mtx vn_finished_secondary_write+0x174 ------------------------------------------------------------------------------- Adaptive mutex block: 7 events in 2.312 seconds (3 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2 29% 29% 0.00 40936 page pv 0xc000000003142640 1 14% 43% 0.00 42115 zio_write_intr taskqueue_enqueue+0x158 1 14% 57% 0.00 68119 page pv moea64_enter+0x86c 1 14% 71% 0.00 26345 kernelpmap moea64_kextract+0x174 1 14% 86% 0.00 54048 kernelpmap (usermode) 1 14% 100% 0.00 20658 CAM device lock 0xc000000002524ac0 ------------------------------------------------------------------------------- Spin lock spin: 18997 events in 2.312 seconds (8216 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 10455 55% 55% 0.00 413830 turnstile chain (usermode) 6812 36% 91% 0.00 338628 turnstile chain turnstile_trywait+0x208 661 3% 94% 0.00 6145 sleepq chain sleepq_lock+0xe4 418 2% 97% 0.00 7097 sleepq chain (usermode) 260 1% 98% 0.00 57313 turnstile chain turnstile_chain_lock+0xdc 133 1% 99% 0.00 452 turnstile lock (usermode) 107 1% 99% 0.00 427 turnstile lock turnstile_trywait+0x150 11 0% 99% 0.00 2741 sched lock 0 0xc000000002d3cccc 9 0% 99% 0.00 447731 sched lock 25 0xc000000002d3cce4 9 0% 99% 0.00 382968 sched lock 25 (usermode) 8 0% 99% 0.00 334452 sched lock 25 0xc000000002d3cccc 6 0% 99% 0.00 1200 turnstile lock turnstile_lookup+0xd4 4 0% 99% 0.00 6731 sched lock 40 (usermode) 4 0% 99% 0.00 2919 sched lock 15 0xc000000002d3cccc 3 0% 99% 0.00 9685 sched lock 9 0xc000000002d3cccc 3 0% 100% 0.00 3430 sched lock 39 sched_add+0x48c 3 0% 100% 0.00 2254 sched lock 35 0xc000000002d3cce4 3 0% 100% 0.00 739 sched lock 26 sched_add+0x48c 3 0% 100% 0.00 2398 sched lock 2 0xc000000002d3cccc 3 0% 100% 0.00 4757 sched lock 2 sched_add+0x48c 3 0% 100% 0.00 2803 sched lock 15 (usermode) 2 0% 100% 0.00 6689 sched lock 71 0xc000000002d3cce4 2 0% 100% 0.00 935 sched lock 71 sched_add+0x48c 2 0% 100% 0.00 2289 sched lock 66 0xc000000002d3cce4 2 0% 100% 0.00 294 sched lock 62 sched_add+0x48c 2 0% 100% 0.00 20753 sched lock 55 0xc000000002d3cce4 2 0% 100% 0.00 1926 sched lock 55 0xc000000002d3cccc 2 0% 100% 0.00 12893 sched lock 41 0xc000000002d3cce4 2 0% 100% 0.00 42473 sched lock 40 0xc000000002d3cce4 2 0% 100% 0.00 1411 sched lock 40 sched_switch+0x81c 2 0% 100% 0.00 774 sched lock 4 0xc000000002d3cccc 2 0% 100% 0.00 6597 sched lock 20 (usermode) 2 0% 100% 0.00 6858 sched lock 17 0xc000000002d3cccc 2 0% 100% 0.00 9150 sched lock 11 0xc000000002d3cccc 2 0% 100% 0.00 11840 sched lock 11 (usermode) 2 0% 100% 0.00 1909 sched lock 0 sched_add+0x48c 2 0% 100% 0.00 792 callout (usermode) 1 0% 100% 0.00 37615 sched lock 9 0xc000000002d3cce4 1 0% 100% 0.00 211 sched lock 71 (usermode) 1 0% 100% 0.00 1893 sched lock 70 (usermode) 1 0% 100% 0.00 512 sched lock 9 sched_add+0x48c 1 0% 100% 0.00 2152 sched lock 7 (usermode) 1 0% 100% 0.00 772 sched lock 69 0xc000000002d3cce4 1 0% 100% 0.00 3469 sched lock 65 sched_add+0x48c 1 0% 100% 0.00 2806 sched lock 62 0xc000000002d3cce4 1 0% 100% 0.00 373 sched lock 63 sched_add+0x48c 1 0% 100% 0.00 6577 sched lock 58 sched_add+0x48c 1 0% 100% 0.00 3347 sched lock 58 (usermode) 1 0% 100% 0.00 805 sched lock 58 0xc000000002d3cccc 1 0% 100% 0.00 567 sched lock 58 0xc000000002d3cce4 1 0% 100% 0.00 6320 sched lock 6 sched_add+0x48c 1 0% 100% 0.00 6167 sched lock 45 0xc000000002d3cce4 1 0% 100% 0.00 1740 sched lock 45 0xc000000002d3cccc 1 0% 100% 0.00 1601 sched lock 45 sched_switch+0x81c 1 0% 100% 0.00 1121 sched lock 41 (usermode) 1 0% 100% 0.00 359 sched lock 43 sched_add+0x48c 1 0% 100% 0.00 1073 sched lock 52 0xc000000002d3cce4 1 0% 100% 0.00 443 sched lock 49 sched_add+0x48c 1 0% 100% 0.00 703 sched lock 48 sched_add+0x48c 1 0% 100% 0.00 304 sched lock 54 sched_add+0x48c 1 0% 100% 0.00 607 sched lock 53 sched_add+0x48c 1 0% 100% 0.00 335 sched lock 55 sched_add+0x48c 1 0% 100% 0.00 21984 sched lock 40 0xc000000002d3cccc 1 0% 100% 0.00 229 sched lock 40 sched_add+0x48c 1 0% 100% 0.00 1244 sched lock 31 sched_add+0x48c 1 0% 100% 0.00 1593 sched lock 30 0xc000000002d3cce4 1 0% 100% 0.00 8310 sched lock 30 0xc000000002d3cccc 1 0% 100% 0.00 686 sched lock 35 (usermode) 1 0% 100% 0.00 1422 sched lock 35 0xc000000002d3cccc 1 0% 100% 0.00 10354 sched lock 21 0xc000000002d3cccc 1 0% 100% 0.00 375 sched lock 21 sched_add+0x48c 1 0% 100% 0.00 217 sched lock 21 (usermode) 1 0% 100% 0.00 654 sched lock 20 sched_add+0x48c 1 0% 100% 0.00 1070 sched lock 20 0xc000000002d3cccc 1 0% 100% 0.00 438 sched lock 26 0xc000000002d3cccc 1 0% 100% 0.00 203 sched lock 3 0xc000000002d3cccc 1 0% 100% 0.00 1881 sched lock 30 (usermode) 1 0% 100% 0.00 1855 sched lock 0 (usermode) 1 0% 100% 0.00 209 et_hw_mtx cpu_new_callout+0x370 1 0% 100% 0.00 236 callout 0xc000000002d1b164 1 0% 100% 0.00 1925 sched lock 11 sched_add+0x48c 1 0% 100% 0.00 4050 sched lock 1 0xc000000002d3cccc 1 0% 100% 0.00 334 sched lock 1 sched_add+0x48c 1 0% 100% 0.00 3675 sched lock 17 (usermode) ------------------------------------------------------------------------------- Thread lock spin: 727 events in 2.312 seconds (314 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 351 48% 48% 0.00 4385 sleepq chain _thread_lock+0xf4 247 34% 82% 0.00 4454 sleepq chain (usermode) 24 3% 86% 0.00 5294 turnstile lock _thread_lock+0xf4 5 1% 86% 0.00 5977 sched lock 55 _thread_lock+0xf4 5 1% 87% 0.00 1697 sched lock 40 (usermode) 5 1% 88% 0.00 4436 sched lock 21 _thread_lock+0xf4 4 1% 88% 0.00 10872 sched lock 9 _thread_lock+0xf4 4 1% 89% 0.00 10238 sched lock 40 _thread_lock+0xf4 4 1% 89% 0.00 68299 sched lock 25 _thread_lock+0xf4 4 1% 90% 0.00 4884 sched lock 0 _thread_lock+0xf4 3 0% 90% 0.00 507 sched lock 71 (usermode) 3 0% 91% 0.00 16464 sched lock 63 _thread_lock+0xf4 3 0% 91% 0.00 5998 sched lock 54 _thread_lock+0xf4 3 0% 91% 0.00 11706 sched lock 49 _thread_lock+0xf4 3 0% 92% 0.00 4124 sched lock 47 _thread_lock+0xf4 3 0% 92% 0.00 4026 sched lock 27 _thread_lock+0xf4 3 0% 93% 0.00 4022 sched lock 20 _thread_lock+0xf4 3 0% 93% 0.00 5020 sched lock 2 _thread_lock+0xf4 2 0% 93% 0.00 6054 sched lock 71 _thread_lock+0xf4 2 0% 94% 0.00 5485 sched lock 53 (usermode) 2 0% 94% 0.00 10692 sched lock 31 _thread_lock+0xf4 2 0% 94% 0.00 11642 sched lock 15 _thread_lock+0xf4 2 0% 94% 0.00 3131 sched lock 15 (usermode) 1 0% 95% 0.00 5613 sched lock 9 (usermode) 1 0% 95% 0.00 6343 sched lock 62 _thread_lock+0xf4 1 0% 95% 0.00 5833 sched lock 62 (usermode) 1 0% 95% 0.00 972054 sched lock 6 (usermode) 1 0% 95% 0.00 3440 sched lock 65 _thread_lock+0xf4 1 0% 95% 0.00 588 sched lock 70 (usermode) 1 0% 95% 0.00 653 sched lock 51 _thread_lock+0xf4 1 0% 96% 0.00 3601 sched lock 50 _thread_lock+0xf4 1 0% 96% 0.00 668 sched lock 49 (usermode) 1 0% 96% 0.00 1940 sched lock 5 (usermode) 1 0% 96% 0.00 582 sched lock 54 (usermode) 1 0% 96% 0.00 5100 sched lock 58 _thread_lock+0xf4 1 0% 96% 0.00 842 sched lock 57 _thread_lock+0xf4 1 0% 96% 0.00 11082 sched lock 59 (usermode) 1 0% 97% 0.00 805 sched lock 59 _thread_lock+0xf4 1 0% 97% 0.00 305 sched lock 3 _thread_lock+0xf4 1 0% 97% 0.00 3712 sched lock 29 _thread_lock+0xf4 1 0% 97% 0.00 3839 sched lock 26 _thread_lock+0xf4 1 0% 97% 0.00 502 sched lock 24 _thread_lock+0xf4 1 0% 97% 0.00 11048 sched lock 25 (usermode) 1 0% 97% 0.00 11427 sched lock 21 (usermode) 1 0% 98% 0.00 335 sched lock 22 _thread_lock+0xf4 1 0% 98% 0.00 379 sched lock 23 (usermode) 1 0% 98% 0.00 21150 sched lock 23 _thread_lock+0xf4 1 0% 98% 0.00 303 sched lock 36 _thread_lock+0xf4 1 0% 98% 0.00 10913 sched lock 35 (usermode) 1 0% 98% 0.00 6157 sched lock 30 _thread_lock+0xf4 1 0% 98% 0.00 578 sched lock 33 (usermode) 1 0% 98% 0.00 683 sched lock 41 _thread_lock+0xf4 1 0% 99% 0.00 1831 sched lock 41 (usermode) 1 0% 99% 0.00 6368 sched lock 45 (usermode) 1 0% 99% 0.00 597 sched lock 44 (usermode) 1 0% 99% 0.00 691 sched lock 46 (usermode) 1 0% 99% 0.00 562 sched lock 20 (usermode) 1 0% 99% 0.00 2355 sched lock 19 _thread_lock+0xf4 1 0% 99% 0.00 10289 sched lock 2 (usermode) 1 0% 100% 0.00 12623 sched lock 17 (usermode) 1 0% 100% 0.00 573 sched lock 10 (usermode) 1 0% 100% 0.00 648 sched lock 10 _thread_lock+0xf4 1 0% 100% 0.00 11873 sched lock 11 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 602 events in 2.312 seconds (260 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 120 20% 20% 0.00 65600020 vm object vm_object_deallocate+0x800 116 19% 39% 0.00 70170795 vm object vm_fault_hold+0x1b28 99 16% 56% 0.00 79556183 vm object vm_map_insert+0x598 65 11% 66% 0.00 76246398 vm object vm_object_reference+0x134 50 8% 75% 0.00 66318279 vm object 0xc0000000030f8558 47 8% 83% 0.00 50317642 vm object (usermode) 38 6% 89% 0.00 67955669 vm object vm_object_shadow+0x390 35 6% 95% 0.00 83620502 vm object vm_object_shadow+0x378 14 2% 97% 0.00 77051714 vm object vm_imgact_map_page+0x3f0 13 2% 99% 0.00 58934910 vm object exec_map_first_page+0x780 2 0% 100% 0.00 232915 vm object 0xc00000000311fc9c 2 0% 100% 0.00 127481 vm object zfs_vmobject_wlock+0x9c 1 0% 100% 0.00 300859 vm object vm_page_sleep_if_busy+0x1f8 ------------------------------------------------------------------------------- R/W writer blocked by readers: 1132 events in 2.312 seconds (490 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 454 40% 40% 0.00 57576093 vm object vm_fault_hold+0x1b28 126 11% 51% 0.00 37714592 vm object (usermode) 125 11% 62% 0.00 58927831 vm object vm_object_deallocate+0x800 100 9% 71% 0.00 58312926 vm object vm_object_deallocate+0x814 92 8% 79% 0.00 57964783 vm object vnode_create_vobject+0x18c 89 8% 87% 0.00 69703307 vm object 0xc0000000030f8558 63 6% 93% 0.00 82995125 vm object vm_object_reference+0x134 25 2% 95% 0.00 55527465 vm object vm_imgact_map_page+0x3f0 21 2% 97% 0.00 56207259 vm object vm_map_insert+0x598 15 1% 98% 0.00 54829884 vm object vm_object_shadow+0x378 10 1% 99% 0.00 139057 vm object zfs_vmobject_wlock+0x9c 5 0% 99% 0.00 43686798 vm object vm_object_shadow+0x390 3 0% 100% 0.00 105632 vm object 0xc00000000311fc9c 2 0% 100% 0.00 134251 vm object vm_fault_hold+0x1298 2 0% 100% 0.00 48594 vm object vm_fault_hold+0x2700 ------------------------------------------------------------------------------- R/W reader blocked by writer: 190 events in 2.312 seconds (82 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 124 65% 65% 0.00 40570981 vm object __rw_rlock_int+0x9c 66 35% 100% 0.00 25898163 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 35 events in 2.312 seconds (15 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 30 86% 86% 0.00 32233243 vm object __rw_rlock_int+0x9c 5 14% 100% 0.00 30406152 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 1434 events in 2.312 seconds (620 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 241 17% 17% 0.00 17235 vm object vm_fault_hold+0x2700 206 14% 31% 0.00 3829318 vm object vm_object_deallocate+0x800 182 13% 44% 0.00 4285887 vm object vm_fault_hold+0x1b28 165 12% 55% 0.00 1290018 vm object (usermode) 135 9% 65% 0.00 5731553 vm object vm_map_insert+0x598 112 8% 73% 0.00 4447503 vm object vm_object_reference+0x134 70 5% 77% 0.00 4698122 vm object 0xc0000000030f8558 64 4% 82% 0.00 4294254 vm object vm_object_shadow+0x378 60 4% 86% 0.00 4529773 vm object vm_object_shadow+0x390 40 3% 89% 0.00 212910 vm object zfs_vmobject_wlock+0x9c 39 3% 92% 0.00 3112878 vm object vm_imgact_map_page+0x3f0 23 2% 93% 0.00 212022 vm object 0xc00000000311fc9c 20 1% 95% 0.00 226226 vm object vm_page_sleep_if_busy+0x1f8 14 1% 96% 0.00 6767745 vm object exec_map_first_page+0x780 14 1% 97% 0.00 199063 kernel vm object 0xc0000000030f09c0 9 1% 97% 0.00 122252 kernel vm object kmem_back_domain+0x340 7 0% 98% 0.00 15009 Per-Filesystem Softdep Lock (usermode) 4 0% 98% 0.00 19470 vm object vm_object_collapse+0x710 4 0% 98% 0.00 187056 kernel vm object (usermode) 3 0% 98% 0.00 11045 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 3 0% 99% 0.00 5204 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 3 0% 99% 0.00 1529 Per-Filesystem Softdep Lock 0xc000000003095904 2 0% 99% 0.00 27024 vm object vm_object_deallocate+0x814 2 0% 99% 0.00 16367 Per-Filesystem Softdep Lock 0xc00000000309c728 1 0% 99% 0.00 291 vm object vnode_create_vobject+0x18c 1 0% 99% 0.00 20105 vm object 0xc00000000311f65c 1 0% 99% 0.00 11064 vm object bdwrite+0x2f4 1 0% 99% 0.00 11231 tcpinp (usermode) 1 0% 100% 0.00 10939 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 1 0% 100% 0.00 12235 Per-Filesystem Softdep Lock 0xc00000000309f2b4 1 0% 100% 0.00 12831 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 1 0% 100% 0.00 23734 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c 1 0% 100% 0.00 367 Per-Filesystem Softdep Lock softdep_slowdown+0x374 1 0% 100% 0.00 11475 Per-Filesystem Softdep Lock 0xc00000000308d320 1 0% 100% 0.00 29768 Per-Filesystem Softdep Lock 0xc00000000309c6f4 1 0% 100% 0.00 11899 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c ------------------------------------------------------------------------------- R/W writer spin on readers: 1426 events in 2.312 seconds (617 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 465 33% 33% 0.00 4745809 vm object vm_fault_hold+0x1b28 155 11% 43% 0.00 2443493 vm object (usermode) 141 10% 53% 0.00 4767680 vm object vm_object_deallocate+0x800 102 7% 61% 0.00 5151280 vm object vm_object_deallocate+0x814 98 7% 67% 0.00 40698 vm object vm_fault_hold+0x1298 95 7% 74% 0.00 4942249 vm object 0xc0000000030f8558 93 7% 81% 0.00 4341850 vm object vnode_create_vobject+0x18c 92 6% 87% 0.00 23810 vm object vm_fault_hold+0x2700 63 4% 91% 0.00 6759396 vm object vm_object_reference+0x134 30 2% 94% 0.00 3905709 vm object vm_imgact_map_page+0x3f0 30 2% 96% 0.00 494953 vm object zfs_vmobject_wlock+0x9c 23 2% 97% 0.00 5843125 vm object vm_map_insert+0x598 16 1% 98% 0.00 4235098 vm object vm_object_shadow+0x378 10 1% 99% 0.00 426254 vm object 0xc00000000311fc9c 6 0% 100% 0.00 7690900 vm object vm_object_shadow+0x390 5 0% 100% 0.00 22883 vm object vm_page_sleep_if_busy+0x1f8 2 0% 100% 0.00 12617 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader spin on writer: 588 events in 2.312 seconds (254 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 458 78% 78% 0.00 610225 vm object __rw_rlock_int+0x9c 130 22% 100% 0.00 694536 vm object (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 96 events in 2.312 seconds (42 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 86 90% 90% 0.00 671401 vm object __rw_rlock_int+0x9c 10 10% 100% 0.00 987559 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 48 events in 2.312 seconds (21 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 36 75% 75% 0.00 48618 vm map (user) (usermode) 10 21% 96% 0.00 161005 vm map (user) _vm_map_lock+0x138 2 4% 100% 0.00 33605 vm map (user) _vm_map_lock_upgrade+0x180 ------------------------------------------------------------------------------- SX shared block: 14 events in 2.312 seconds (6 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 14 100% 100% 0.00 408588 vm map (user) _sx_slock_int+0xa0 ------------------------------------------------------------------------------- SX exclusive spin: 2223 events in 2.312 seconds (961 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 719 32% 32% 0.00 32320 vm map (user) (usermode) 336 15% 47% 0.00 27115 vm map (user) _vm_map_lock+0x138 154 7% 54% 0.00 4923200 vq->vq_lock (usermode) 122 5% 60% 0.00 3197 rrl->rr_lock 0xc00000000260abbc 97 4% 64% 0.00 3980 rrl->rr_lock rrw_exit+0x310 90 4% 68% 0.00 12215 proctree kern_wait6+0x3e8 87 4% 72% 0.00 34997 tm->tm_lock trim_map_free+0x19c 63 3% 75% 0.00 1257528 vq->vq_lock vdev_queue_io+0x268 57 3% 78% 0.00 1654 zp->z_acl_lock 0xc0000000026790e0 46 2% 80% 0.00 714994 os->os_userused_lock 0xc0000000025b7cdc 45 2% 82% 0.00 537526 os->os_userused_lock (usermode) 41 2% 84% 0.00 27597 vm map (user) _vm_map_lock_upgrade+0x180 28 1% 85% 0.00 4524 zio->io_lock (usermode) 26 1% 86% 0.00 462887 os->os_userused_lock 0xc0000000025b7e74 24 1% 87% 0.00 14184 zio->io_lock 0xc0000000026bfad8 24 1% 88% 0.00 83839 db->db_mtx (usermode) 23 1% 89% 0.00 353492 h->hash_mutexes[i] dbuf_find+0x114 19 1% 90% 0.00 2488 zp->z_acl_lock (usermode) 17 1% 91% 0.00 337285 h->hash_mutexes[i] (usermode) 16 1% 91% 0.00 832365 vq->vq_lock 0xc0000000026563c8 14 1% 92% 0.00 15420 zio->io_lock zio_add_child+0x274 13 1% 93% 0.00 34671 zio->io_lock 0xc0000000026c1ac0 13 1% 93% 0.00 941606 vq->vq_lock vdev_queue_io_done+0x278 13 1% 94% 0.00 5141 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x268 11 0% 94% 0.00 5830 rrl->rr_lock (usermode) 11 0% 95% 0.00 7297 db->db_mtx dbuf_read+0xa44 11 0% 95% 0.00 229865 db->db_mtx dbuf_find+0x12c 10 0% 96% 0.00 5921 ms->ms_lock metaslab_alloc_dva+0x1138 9 0% 96% 0.00 3305 mc->mc_lock metaslab_class_throttle_unreserve+0x154 7 0% 97% 0.00 334731 h->hash_mutexes[i] dbuf_destroy+0x6dc 7 0% 97% 0.00 4303 dd->dd_lock (usermode) 6 0% 97% 0.00 1127 scl->scl_lock spa_config_exit+0x84 6 0% 97% 0.00 1395 ms->ms_lock (usermode) 6 0% 98% 0.00 6373 dd->dd_lock dsl_dir_tempreserve_clear+0x88 5 0% 98% 0.00 86525 tm->tm_lock (usermode) 5 0% 98% 0.00 5409 db->db_mtx dbuf_rele+0xbc 4 0% 98% 0.00 8671 vq->vq_lock vdev_queue_io_done+0x12c 4 0% 98% 0.00 6213 db->db_mtx dmu_buf_hold_array_by_dnode+0x6d0 3 0% 99% 0.00 11755 proctree (usermode) 3 0% 99% 0.00 412 dd->dd_lock 0xc0000000025e7c24 2 0% 99% 0.00 220 scl->scl_lock spa_config_enter+0x140 2 0% 99% 0.00 20958 proctree killjobc+0x600 2 0% 99% 0.00 176 mg->mg_lock metaslab_alloc_dva+0x3a8 2 0% 99% 0.00 249 mc->mc_lock (usermode) 2 0% 99% 0.00 1626 dp->dp_lock dsl_pool_undirty_space+0x18c 2 0% 99% 0.00 3073 dd->dd_lock dsl_dir_diduse_space+0x270 2 0% 99% 0.00 342 db->db_mtx dmu_buf_will_dirty+0x2e8 1 0% 99% 0.00 453 zp->z_range_lock zfs_range_lock+0x854 1 0% 99% 0.00 11606 zio->io_lock 0xc0000000026be04c 1 0% 100% 0.00 11352 zio->io_lock zio_wait+0x1e4 1 0% 100% 0.00 207 scl->scl_lock (usermode) 1 0% 100% 0.00 48174 proctree exit1+0x1aec 1 0% 100% 0.00 197 ds->ds_lock (usermode) 1 0% 100% 0.00 322 dp->dp_lock dsl_pool_need_dirty_delay+0x1ac 1 0% 100% 0.00 5851 dn->dn_mtx 0xc0000000025a65f8 1 0% 100% 0.00 219 dn->dn_mtx dnode_diduse_space+0x184 1 0% 100% 0.00 217 dp->dp_lock (usermode) 1 0% 100% 0.00 166 ds->ds_lock dsl_dataset_block_born+0x3d4 1 0% 100% 0.00 785 hdl->sa_lock (usermode) 1 0% 100% 0.00 11563 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 1 0% 100% 0.00 13756 dn->dn_dbufs_mtx (usermode) ------------------------------------------------------------------------------- SX shared spin: 904 events in 2.312 seconds (391 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 794 88% 88% 0.00 31060 vm map (user) _sx_slock_int+0xa0 109 12% 100% 0.00 27951 vm map (user) (usermode) 1 0% 100% 0.00 406 zf->zf_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- ^[[A root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 47376 events in 1.503 seconds (31522 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 14258 30% 30% 0.00 13394 page pv (usermode) 12766 27% 57% 0.00 13235 page pv 0xc000000003142640 11307 24% 81% 0.00 13476 page pv moea64_enter+0x86c 1380 3% 84% 0.00 5411 vm active pagequeue 0xc00000000310e854 1337 3% 87% 0.00 7636 vm page vm_page_change_lock+0x160 981 2% 89% 0.00 18599 vnode interlock vget+0x2f8 632 1% 90% 0.00 18387 vnode interlock (usermode) 363 1% 91% 0.00 167099 zio_write_issue 0xc000000002d7c86c 320 1% 91% 0.00 9960 struct mount mtx vfs_unbusy+0x15c 299 1% 92% 0.00 8679 vm active pagequeue (usermode) 293 1% 93% 0.00 11256 vnode interlock 0xc000000002e224a0 264 1% 93% 0.00 7383 struct mount mtx vfs_busy+0x434 248 1% 94% 0.00 216311 zio_write_issue taskqueue_enqueue+0x158 241 1% 94% 0.00 15302 dp_sync_taskq 0xc000000002cc30c0 237 1% 95% 0.00 27289 dp_sync_taskq 0xc000000002d7c86c 235 0% 95% 0.00 16748 dp_sync_taskq (usermode) 210 0% 96% 0.00 6404 vm page vm_fault_hold+0x2724 169 0% 96% 0.00 207814 zio_write_issue (usermode) 166 0% 96% 0.00 6310 vm page (usermode) 164 0% 97% 0.00 52904 kernelpmap moea64_kextract+0x174 157 0% 97% 0.00 23757 zio_write_intr (usermode) 140 0% 97% 0.00 23039 zio_write_intr 0xc000000002d7c86c 140 0% 98% 0.00 6016 struct mount mtx (usermode) 139 0% 98% 0.00 80279 zio_write_issue 0xc000000002cc30c0 118 0% 98% 0.00 17551 dp_sync_taskq taskqueue_enqueue+0x158 102 0% 99% 0.00 12447 zio_write_intr 0xc000000002cc30c0 93 0% 99% 0.00 46579 vnode interlock 0xc0000000030ce5b8 93 0% 99% 0.00 11496 page pv moea64_enter+0x910 77 0% 99% 0.00 12316 vnode interlock 0xc000000002e22484 57 0% 99% 0.00 35200 kernelpmap moea64_enter+0x880 52 0% 99% 0.00 3316 vm active pagequeue vm_page_dequeue+0x1b4 44 0% 99% 0.00 14483 zio_write_intr taskqueue_enqueue+0x158 37 0% 99% 0.00 3486 struct mount vlist mtx _vdrop+0x6f4 29 0% 100% 0.00 2334 struct mount vlist mtx _vhold+0x474 26 0% 100% 0.00 62458 kernelpmap (usermode) 16 0% 100% 0.00 2599 struct mount vlist mtx (usermode) 14 0% 100% 0.00 4272 select mtxpool selrecord+0x1dc 12 0% 100% 0.00 5050 ttymtx ttydisc_write+0x388 12 0% 100% 0.00 4315 ttymtx ttyoutq_read_uio+0x34c 10 0% 100% 0.00 5107 struct mount mtx vfs_ref+0x114 10 0% 100% 0.00 6544 select mtxpool 0xc000000002d93038 10 0% 100% 0.00 81735 kernelpmap moea64_remove+0x200 8 0% 100% 0.00 722 UPVO entry (usermode) 7 0% 100% 0.00 711 vm page 0xc0000000030ec28c 6 0% 100% 0.00 10825 vm page free queue 0xc000000003111274 6 0% 100% 0.00 2272 ttymtx 0xc000000002db18d8 6 0% 100% 0.00 195 pipe mutex 0xc000000002d95074 6 0% 100% 0.00 1699 pipe mutex 0xc000000002d93828 5 0% 100% 0.00 9570 vm page free queue (usermode) 5 0% 100% 0.00 1441 ttymtx 0xc000000002db84b4 5 0% 100% 0.00 1632 ttymtx 0xc000000002db7520 5 0% 100% 0.00 1883 UPVO entry uma_zfree_arg+0x5c8 4 0% 100% 0.00 1066 vnode interlock _vhold+0x4a4 4 0% 100% 0.00 722 vm page vm_page_rename+0x224 4 0% 100% 0.00 15869 struct mount mtx vfs_rel+0x128 4 0% 100% 0.00 4113 pipe mutex 0xc000000002d9456c 4 0% 100% 0.00 29988 nvme qpair lock 0xc000000002a101b0 4 0% 100% 0.00 17700 dp_sync_taskq taskqueue_quiesce+0x2e8 3 0% 100% 0.00 48685 vm page exec_map_first_page+0x7b8 3 0% 100% 0.00 902 UPVO entry uma_zalloc_arg+0x98c 3 0% 100% 0.00 1175 CAM device lock 0xc000000002524148 2 0% 100% 0.00 12892 vm page exec_unmap_first_page+0x188 2 0% 100% 0.00 232 pipe mutex 0xc000000002d95914 2 0% 100% 0.00 13404 metaslab_group_taskq (usermode) 1 0% 100% 0.00 224 zio_free_issue_6 (usermode) 1 0% 100% 0.00 709 zio_free_issue_2 0xc000000002d7c86c 1 0% 100% 0.00 260 zio_null_issue (usermode) 1 0% 100% 0.00 5950 vm page free queue 0xc000000003110fb4 1 0% 100% 0.00 3259 vnode interlock vinactive+0x25c 1 0% 100% 0.00 184 zio_free_issue_1 taskqueue_enqueue+0x158 1 0% 100% 0.00 940 ttymtx (usermode) 1 0% 100% 0.00 10372 vm page vm_imgact_unmap_page+0x184 1 0% 100% 0.00 11419 nvme qpair lock nvme_qpair_submit_request+0x120 1 0% 100% 0.00 641 metaslab_group_taskq taskqueue_enqueue+0x158 1 0% 100% 0.00 2879 kernel arena domain vmem_xalloc+0x8c0 1 0% 100% 0.00 326 CAM device lock (usermode) 1 0% 100% 0.00 3236 reseed mutex 0xc000000002a42484 1 0% 100% 0.00 1111 process lock kern_sigprocmask+0x378 1 0% 100% 0.00 187 pipe mutex (usermode) 1 0% 100% 0.00 440 pipe mutex 0xc000000002d93a60 1 0% 100% 0.00 543 sellck 0xc000000002d92870 1 0% 100% 0.00 11152 sellck 0xc000000002cc30c0 1 0% 100% 0.00 1716 select mtxpool (usermode) 1 0% 100% 0.00 176 struct mount mtx vn_start_write+0x1a0 ------------------------------------------------------------------------------- Adaptive mutex block: 10 events in 1.503 seconds (7 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 10% 10% 0.00 21103 zio_write_issue 0xc000000002d7c86c 1 10% 20% 0.00 36260 zio_write_intr 0xc000000002cc30c0 1 10% 30% 0.00 37720 zio_write_intr (usermode) 1 10% 40% 0.00 89668 vm page (usermode) 1 10% 50% 0.00 153639 vm page vm_page_change_lock+0x160 1 10% 60% 0.00 230542 page pv moea64_enter+0x86c 1 10% 70% 0.00 19484 CAM device lock 0xc000000002524ac0 1 10% 80% 0.00 54707 page pv 0xc000000003142640 1 10% 90% 0.00 28604 select mtxpool (usermode) 1 10% 100% 0.00 26773 sellck 0xc000000002cc30c0 ------------------------------------------------------------------------------- Spin lock spin: 9504 events in 1.503 seconds (6324 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 4304 45% 45% 0.00 376100 turnstile chain (usermode) 3536 37% 82% 0.00 347633 turnstile chain turnstile_trywait+0x208 950 10% 92% 0.00 22041 sleepq chain sleepq_lock+0xe4 338 4% 96% 0.00 193010 sleepq chain (usermode) 115 1% 97% 0.00 118193 turnstile chain turnstile_chain_lock+0xdc 55 1% 98% 0.00 641 turnstile lock (usermode) 53 1% 98% 0.00 482 turnstile lock turnstile_trywait+0x150 36 0% 99% 0.00 49812 sched lock 65 0xc000000002d3cce4 20 0% 99% 0.00 37905 sched lock 65 (usermode) 8 0% 99% 0.00 11243 sched lock 50 0xc000000002d3cce4 5 0% 99% 0.00 35331 sched lock 10 0xc000000002d3cccc 4 0% 99% 0.00 101031 sched lock 65 0xc000000002d3cccc 4 0% 99% 0.00 4284 sched lock 65 sched_switch+0x81c 4 0% 99% 0.00 16075 sched lock 50 0xc000000002d3cccc 3 0% 99% 0.00 9448 sched lock 70 0xc000000002d3cce4 3 0% 99% 0.00 1520 sched lock 7 0xc000000002d3cccc 3 0% 99% 0.00 19042 sched lock 5 0xc000000002d3cccc 3 0% 99% 0.00 10440 sched lock 3 sched_add+0x48c 3 0% 99% 0.00 21343 sched lock 20 0xc000000002d3cccc 3 0% 99% 0.00 1305 sched lock 15 sched_add+0x48c 2 0% 99% 0.00 1898 sched lock 70 sched_add+0x48c 2 0% 99% 0.00 1079 sched lock 57 sched_add+0x48c 2 0% 99% 0.00 1101 sched lock 51 sched_add+0x48c 2 0% 100% 0.00 2952 sched lock 30 0xc000000002d3cccc 2 0% 100% 0.00 1876 sched lock 30 sched_switch+0x81c 2 0% 100% 0.00 46217 sched lock 10 0xc000000002d3cce4 1 0% 100% 0.00 129 turnstile lock turnstile_lookup+0xd4 1 0% 100% 0.00 273 td_contested turnstile_broadcast+0x1bc 1 0% 100% 0.00 6226 td_contested turnstile_wait+0x298 1 0% 100% 0.00 244 sched lock 70 0xc000000002d3cccc 1 0% 100% 0.00 6465 sched lock 71 sched_switch+0xca0 1 0% 100% 0.00 615 sched lock 8 0xc000000002d3cccc 1 0% 100% 0.00 398 sched lock 70 sched_switch+0xca0 1 0% 100% 0.00 10907 sched lock 70 (usermode) 1 0% 100% 0.00 13260 sched lock 57 (usermode) 1 0% 100% 0.00 3361 sched lock 6 sched_add+0x48c 1 0% 100% 0.00 168 sched lock 64 sched_add+0x48c 1 0% 100% 0.00 1067 sched lock 50 sched_add+0x48c 1 0% 100% 0.00 1711 sched lock 50 sched_switch+0x81c 1 0% 100% 0.00 3308 sched lock 54 (usermode) 1 0% 100% 0.00 26292 sched lock 54 sched_add+0x48c 1 0% 100% 0.00 2058 sched lock 55 0xc000000002d3cce4 1 0% 100% 0.00 476 sched lock 56 sched_add+0x48c 1 0% 100% 0.00 919 sched lock 5 sched_switch+0x81c 1 0% 100% 0.00 41275 sched lock 5 (usermode) 1 0% 100% 0.00 184 sched lock 49 sched_add+0x48c 1 0% 100% 0.00 164 sched lock 41 sched_add+0x48c 1 0% 100% 0.00 1925 sched lock 45 0xc000000002d3cce4 1 0% 100% 0.00 379 sched lock 48 sched_switch+0xca0 1 0% 100% 0.00 1961 sched lock 36 sched_add+0x48c 1 0% 100% 0.00 1545 sched lock 34 0xc000000002d3cce4 1 0% 100% 0.00 1196 sched lock 37 sched_add+0x48c 1 0% 100% 0.00 5519 sched lock 40 sched_switch+0x81c 1 0% 100% 0.00 3420 sched lock 40 sched_add+0x48c 1 0% 100% 0.00 209 sched lock 25 0xc000000002d3cccc 1 0% 100% 0.00 3365 sched lock 21 sched_add+0x48c 1 0% 100% 0.00 1869 sched lock 22 sched_add+0x48c 1 0% 100% 0.00 3497 sched lock 30 sched_add+0x48c 1 0% 100% 0.00 156 sched lock 32 sched_add+0x48c 1 0% 100% 0.00 88109 sched lock 33 (usermode) 1 0% 100% 0.00 1696 sched lock 33 sched_add+0x48c 1 0% 100% 0.00 560 sched lock 12 0xc000000002d3cccc 1 0% 100% 0.00 2001 sched lock 12 sched_add+0x48c 1 0% 100% 0.00 5860 sched lock 1 sched_add+0x48c 1 0% 100% 0.00 6212 sched lock 15 (usermode) 1 0% 100% 0.00 3492 sched lock 13 (usermode) 1 0% 100% 0.00 5939 sched lock 20 (usermode) 1 0% 100% 0.00 9598 sched lock 20 sched_switch+0x81c ------------------------------------------------------------------------------- Thread lock spin: 349 events in 1.503 seconds (232 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 201 58% 58% 0.00 3848 sleepq chain _thread_lock+0xf4 83 24% 81% 0.00 5363 sleepq chain (usermode) 5 1% 83% 0.00 5200 turnstile lock _thread_lock+0xf4 4 1% 84% 0.00 8839 sched lock 50 _thread_lock+0xf4 4 1% 85% 0.00 2040 sched lock 49 (usermode) 3 1% 86% 0.00 5949 sched lock 70 _thread_lock+0xf4 2 1% 87% 0.00 129094 sched lock 65 _thread_lock+0xf4 2 1% 87% 0.00 8521 sched lock 50 (usermode) 2 1% 88% 0.00 11548 sched lock 5 (usermode) 2 1% 88% 0.00 5681 sched lock 33 _thread_lock+0xf4 2 1% 89% 0.00 5465 sched lock 30 _thread_lock+0xf4 2 1% 89% 0.00 6516 sched lock 20 _thread_lock+0xf4 2 1% 90% 0.00 10624 sched lock 2 _thread_lock+0xf4 2 1% 91% 0.00 5509 sched lock 19 _thread_lock+0xf4 2 1% 91% 0.00 3378 sched lock 15 _thread_lock+0xf4 2 1% 92% 0.00 17160 sched lock 15 (usermode) 2 1% 92% 0.00 3396 sched lock 10 _thread_lock+0xf4 1 0% 93% 0.00 904 sched lock 71 _thread_lock+0xf4 1 0% 93% 0.00 3318 sched lock 70 (usermode) 1 0% 93% 0.00 576 sched lock 67 _thread_lock+0xf4 1 0% 93% 0.00 729 sched lock 62 (usermode) 1 0% 94% 0.00 11655 sched lock 51 (usermode) 1 0% 94% 0.00 2402 sched lock 57 _thread_lock+0xf4 1 0% 94% 0.00 6126 sched lock 5 _thread_lock+0xf4 1 0% 95% 0.00 2306 sched lock 44 _thread_lock+0xf4 1 0% 95% 0.00 356 sched lock 41 _thread_lock+0xf4 1 0% 95% 0.00 22642 sched lock 41 (usermode) 1 0% 95% 0.00 11524 sched lock 37 (usermode) 1 0% 96% 0.00 11683 sched lock 40 (usermode) 1 0% 96% 0.00 24962 sched lock 40 _thread_lock+0xf4 1 0% 96% 0.00 11249 sched lock 3 _thread_lock+0xf4 1 0% 97% 0.00 1919 sched lock 3 (usermode) 1 0% 97% 0.00 3215 sched lock 33 (usermode) 1 0% 97% 0.00 40221 sched lock 21 _thread_lock+0xf4 1 0% 97% 0.00 802 sched lock 19 (usermode) 1 0% 98% 0.00 643 sched lock 24 _thread_lock+0xf4 1 0% 98% 0.00 665 sched lock 24 (usermode) 1 0% 98% 0.00 824 sched lock 26 (usermode) 1 0% 99% 0.00 453 sched lock 28 (usermode) 1 0% 99% 0.00 664 sched lock 28 _thread_lock+0xf4 1 0% 99% 0.00 449 sched lock 18 _thread_lock+0xf4 1 0% 99% 0.00 330 sched lock 16 _thread_lock+0xf4 1 0% 100% 0.00 611 sched lock 0 _thread_lock+0xf4 1 0% 100% 0.00 40977 sched lock 0 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 246 events in 1.503 seconds (164 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 58 24% 24% 0.00 59533033 vm object vm_object_deallocate+0x800 42 17% 41% 0.00 81574346 vm object vm_object_reference+0x134 42 17% 58% 0.00 62372692 vm object vm_fault_hold+0x1b28 37 15% 73% 0.00 85441542 vm object vm_map_insert+0x598 25 10% 83% 0.00 66643531 vm object 0xc0000000030f8558 15 6% 89% 0.00 90111729 vm object vm_object_shadow+0x390 9 4% 93% 0.00 62429309 vm object vm_object_shadow+0x378 6 2% 95% 0.00 32695659 vm object vm_imgact_map_page+0x3f0 6 2% 98% 0.00 39823797 vm object (usermode) 6 2% 100% 0.00 60287676 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W writer blocked by readers: 523 events in 1.503 seconds (348 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 175 33% 33% 0.00 54982774 vm object vm_fault_hold+0x1b28 80 15% 49% 0.00 45002671 vm object vm_object_deallocate+0x800 59 11% 60% 0.00 59951429 vm object 0xc0000000030f8558 52 10% 70% 0.00 53984328 vm object vm_object_deallocate+0x814 50 10% 80% 0.00 63776763 vm object vnode_create_vobject+0x18c 37 7% 87% 0.00 16396230 vm object (usermode) 30 6% 92% 0.00 77831910 vm object vm_object_reference+0x134 14 3% 95% 0.00 54597688 vm object vm_object_shadow+0x378 13 2% 98% 0.00 51086197 vm object vm_imgact_map_page+0x3f0 12 2% 100% 0.00 86353328 vm object vm_map_insert+0x598 1 0% 100% 0.00 6295359 vm object vm_object_shadow+0x390 ------------------------------------------------------------------------------- R/W reader blocked by writer: 96 events in 1.503 seconds (64 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 69 72% 72% 0.00 32469907 vm object __rw_rlock_int+0x9c 27 28% 100% 0.00 24738546 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 23 events in 1.503 seconds (15 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 15 65% 65% 0.00 40768432 vm object __rw_rlock_int+0x9c 8 35% 100% 0.00 23753055 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 493 events in 1.503 seconds (328 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 95 19% 19% 0.00 3710333 vm object vm_object_deallocate+0x800 72 15% 34% 0.00 3425925 vm object vm_fault_hold+0x1b28 69 14% 48% 0.00 4049435 vm object vm_map_insert+0x598 59 12% 60% 0.00 4977206 vm object vm_object_reference+0x134 37 8% 67% 0.00 512624 vm object (usermode) 36 7% 75% 0.00 4695729 vm object 0xc0000000030f8558 26 5% 80% 0.00 4365140 vm object vm_object_shadow+0x390 24 5% 85% 0.00 280933 kernel vm object kmem_back_domain+0x340 18 4% 88% 0.00 3036432 vm object vm_object_shadow+0x378 16 3% 92% 0.00 217799 kernel vm object 0xc0000000030f09c0 11 2% 94% 0.00 200657 kernel vm object (usermode) 9 2% 96% 0.00 3566829 vm object vm_imgact_map_page+0x3f0 6 1% 97% 0.00 5673828 vm object exec_map_first_page+0x780 6 1% 98% 0.00 14869 Per-Filesystem Softdep Lock (usermode) 1 0% 98% 0.00 45296 vm object vnode_create_vobject+0x18c 1 0% 99% 0.00 1855 vm object vm_object_collapse+0x710 1 0% 99% 0.00 180944 vm object vm_object_deallocate+0x814 1 0% 99% 0.00 3174 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 1 0% 99% 0.00 1091 Per-Filesystem Softdep Lock 0xc00000000309c6f4 1 0% 99% 0.00 2201 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 1 0% 100% 0.00 735 Per-Filesystem Softdep Lock 0xc000000003095904 1 0% 100% 0.00 233 Per-Filesystem Softdep Lock 0xc00000000309a308 1 0% 100% 0.00 7681 Per-Filesystem Softdep Lock 0xc0000000030adb58 ------------------------------------------------------------------------------- R/W writer spin on readers: 536 events in 1.503 seconds (357 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 180 34% 34% 0.00 4405519 vm object vm_fault_hold+0x1b28 84 16% 49% 0.00 4139699 vm object vm_object_deallocate+0x800 62 12% 61% 0.00 5375531 vm object 0xc0000000030f8558 52 10% 71% 0.00 5274038 vm object vm_object_deallocate+0x814 50 9% 80% 0.00 4676546 vm object vnode_create_vobject+0x18c 36 7% 87% 0.00 1597971 vm object (usermode) 30 6% 92% 0.00 6045563 vm object vm_object_reference+0x134 15 3% 95% 0.00 4643349 vm object vm_object_shadow+0x378 13 2% 97% 0.00 4699947 vm object vm_imgact_map_page+0x3f0 12 2% 100% 0.00 7759733 vm object vm_map_insert+0x598 1 0% 100% 0.00 5589 vm object 0xc00000000311f65c 1 0% 100% 0.00 2394476 vm object vm_object_shadow+0x390 ------------------------------------------------------------------------------- R/W reader spin on writer: 155 events in 1.503 seconds (103 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 102 66% 66% 0.00 1086387 vm object __rw_rlock_int+0x9c 52 34% 99% 0.00 748430 vm object (usermode) 1 1% 100% 0.00 287 ncbuc __rw_rlock_int+0x9c ------------------------------------------------------------------------------- R/W reader spin on write wanted: 24 events in 1.503 seconds (16 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 18 75% 75% 0.00 1292470 vm object __rw_rlock_int+0x9c 6 25% 100% 0.00 1084267 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 2 events in 1.503 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 50% 50% 0.00 122164 mc->mc_lock metaslab_class_throttle_unreserve+0x154 1 50% 100% 0.00 129667 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 ------------------------------------------------------------------------------- SX exclusive spin: 2184 events in 1.503 seconds (1453 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 387 18% 18% 0.00 4913 rrl->rr_lock 0xc00000000260abbc 292 13% 31% 0.00 5321 rrl->rr_lock rrw_exit+0x310 205 9% 40% 0.00 164734 vq->vq_lock (usermode) 153 7% 47% 0.00 4205 zp->z_acl_lock 0xc0000000026790e0 150 7% 54% 0.00 132933 vq->vq_lock vdev_queue_io+0x268 129 6% 60% 0.00 4107 zp->z_acl_lock (usermode) 96 4% 65% 0.00 23811 tm->tm_lock trim_map_free+0x19c 96 4% 69% 0.00 14861 proctree kern_wait6+0x3e8 55 3% 72% 0.00 602825 os->os_userused_lock 0xc0000000025b7cdc 48 2% 74% 0.00 15757 ms->ms_lock metaslab_alloc_dva+0x1138 41 2% 76% 0.00 9079 dd->dd_lock dsl_dir_tempreserve_clear+0x88 39 2% 77% 0.00 9853 zio->io_lock (usermode) 38 2% 79% 0.00 6474 rrl->rr_lock (usermode) 36 2% 81% 0.00 7142 zio->io_lock 0xc0000000026bfad8 33 2% 82% 0.00 7860 dd->dd_lock (usermode) 32 1% 84% 0.00 10434 mc->mc_lock metaslab_class_throttle_unreserve+0x154 28 1% 85% 0.00 521254 os->os_userused_lock 0xc0000000025b7e74 27 1% 86% 0.00 7045 zio->io_lock zio_add_child+0x274 24 1% 87% 0.00 749419 os->os_userused_lock (usermode) 23 1% 88% 0.00 10630 zio->io_lock 0xc0000000026c1ac0 20 1% 89% 0.00 280574 as->as_lock aggsum_add+0x360 19 1% 90% 0.00 139604 vq->vq_lock 0xc0000000026563c8 19 1% 91% 0.00 1640 dd->dd_lock 0xc0000000025e7c24 17 1% 92% 0.00 40433 vq->vq_lock vdev_queue_io_done+0x278 16 1% 93% 0.00 17570 scl->scl_lock spa_config_exit+0x84 14 1% 93% 0.00 3939 mg->mg_lock metaslab_alloc_dva+0x3a8 13 1% 94% 0.00 3842 scl->scl_lock spa_config_enter+0x140 11 1% 94% 0.00 68712 vq->vq_lock vdev_queue_io_done+0x12c 11 1% 95% 0.00 6564 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x268 11 1% 95% 0.00 3212 mg->mg_lock metaslab_alloc_dva+0x7d4 10 0% 96% 0.00 12999 dd->dd_lock dsl_dir_diduse_space+0x270 9 0% 96% 0.00 20238 ms->ms_lock (usermode) 9 0% 97% 0.00 3800 db->db_mtx dmu_buf_will_dirty+0x2e8 8 0% 97% 0.00 6769 spa->spa_alloc_locks[i] 0xc0000000026bd584 8 0% 97% 0.00 4552 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 7 0% 98% 0.00 208195 as->as_lock (usermode) 5 0% 98% 0.00 6372 spa->spa_alloc_locks[i] (usermode) 3 0% 98% 0.00 36967 scl->scl_lock (usermode) 3 0% 98% 0.00 281 mc->mc_lock (usermode) 3 0% 98% 0.00 1237 dp->dp_lock dsl_pool_undirty_space+0x18c 3 0% 98% 0.00 37358 dd->dd_lock dsl_dir_willuse_space+0x204 3 0% 99% 0.00 1245 bpl->bpl_lock bplist_append+0x158 2 0% 99% 0.00 13683 proctree exit1+0x1aec 2 0% 99% 0.00 17027 proctree (usermode) 2 0% 99% 0.00 1661 mg->mg_lock (usermode) 2 0% 99% 0.00 1070 hdl->sa_lock sa_lookup_uio+0x198 2 0% 99% 0.00 5529 ds->ds_lock dsl_dataset_block_kill+0x614 2 0% 99% 0.00 1397 dp->dp_lock (usermode) 2 0% 99% 0.00 1693 dn->dn_mtx dnode_diduse_space+0x184 2 0% 99% 0.00 3115 dn->dn_mtx (usermode) 1 0% 99% 0.00 920 zp->z_range_lock zfs_range_unlock+0x3b4 1 0% 99% 0.00 10391 vm map (user) _vm_map_lock+0x138 1 0% 99% 0.00 5902 tm->tm_lock (usermode) 1 0% 100% 0.00 6977 proctree fork1+0x28bc 1 0% 100% 0.00 23035 proctree killjobc+0x600 1 0% 100% 0.00 814 hdl->sa_lock sa_bulk_lookup+0x154 1 0% 100% 0.00 474 hdl->sa_lock (usermode) 1 0% 100% 0.00 230 ds->ds_lock dsl_dataset_block_kill+0x5e4 1 0% 100% 0.00 269 dp->dp_lock dsl_pool_need_dirty_delay+0x1ac 1 0% 100% 0.00 19709 ds->ds_lock dsl_dataset_block_born+0x3d4 1 0% 100% 0.00 6041 dn->dn_mtx 0xc0000000025a65f8 1 0% 100% 0.00 10759 dn->dn_mtx dnode_rele+0xbc 1 0% 100% 0.00 5935 db->db_mtx (usermode) 1 0% 100% 0.00 992 bpl->bpl_lock (usermode) ------------------------------------------------------------------------------- SX shared spin: 2 events in 1.503 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2 100% 100% 0.00 402 zf->zf_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 52085 events in 2.449 seconds (21266 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 16042 31% 31% 0.00 14935 page pv moea64_enter+0x86c 15472 30% 61% 0.00 14815 page pv (usermode) 14042 27% 87% 0.00 14156 page pv 0xc000000003142640 952 2% 89% 0.00 16698 vm active pagequeue 0xc00000000310e854 723 1% 91% 0.00 9464 vm page vm_page_change_lock+0x160 437 1% 92% 0.00 2587 vnode interlock vget+0x2f8 310 1% 92% 0.00 194904 zio_write_issue 0xc000000002d7c86c 285 1% 93% 0.00 520081 zio_write_issue (usermode) 278 1% 93% 0.00 15417 vm page vm_fault_hold+0x2724 270 1% 94% 0.00 5524 vnode interlock (usermode) 259 0% 94% 0.00 27102 dp_sync_taskq 0xc000000002d7c86c 228 0% 95% 0.00 20704 dp_sync_taskq 0xc000000002cc30c0 227 0% 95% 0.00 16378 dp_sync_taskq (usermode) 219 0% 96% 0.00 20605 vm active pagequeue (usermode) 167 0% 96% 0.00 28523 zio_write_intr (usermode) 164 0% 96% 0.00 141939 zio_write_issue 0xc000000002cc30c0 153 0% 96% 0.00 46093 vm page (usermode) 152 0% 97% 0.00 8143 vnode interlock 0xc000000002e224a0 118 0% 97% 0.00 140727 zio_write_issue taskqueue_enqueue+0x158 113 0% 97% 0.00 7378 struct mount mtx vfs_unbusy+0x15c 108 0% 97% 0.00 17262 dp_sync_taskq taskqueue_enqueue+0x158 101 0% 98% 0.00 11815 pmap moea64_enter+0x880 96 0% 98% 0.00 24701 zio_write_intr 0xc000000002d7c86c 96 0% 98% 0.00 56611 kernelpmap moea64_kextract+0x174 83 0% 98% 0.00 2701 struct mount mtx vfs_busy+0x434 82 0% 98% 0.00 15017 zio_write_intr 0xc000000002cc30c0 70 0% 98% 0.00 589094 process lock ast+0x7e0 65 0% 99% 0.00 6086 struct mount mtx (usermode) 63 0% 99% 0.00 13201 vnode interlock 0xc000000002e22484 57 0% 99% 0.00 4876 select mtxpool selrecord+0x1dc 55 0% 99% 0.00 12864 page pv moea64_enter+0x910 50 0% 99% 0.00 11309 pmap (usermode) 49 0% 99% 0.00 23537 zio_write_intr taskqueue_enqueue+0x158 45 0% 99% 0.00 7366 umtxql (usermode) 45 0% 99% 0.00 37432 kernelpmap moea64_enter+0x880 38 0% 99% 0.00 6347 vnode interlock 0xc0000000030ce5b8 33 0% 99% 0.00 208682 process lock kern_thr_exit+0x12c 27 0% 99% 0.00 34670 kernelpmap (usermode) 23 0% 99% 0.00 8609 umtxql 0xc000000002d245d4 21 0% 99% 0.00 2067 struct mount vlist mtx _vdrop+0x6f4 20 0% 100% 0.00 12294 vm active pagequeue vm_page_dequeue+0x1b4 18 0% 100% 0.00 7046 umtxql 0xc000000002d228cc 16 0% 100% 0.00 5471 select mtxpool 0xc000000002d93038 16 0% 100% 0.00 101150 process lock kern_thr_exit+0x24c 15 0% 100% 0.00 1201 struct mount vlist mtx _vhold+0x474 13 0% 100% 0.00 5956 umtxql 0xc000000002d253f8 11 0% 100% 0.00 7011 umtxql 0xc000000002d22b78 10 0% 100% 0.00 12075 vm page free queue 0xc000000003111274 10 0% 100% 0.00 9026 vm page vm_page_rename+0x224 7 0% 100% 0.00 7380 vm page 0xc0000000030ec28c 7 0% 100% 0.00 7148 umtxql 0xc000000002d25428 7 0% 100% 0.00 1076 struct mount vlist mtx (usermode) 6 0% 100% 0.00 7952 vm active pagequeue 0xc000000003114894 6 0% 100% 0.00 6982 ttymtx 0xc000000002db84b4 6 0% 100% 0.00 4513 ttymtx ttyoutq_read_uio+0x34c 6 0% 100% 0.00 363226 process lock (usermode) 6 0% 100% 0.00 38812 kernelpmap moea64_remove+0x200 5 0% 100% 0.00 564 UPVO entry uma_zalloc_arg+0x98c 5 0% 100% 0.00 731 UPVO entry (usermode) 4 0% 100% 0.00 14960 ttymtx ttydisc_write+0x388 3 0% 100% 0.00 24953 vm page free queue 0xc000000003110fb4 3 0% 100% 0.00 10540 vm page free queue (usermode) 3 0% 100% 0.00 5516 vm page vm_imgact_unmap_page+0x184 3 0% 100% 0.00 10073 umtxql 0xc000000002cc30c0 3 0% 100% 0.00 1852 UPVO entry uma_zfree_arg+0x5c8 3 0% 100% 0.00 4451 SLB table moea64_get_unique_vsid+0x2a8 3 0% 100% 0.00 11326 CAM device lock 0xc000000002524ac0 3 0% 100% 0.00 3742 CAM device lock 0xc000000002524148 2 0% 100% 0.00 8442 vnode interlock vinactive+0x25c 2 0% 100% 0.00 604 vnode interlock _vhold+0x4a4 2 0% 100% 0.00 6053 vm page vm_imgact_map_page+0x43c 2 0% 100% 0.00 7090 vm page exec_unmap_first_page+0x188 2 0% 100% 0.00 1677 umtxql 0xc000000002d2546c 2 0% 100% 0.00 1098 ttymtx 0xc000000002db7520 2 0% 100% 0.00 10435 ttymtx (usermode) 2 0% 100% 0.00 5671 ttymtx 0xc000000002db18d8 2 0% 100% 0.00 800 sellck 0xc000000002d92870 2 0% 100% 0.00 5388 select mtxpool 0xc000000002d92a68 2 0% 100% 0.00 141357 nvme qpair lock 0xc000000002a101b0 2 0% 100% 0.00 1781 ncneglh (usermode) 2 0% 100% 0.00 25179 metaslab_group_taskq (usermode) 2 0% 100% 0.00 1086 CAM device lock (usermode) 1 0% 100% 0.00 275 zio_free_issue_3 taskqueue_enqueue+0x158 1 0% 100% 0.00 495 zio_free_issue_4 0xc000000002d7c86c 1 0% 100% 0.00 500 vnode interlock 0xc000000002e09c84 1 0% 100% 0.00 263 vm page exec_map_first_page+0x7b8 1 0% 100% 0.00 3115 vm active pagequeue 0xc000000003114bec 1 0% 100% 0.00 287 vm page 0xc000000002df9ce0 1 0% 100% 0.00 12802 vm page vm_object_madvise+0x54c 1 0% 100% 0.00 164 struct mount mtx vfs_ref+0x114 1 0% 100% 0.00 11424 metaslab_group_taskq 0xc000000002d7c86c 1 0% 100% 0.00 322 metaslab_group_taskq taskqueue_enqueue+0x158 1 0% 100% 0.00 5867 dp_sync_taskq taskqueue_quiesce+0x2e8 1 0% 100% 0.00 6718 SLB table moea64_release_vsid+0x144 1 0% 100% 0.00 373 UMA Slabs uma_large_malloc_domain+0x2ac 1 0% 100% 0.00 11366 bge0 (usermode) 1 0% 100% 0.00 215 pipe mutex 0xc000000002d95074 1 0% 100% 0.00 20205 pipe mutex 0xc000000002d93828 1 0% 100% 0.00 1635 ncvn 0xc000000002e04988 1 0% 100% 0.00 985 ncvn (usermode) 1 0% 100% 0.00 1746 process lock kern_sigprocmask+0x378 1 0% 100% 0.00 5507 process lock proc_reap+0xda8 1 0% 100% 0.00 5571 process lock 0xc000000002c8e430 ------------------------------------------------------------------------------- Adaptive mutex block: 7 events in 2.449 seconds (3 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2 29% 29% 0.00 83413 page pv 0xc000000003142640 1 14% 43% 0.00 61894 zio_write_issue 0xc000000002d7c86c 1 14% 57% 0.00 46534 zio_write_issue taskqueue_enqueue+0x158 1 14% 71% 0.00 51843 kernelpmap moea64_enter+0x880 1 14% 86% 0.00 63180 kernelpmap moea64_kextract+0x174 1 14% 100% 0.00 26434 page pv (usermode) ------------------------------------------------------------------------------- Spin lock spin: 17484 events in 2.449 seconds (7139 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 9345 53% 53% 0.00 450268 turnstile chain (usermode) 6657 38% 92% 0.00 398981 turnstile chain turnstile_trywait+0x208 691 4% 95% 0.00 17080 sleepq chain sleepq_lock+0xe4 292 2% 97% 0.00 15198 sleepq chain (usermode) 223 1% 98% 0.00 86712 turnstile chain turnstile_chain_lock+0xdc 99 1% 99% 0.00 531 turnstile lock (usermode) 77 0% 99% 0.00 473 turnstile lock turnstile_trywait+0x150 6 0% 99% 0.00 23090 sched lock 55 0xc000000002d3cce4 6 0% 99% 0.00 3462 sched lock 0 0xc000000002d3cccc 4 0% 100% 0.00 5106 sched lock 5 0xc000000002d3cccc 4 0% 100% 0.00 2382 sched lock 30 0xc000000002d3cccc 3 0% 100% 0.00 7288 sched lock 31 0xc000000002d3cce4 3 0% 100% 0.00 20410 sched lock 31 0xc000000002d3cccc 3 0% 100% 0.00 6950 sched lock 15 0xc000000002d3cccc 3 0% 100% 0.00 9758 sched lock 0 sched_add+0x48c 2 0% 100% 0.00 523 turnstile lock turnstile_lookup+0xd4 2 0% 100% 0.00 435 sched lock 69 sched_add+0x48c 2 0% 100% 0.00 718 sched lock 68 0xc000000002d3cce4 2 0% 100% 0.00 502 sched lock 65 (usermode) 2 0% 100% 0.00 662 sched lock 64 (usermode) 2 0% 100% 0.00 3505 sched lock 5 sched_add+0x48c 2 0% 100% 0.00 2200 sched lock 47 0xc000000002d3cce4 2 0% 100% 0.00 12249 sched lock 43 sched_add+0x48c 2 0% 100% 0.00 1865 sched lock 36 0xc000000002d3cce4 2 0% 100% 0.00 6582 sched lock 35 0xc000000002d3cccc 2 0% 100% 0.00 281 sched lock 31 sched_add+0x48c 2 0% 100% 0.00 6577 sched lock 20 sched_add+0x48c 2 0% 100% 0.00 2413 sched lock 20 sched_switch+0x81c 2 0% 100% 0.00 1137 sched lock 17 sched_add+0x48c 2 0% 100% 0.00 6508 sched lock 10 sched_switch+0x81c 2 0% 100% 0.00 4194 sched lock 0 (usermode) 1 0% 100% 0.00 767 sched lock 70 sched_add+0x48c 1 0% 100% 0.00 438 sched lock 65 0xc000000002d3cce4 1 0% 100% 0.00 2527 sched lock 65 sched_switch+0x81c 1 0% 100% 0.00 5824 sched lock 70 sched_switch+0x81c 1 0% 100% 0.00 645 sched lock 61 sched_add+0x48c 1 0% 100% 0.00 1859 sched lock 58 (usermode) 1 0% 100% 0.00 1744 sched lock 55 (usermode) 1 0% 100% 0.00 6273 sched lock 55 0xc000000002d3cccc 1 0% 100% 0.00 225 sched lock 48 sched_add+0x48c 1 0% 100% 0.00 954 sched lock 52 (usermode) 1 0% 100% 0.00 771 sched lock 54 sched_add+0x48c 1 0% 100% 0.00 3208 sched lock 54 0xc000000002d3cce4 1 0% 100% 0.00 18830 sched lock 35 0xc000000002d3cce4 1 0% 100% 0.00 850 sched lock 32 0xc000000002d3cce4 1 0% 100% 0.00 3324 sched lock 35 sched_add+0x48c 1 0% 100% 0.00 3419 sched lock 41 0xc000000002d3cccc 1 0% 100% 0.00 795 sched lock 39 (usermode) 1 0% 100% 0.00 1861 sched lock 37 (usermode) 1 0% 100% 0.00 2564 sched lock 48 sched_switch+0xca0 1 0% 100% 0.00 412 sched lock 32 sched_add+0x48c 1 0% 100% 0.00 41564 sched lock 30 0xc000000002d3cce4 1 0% 100% 0.00 1314 sched lock 21 0xc000000002d3cce4 1 0% 100% 0.00 863 sched lock 20 0xc000000002d3cce4 1 0% 100% 0.00 795 sched lock 20 0xc000000002d3cccc 1 0% 100% 0.00 543 sched lock 24 (usermode) 1 0% 100% 0.00 1240 sched lock 3 0xc000000002d3cccc 1 0% 100% 0.00 338 sched lock 30 (usermode) 1 0% 100% 0.00 898 sched lock 16 0xc000000002d3cccc 1 0% 100% 0.00 3089 sched lock 14 0xc000000002d3cccc 1 0% 100% 0.00 2088 sched lock 2 0xc000000002d3cce4 1 0% 100% 0.00 755 sched lock 2 (usermode) 1 0% 100% 0.00 1687 sched lock 17 0xc000000002d3cccc 1 0% 100% 0.00 3560 sched lock 20 (usermode) 1 0% 100% 0.00 225 et_hw_mtx cpu_new_callout+0x370 1 0% 100% 0.00 222 callout 0xc000000002d1b164 1 0% 100% 0.00 218 sched lock 0 sched_switch+0x81c ------------------------------------------------------------------------------- Thread lock spin: 331 events in 2.449 seconds (135 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 169 51% 51% 0.00 4021 sleepq chain _thread_lock+0xf4 64 19% 70% 0.00 3304 sleepq chain (usermode) 15 5% 75% 0.00 4943 turnstile lock _thread_lock+0xf4 6 2% 77% 0.00 777005 sched lock 0 _thread_lock+0xf4 5 2% 78% 0.00 6124 sched lock 20 _thread_lock+0xf4 5 2% 80% 0.00 4037 sched lock 0 (usermode) 4 1% 81% 0.00 3440 sched lock 30 _thread_lock+0xf4 3 1% 82% 0.00 8285 sched lock 5 _thread_lock+0xf4 3 1% 83% 0.00 4657 sched lock 17 _thread_lock+0xf4 2 1% 83% 0.00 18901 sched lock 69 (usermode) 2 1% 84% 0.00 9474 sched lock 68 _thread_lock+0xf4 2 1% 85% 0.00 612 sched lock 67 _thread_lock+0xf4 2 1% 85% 0.00 1788 sched lock 65 (usermode) 2 1% 86% 0.00 3698 sched lock 63 _thread_lock+0xf4 2 1% 86% 0.00 1041 sched lock 60 _thread_lock+0xf4 2 1% 87% 0.00 364 sched lock 57 (usermode) 2 1% 88% 0.00 780 sched lock 35 (usermode) 2 1% 88% 0.00 18004 sched lock 32 (usermode) 2 1% 89% 0.00 39126 sched lock 31 _thread_lock+0xf4 2 1% 89% 0.00 4503 sched lock 31 (usermode) 2 1% 90% 0.00 6078 sched lock 27 _thread_lock+0xf4 2 1% 91% 0.00 10516 sched lock 21 _thread_lock+0xf4 2 1% 91% 0.00 13273 sched lock 20 (usermode) 2 1% 92% 0.00 10158 sched lock 15 _thread_lock+0xf4 1 0% 92% 0.00 11435 turnstile lock (usermode) 1 0% 92% 0.00 11858 sched lock 70 (usermode) 1 0% 93% 0.00 539 sched lock 64 (usermode) 1 0% 93% 0.00 6859 sched lock 61 (usermode) 1 0% 93% 0.00 1508 sched lock 5 (usermode) 1 0% 94% 0.00 1896 sched lock 52 _thread_lock+0xf4 1 0% 94% 0.00 2705 sched lock 55 (usermode) 1 0% 94% 0.00 5138 sched lock 55 _thread_lock+0xf4 1 0% 95% 0.00 1576 sched lock 48 _thread_lock+0xf4 1 0% 95% 0.00 5517 sched lock 48 (usermode) 1 0% 95% 0.00 546 sched lock 47 _thread_lock+0xf4 1 0% 95% 0.00 6299 sched lock 43 (usermode) 1 0% 96% 0.00 572 sched lock 44 (usermode) 1 0% 96% 0.00 603 sched lock 46 _thread_lock+0xf4 1 0% 96% 0.00 392 sched lock 38 _thread_lock+0xf4 1 0% 97% 0.00 286 sched lock 37 (usermode) 1 0% 97% 0.00 11060 sched lock 39 (usermode) 1 0% 97% 0.00 11428 sched lock 40 _thread_lock+0xf4 1 0% 98% 0.00 1058 sched lock 42 _thread_lock+0xf4 1 0% 98% 0.00 5556 sched lock 25 _thread_lock+0xf4 1 0% 98% 0.00 1068 sched lock 22 (usermode) 1 0% 98% 0.00 633 sched lock 21 (usermode) 1 0% 99% 0.00 397 sched lock 29 (usermode) 1 0% 99% 0.00 451 sched lock 33 (usermode) 1 0% 99% 0.00 390 sched lock 12 (usermode) 1 0% 100% 0.00 342 sched lock 17 (usermode) 1 0% 100% 0.00 482 sched lock 10 _thread_lock+0xf4 ------------------------------------------------------------------------------- R/W writer blocked by writer: 550 events in 2.449 seconds (225 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 127 23% 23% 0.00 72980859 vm object vm_fault_hold+0x1b28 98 18% 41% 0.00 64621427 vm object vm_map_insert+0x598 77 14% 55% 0.00 84944804 vm object vm_object_reference+0x134 76 14% 69% 0.00 62279361 vm object vm_object_deallocate+0x800 47 9% 77% 0.00 84796310 vm object vm_object_shadow+0x390 37 7% 84% 0.00 72338943 vm object vm_object_shadow+0x378 28 5% 89% 0.00 54767014 vm object 0xc0000000030f8558 26 5% 94% 0.00 71781213 vm object (usermode) 22 4% 98% 0.00 62232865 vm object vm_imgact_map_page+0x3f0 9 2% 99% 0.00 78437043 vm object exec_map_first_page+0x780 2 0% 100% 0.00 66754603 vm object vm_object_deallocate+0x814 1 0% 100% 0.00 403429 kernel vm object 0xc0000000030f09c0 ------------------------------------------------------------------------------- R/W writer blocked by readers: 926 events in 2.449 seconds (378 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 406 44% 44% 0.00 55154869 vm object vm_fault_hold+0x1b28 107 12% 55% 0.00 71945889 vm object vm_object_deallocate+0x800 83 9% 64% 0.00 54324659 vm object vnode_create_vobject+0x18c 76 8% 73% 0.00 37081229 vm object (usermode) 69 7% 80% 0.00 44460029 vm object vm_object_deallocate+0x814 65 7% 87% 0.00 51027779 vm object 0xc0000000030f8558 50 5% 92% 0.00 39554754 vm object vm_object_reference+0x134 28 3% 95% 0.00 70395645 vm object vm_object_shadow+0x378 19 2% 98% 0.00 43527129 vm object vm_imgact_map_page+0x3f0 18 2% 99% 0.00 64409322 vm object vm_map_insert+0x598 3 0% 100% 0.00 88605544 vm object vm_object_shadow+0x390 2 0% 100% 0.00 8998992 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader blocked by writer: 174 events in 2.449 seconds (71 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 124 71% 71% 0.00 36527288 vm object __rw_rlock_int+0x9c 50 29% 100% 0.00 24343729 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 30 events in 2.449 seconds (12 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 24 80% 80% 0.00 23554932 vm object __rw_rlock_int+0x9c 6 20% 100% 0.00 5761939 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 952 events in 2.449 seconds (389 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 186 20% 20% 0.00 4982056 vm object vm_fault_hold+0x1b28 133 14% 34% 0.00 4457766 vm object vm_map_insert+0x598 114 12% 45% 0.00 6315921 vm object vm_object_reference+0x134 112 12% 57% 0.00 4410365 vm object vm_object_deallocate+0x800 83 9% 66% 0.00 5210590 vm object (usermode) 67 7% 73% 0.00 8603 vm object vm_fault_hold+0x2700 61 6% 79% 0.00 7074571 vm object vm_object_shadow+0x390 50 5% 85% 0.00 5311289 vm object vm_object_shadow+0x378 39 4% 89% 0.00 3776572 vm object vm_imgact_map_page+0x3f0 38 4% 93% 0.00 5256832 vm object 0xc0000000030f8558 17 2% 95% 0.00 143825 kernel vm object kmem_back_domain+0x340 14 1% 96% 0.00 4008621 vm object exec_map_first_page+0x780 10 1% 97% 0.00 255787 kernel vm object 0xc0000000030f09c0 5 1% 98% 0.00 13836 Per-Filesystem Softdep Lock (usermode) 4 0% 98% 0.00 636402 kernel vm object (usermode) 3 0% 98% 0.00 9612 Per-Filesystem Softdep Lock 0xc0000000030adb58 2 0% 99% 0.00 14742 vm object 0xc00000000311f65c 2 0% 99% 0.00 5613515 vm object vm_object_deallocate+0x814 2 0% 99% 0.00 7787 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 2 0% 99% 0.00 4242 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 1 0% 99% 0.00 22530 Per-Filesystem Softdep Lock softdep_slowdown+0x374 1 0% 99% 0.00 1720 Per-Filesystem Softdep Lock 0xc000000003095904 1 0% 99% 0.00 1780 Per-Filesystem Softdep Lock 0xc000000002cec288 1 0% 100% 0.00 7447 Per-Filesystem Softdep Lock softdep_freefile+0x944 1 0% 100% 0.00 20060 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c 1 0% 100% 0.00 1147 Per-Filesystem Softdep Lock 0xc00000000309c6f4 1 0% 100% 0.00 256 Per-Filesystem Softdep Lock softdep_setup_directory_add+0xc2c 1 0% 100% 0.00 7428 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c ------------------------------------------------------------------------------- R/W writer spin on readers: 1015 events in 2.449 seconds (414 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 415 41% 41% 0.00 5290216 vm object vm_fault_hold+0x1b28 118 12% 53% 0.00 7212041 vm object vm_object_deallocate+0x800 84 8% 61% 0.00 4457959 vm object vnode_create_vobject+0x18c 81 8% 69% 0.00 5955759 vm object (usermode) 69 7% 76% 0.00 4418020 vm object vm_object_deallocate+0x814 66 7% 82% 0.00 5662500 vm object 0xc0000000030f8558 54 5% 87% 0.00 5112677 vm object vm_object_reference+0x134 33 3% 91% 0.00 19521 vm object vm_fault_hold+0x2700 28 3% 93% 0.00 7077881 vm object vm_object_shadow+0x378 23 2% 96% 0.00 3905218 vm object vm_map_insert+0x598 20 2% 98% 0.00 3524316 vm object vm_imgact_map_page+0x3f0 15 1% 99% 0.00 17477 vm object vm_fault_hold+0x1298 4 0% 100% 0.00 298472 vm object exec_map_first_page+0x780 3 0% 100% 0.00 6077247 vm object vm_object_shadow+0x390 2 0% 100% 0.00 34542 vm object 0xc00000000311f65c ------------------------------------------------------------------------------- R/W reader spin on writer: 323 events in 2.449 seconds (132 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 239 74% 74% 0.00 1094041 vm object __rw_rlock_int+0x9c 84 26% 100% 0.00 810878 vm object (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 30 events in 2.449 seconds (12 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 26 87% 87% 0.00 1377751 vm object __rw_rlock_int+0x9c 4 13% 100% 0.00 779620 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 1 events in 2.449 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 25604 vq->vq_lock (usermode) ------------------------------------------------------------------------------- SX exclusive spin: 2075 events in 2.449 seconds (847 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 362 17% 17% 0.00 4964 rrl->rr_lock 0xc00000000260abbc 256 12% 30% 0.00 5113 rrl->rr_lock rrw_exit+0x310 233 11% 41% 0.00 26542 vm map (user) (usermode) 160 8% 49% 0.00 203524 vq->vq_lock (usermode) 129 6% 55% 0.00 77396 tm->tm_lock trim_map_free+0x19c 103 5% 60% 0.00 230905 vq->vq_lock vdev_queue_io+0x268 76 4% 64% 0.00 5126 rrl->rr_lock (usermode) 67 3% 67% 0.00 703825 os->os_userused_lock 0xc0000000025b7cdc 58 3% 70% 0.00 798 zp->z_acl_lock 0xc0000000026790e0 52 3% 72% 0.00 13311 proctree kern_wait6+0x3e8 42 2% 74% 0.00 1813 zp->z_acl_lock (usermode) 40 2% 76% 0.00 25152 vm map (user) _vm_map_lock+0x138 35 2% 78% 0.00 713309 os->os_userused_lock 0xc0000000025b7e74 28 1% 79% 0.00 12715 scl->scl_lock spa_config_exit+0x84 27 1% 80% 0.00 20067 vq->vq_lock vdev_queue_io_done+0x278 27 1% 82% 0.00 30384 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x268 26 1% 83% 0.00 20439 mc->mc_lock metaslab_class_throttle_unreserve+0x154 24 1% 84% 0.00 88114 vq->vq_lock 0xc0000000026563c8 20 1% 85% 0.00 15781 zio->io_lock 0xc0000000026bfad8 20 1% 86% 0.00 1015665 os->os_userused_lock (usermode) 20 1% 87% 0.00 11250 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 17 1% 88% 0.00 12447 zio->io_lock 0xc0000000026c1ac0 17 1% 89% 0.00 30189 vm map (user) _vm_map_lock_upgrade+0x180 17 1% 89% 0.00 3597 ms->ms_lock metaslab_alloc_dva+0x1138 16 1% 90% 0.00 15314 spa->spa_alloc_locks[i] 0xc0000000026bd584 16 1% 91% 0.00 13403 scl->scl_lock spa_config_enter+0x140 16 1% 92% 0.00 8590 mg->mg_lock metaslab_alloc_dva+0x7d4 15 1% 92% 0.00 13320 mg->mg_lock metaslab_alloc_dva+0x3a8 13 1% 93% 0.00 9601 zio->io_lock (usermode) 13 1% 94% 0.00 87817 spa->spa_alloc_locks[i] (usermode) 12 1% 94% 0.00 27668 zio->io_lock zio_add_child+0x274 11 1% 95% 0.00 5076 dd->dd_lock (usermode) 10 0% 95% 0.00 6975 dd->dd_lock dsl_dir_diduse_space+0x270 10 0% 96% 0.00 12512 dd->dd_lock dsl_dir_tempreserve_clear+0x88 10 0% 96% 0.00 11565 db->db_mtx (usermode) 8 0% 97% 0.00 6010 vq->vq_lock vdev_queue_io_done+0x12c 8 0% 97% 0.00 11785 mg->mg_lock (usermode) 8 0% 97% 0.00 9918 db->db_mtx dmu_buf_will_dirty+0x2e8 7 0% 98% 0.00 15736 ds->ds_lock dsl_dataset_block_born+0x3d4 6 0% 98% 0.00 6940 ms->ms_lock (usermode) 6 0% 98% 0.00 343 dd->dd_lock 0xc0000000025e7c24 3 0% 99% 0.00 27646 proctree fork1+0x28bc 3 0% 99% 0.00 1721 hdl->sa_lock sa_bulk_lookup+0x154 3 0% 99% 0.00 943 h->hash_mutexes[i] dbuf_find+0x114 3 0% 99% 0.00 195 dp->dp_lock dsl_pool_undirty_space+0x18c 2 0% 99% 0.00 160132 tm->tm_lock (usermode) 2 0% 99% 0.00 24607 proctree exit1+0x1aec 2 0% 99% 0.00 1403 hdl->sa_lock (usermode) 2 0% 99% 0.00 958 bpl->bpl_lock bplist_append+0x158 1 0% 99% 0.00 179 tm->tm_lock trim_map_write_done+0x188 1 0% 99% 0.00 223 scl->scl_lock (usermode) 1 0% 99% 0.00 175 mc->mc_lock (usermode) 1 0% 100% 0.00 422 hdl->sa_lock sa_lookup_uio+0x198 1 0% 100% 0.00 975 ms->ms_lock metaslab_free_concrete+0x2e8 1 0% 100% 0.00 2913 ds->ds_lock dsl_dataset_check_quota+0x2f0 1 0% 100% 0.00 332 dp->dp_lock dsl_pool_need_dirty_delay+0x1ac 1 0% 100% 0.00 10235 ds->ds_lock dsl_dataset_block_kill+0x5e4 1 0% 100% 0.00 11542 dn->dn_mtx (usermode) 1 0% 100% 0.00 41200 dd->dd_lock dsl_dir_willuse_space+0x204 1 0% 100% 0.00 334 dn->dn_mtx dbuf_dirty+0xce8 1 0% 100% 0.00 156 dp->dp_lock (usermode) 1 0% 100% 0.00 11448 bpl->bpl_lock (usermode) 1 0% 100% 0.00 252 as->as_lock aggsum_add+0x360 ------------------------------------------------------------------------------- SX shared spin: 284 events in 2.449 seconds (116 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 240 85% 85% 0.00 21936 vm map (user) _sx_slock_int+0xa0 39 14% 98% 0.00 31479 vm map (user) (usermode) 3 1% 99% 0.00 379 zf->zf_rwlock _sx_slock_int+0xa0 2 1% 100% 0.00 413 zf->zf_rwlock (usermode) ------------------------------------------------------------------------------- root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 74094 events in 2.265 seconds (32719 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 24493 33% 33% 0.00 16377 page pv 0xc000000003142640 21580 29% 62% 0.00 16778 page pv (usermode) 16556 22% 85% 0.00 15606 page pv moea64_enter+0x86c 3189 4% 89% 0.00 10627 vm active pagequeue 0xc00000000310e854 2782 4% 93% 0.00 14479 vm page vm_page_change_lock+0x160 842 1% 94% 0.00 11128 vm active pagequeue (usermode) 463 1% 94% 0.00 19381 vm page vm_fault_hold+0x2724 362 0% 95% 0.00 17556 vm page (usermode) 339 0% 95% 0.00 258714 zio_write_issue 0xc000000002d7c86c 251 0% 96% 0.00 20660 dp_sync_taskq (usermode) 244 0% 96% 0.00 39597 dp_sync_taskq 0xc000000002d7c86c 238 0% 96% 0.00 2793 vnode interlock vget+0x2f8 234 0% 97% 0.00 344631 zio_write_issue (usermode) 221 0% 97% 0.00 238068 zio_write_issue 0xc000000002cc30c0 207 0% 97% 0.00 336375 zio_write_issue taskqueue_enqueue+0x158 174 0% 97% 0.00 17916 dp_sync_taskq 0xc000000002cc30c0 148 0% 98% 0.00 27020 zio_write_intr (usermode) 133 0% 98% 0.00 15401 pmap moea64_enter+0x880 122 0% 98% 0.00 71370 kernelpmap moea64_kextract+0x174 106 0% 98% 0.00 5070 select mtxpool selrecord+0x1dc 99 0% 98% 0.00 6920 vnode interlock (usermode) 94 0% 98% 0.00 20175 zio_write_intr 0xc000000002d7c86c 94 0% 98% 0.00 23076 zio_write_intr 0xc000000002cc30c0 93 0% 99% 0.00 7657 vnode interlock 0xc000000002e224a0 82 0% 99% 0.00 15459 dp_sync_taskq taskqueue_enqueue+0x158 79 0% 99% 0.00 6633 page pv moea64_enter+0x910 63 0% 99% 0.00 16313 zio_write_intr taskqueue_enqueue+0x158 56 0% 99% 0.00 20861 pmap (usermode) 47 0% 99% 0.00 7282 struct mount mtx vfs_unbusy+0x15c 46 0% 99% 0.00 6979 select mtxpool 0xc000000002d93038 46 0% 99% 0.00 587238 process lock ast+0x7e0 39 0% 99% 0.00 65650 kernelpmap (usermode) 36 0% 99% 0.00 11974 vnode interlock 0xc000000002e22484 35 0% 99% 0.00 14349 umtxql (usermode) 33 0% 99% 0.00 69725 kernelpmap moea64_enter+0x880 30 0% 99% 0.00 15392 vm page free queue 0xc000000003111274 30 0% 99% 0.00 3595 vm active pagequeue vm_page_dequeue+0x1b4 29 0% 99% 0.00 761006 process lock (usermode) 27 0% 100% 0.00 309831 process lock kern_thr_exit+0x12c 26 0% 100% 0.00 2143 struct mount mtx vfs_busy+0x434 24 0% 100% 0.00 2664 struct mount mtx (usermode) 22 0% 100% 0.00 26280 umtxql 0xc000000002d245d4 20 0% 100% 0.00 6342 umtxql 0xc000000002d228cc 20 0% 100% 0.00 142315 process lock kern_thr_exit+0x24c 17 0% 100% 0.00 21418 vm page 0xc0000000030ec28c 16 0% 100% 0.00 5103 pipe mutex 0xc000000002d9456c 12 0% 100% 0.00 7455 vnode interlock 0xc0000000030ce5b8 11 0% 100% 0.00 885 struct mount vlist mtx _vdrop+0x6f4 10 0% 100% 0.00 410 pipe mutex 0xc000000002d95914 10 0% 100% 0.00 1184 UPVO entry uma_zfree_arg+0x5c8 9 0% 100% 0.00 133483 nvme qpair lock 0xc000000002a101b0 9 0% 100% 0.00 611 UPVO entry (usermode) 7 0% 100% 0.00 861 struct mount vlist mtx _vhold+0x474 6 0% 100% 0.00 10458 umtxql 0xc000000002d253f8 6 0% 100% 0.00 8007 umtxql 0xc000000002d22b78 6 0% 100% 0.00 4279 ttymtx ttyoutq_read_uio+0x34c 6 0% 100% 0.00 8151 ttymtx (usermode) 6 0% 100% 0.00 31655 kernelpmap moea64_remove+0x200 6 0% 100% 0.00 567 UPVO entry uma_zalloc_arg+0x98c 5 0% 100% 0.00 1273 vm page vm_page_rename+0x224 5 0% 100% 0.00 4915 ttymtx 0xc000000002db84b4 5 0% 100% 0.00 4811 select mtxpool 0xc000000002d92a68 5 0% 100% 0.00 5859 select mtxpool (usermode) 5 0% 100% 0.00 205 pipe mutex 0xc000000002d95074 4 0% 100% 0.00 7622 vm page free queue 0xc000000003110fb4 4 0% 100% 0.00 526 vm page vm_imgact_map_page+0x43c 4 0% 100% 0.00 859 ttymtx 0xc000000002db7520 4 0% 100% 0.00 6015 struct mount vlist mtx (usermode) 4 0% 100% 0.00 732 pipe mutex 0xc000000002d93a60 4 0% 100% 0.00 1254 pipe mutex 0xc000000002d93828 3 0% 100% 0.00 1575 vm page vm_imgact_unmap_page+0x184 3 0% 100% 0.00 2427 vm page exec_map_first_page+0x7b8 3 0% 100% 0.00 7489 ttymtx 0xc000000002db18d8 3 0% 100% 0.00 5555 struct mount mtx vfs_rel+0x128 3 0% 100% 0.00 4928 CAM device lock 0xc000000002524148 2 0% 100% 0.00 552 vnode interlock _vhold+0x4a4 2 0% 100% 0.00 168 vnode interlock 0xc000000002e09c84 2 0% 100% 0.00 636 vm pgcache uma_zfree_arg+0x5c8 2 0% 100% 0.00 11820 umtxql 0xc000000002d25428 2 0% 100% 0.00 279 struct mount mtx vfs_ref+0x114 2 0% 100% 0.00 78453 page pv moea64_page_wired_mappings+0x1b8 2 0% 100% 0.00 25691 metaslab_group_taskq (usermode) 2 0% 100% 0.00 1628 kernel arena domain vmem_xfree+0x39c 2 0% 100% 0.00 2255 CAM device lock 0xc000000002524ac0 1 0% 100% 0.00 1810 zio_free_issue_3 (usermode) 1 0% 100% 0.00 568 zio_free_issue_4 0xc000000002cc30c0 1 0% 100% 0.00 14079 vnode interlock 0xc0000000030cdfdc 1 0% 100% 0.00 754 zio_free_issue_1 0xc000000002cc30c0 1 0% 100% 0.00 22359 vnode interlock vinactive+0x25c 1 0% 100% 0.00 154 vm page exec_unmap_first_page+0x188 1 0% 100% 0.00 1181 vm page 0xc000000002df9ce0 1 0% 100% 0.00 154 struct mount mtx vn_finished_write+0x174 1 0% 100% 0.00 19662 struct mount mtx vn_start_write+0x1a0 1 0% 100% 0.00 21252 struct mount mtx vn_finished_secondary_write+0x174 1 0% 100% 0.00 10410 ttymtx ttydisc_write+0x388 1 0% 100% 0.00 22648 umtxql 0xc000000002d2546c 1 0% 100% 0.00 11105 nvme qpair lock nvme_qpair_submit_request+0x120 1 0% 100% 0.00 10046 CAM device lock (usermode) 1 0% 100% 0.00 324 bge0 (usermode) 1 0% 100% 0.00 10333 dp_sync_taskq taskqueue_quiesce+0x2e8 1 0% 100% 0.00 383 process lock fdescfree+0x6b4 1 0% 100% 0.00 23142 pmap moea64_is_prefaultable+0x13c 1 0% 100% 0.00 165 pipe mutex (usermode) 1 0% 100% 0.00 297 so_snd (usermode) 1 0% 100% 0.00 697 sellck 0xc000000002d92870 ------------------------------------------------------------------------------- Adaptive mutex block: 96 events in 2.265 seconds (42 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 28 29% 29% 0.00 1639528 zio_write_issue (usermode) 24 25% 54% 0.00 1067182 zio_write_issue 0xc000000002cc30c0 10 10% 65% 0.00 1589920 vm page vm_page_change_lock+0x160 8 8% 73% 0.00 555158 page pv moea64_enter+0x86c 6 6% 79% 0.00 2199666 zio_write_issue taskqueue_enqueue+0x158 4 4% 83% 0.00 1146493 page pv (usermode) 3 3% 86% 0.00 2877171 vm page vm_fault_hold+0x2724 2 2% 89% 0.00 2334738 zio_write_issue 0xc000000002d7c86c 2 2% 91% 0.00 1442033 vnode interlock vget+0x2f8 2 2% 93% 0.00 2005764 page pv 0xc000000003142640 2 2% 95% 0.00 3218500 vm page (usermode) 2 2% 97% 0.00 179066 nvme qpair lock 0xc000000002a101b0 1 1% 98% 0.00 48327 kernelpmap moea64_enter+0x880 1 1% 99% 0.00 41505 kernelpmap moea64_kextract+0x174 1 1% 100% 0.00 31561 kernelpmap moea64_remove+0x200 ------------------------------------------------------------------------------- Spin lock spin: 17355 events in 2.265 seconds (7664 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 9234 53% 53% 0.00 462671 turnstile chain (usermode) 6760 39% 92% 0.00 378678 turnstile chain turnstile_trywait+0x208 561 3% 95% 0.00 20461 sleepq chain sleepq_lock+0xe4 250 1% 97% 0.00 30812 sleepq chain (usermode) 218 1% 98% 0.00 167501 turnstile chain turnstile_chain_lock+0xdc 116 1% 99% 0.00 1314 turnstile lock turnstile_trywait+0x150 102 1% 99% 0.00 475 turnstile lock (usermode) 8 0% 99% 0.00 77033 sched lock 30 0xc000000002d3cce4 6 0% 99% 0.00 13846 sched lock 70 0xc000000002d3cce4 5 0% 99% 0.00 8450 sched lock 10 0xc000000002d3cccc 4 0% 99% 0.00 16566 sched lock 5 0xc000000002d3cccc 4 0% 99% 0.00 36294 sched lock 30 0xc000000002d3cccc 4 0% 100% 0.00 6782 sched lock 0 0xc000000002d3cccc 3 0% 100% 0.00 12882 turnstile lock turnstile_lookup+0xd4 3 0% 100% 0.00 9662 sched lock 40 0xc000000002d3cce4 2 0% 100% 0.00 1614 sched lock 64 sched_add+0x48c 2 0% 100% 0.00 2080 sched lock 53 0xc000000002d3cccc 2 0% 100% 0.00 32622 sched lock 53 (usermode) 2 0% 100% 0.00 5472 sched lock 5 (usermode) 2 0% 100% 0.00 1948 sched lock 46 sched_add+0x48c 2 0% 100% 0.00 1609 sched lock 45 0xc000000002d3cce4 2 0% 100% 0.00 1602 sched lock 35 0xc000000002d3cce4 2 0% 100% 0.00 1523 sched lock 17 0xc000000002d3cccc 2 0% 100% 0.00 22723 sched lock 15 0xc000000002d3cce4 2 0% 100% 0.00 12525 sched lock 15 (usermode) 2 0% 100% 0.00 5268 sched lock 12 (usermode) 1 0% 100% 0.00 414 sched lock 71 sched_add+0x48c 1 0% 100% 0.00 751 sched lock 8 0xc000000002d3cce4 1 0% 100% 0.00 243 sched lock 7 sched_switch+0x81c 1 0% 100% 0.00 24401 sched lock 7 (usermode) 1 0% 100% 0.00 2537 sched lock 69 sched_add+0x48c 1 0% 100% 0.00 1797 sched lock 7 sched_add+0x48c 1 0% 100% 0.00 2858 sched lock 70 sched_switch+0x81c 1 0% 100% 0.00 12682 sched lock 61 (usermode) 1 0% 100% 0.00 1232 sched lock 60 0xc000000002d3cce4 1 0% 100% 0.00 246 sched lock 60 sched_add+0x48c 1 0% 100% 0.00 440 sched lock 55 0xc000000002d3cce4 1 0% 100% 0.00 1064 sched lock 56 sched_add+0x48c 1 0% 100% 0.00 1015 sched lock 57 sched_add+0x48c 1 0% 100% 0.00 1879 sched lock 62 sched_add+0x48c 1 0% 100% 0.00 219 sched lock 61 0xc000000002d3cce4 1 0% 100% 0.00 1106 sched lock 64 0xc000000002d3cce4 1 0% 100% 0.00 1094 sched lock 65 0xc000000002d3cce4 1 0% 100% 0.00 285 sched lock 67 0xc000000002d3cce4 1 0% 100% 0.00 1164 sched lock 41 0xc000000002d3cce4 1 0% 100% 0.00 2109 sched lock 40 (usermode) 1 0% 100% 0.00 659 sched lock 35 0xc000000002d3cccc 1 0% 100% 0.00 166 sched lock 4 sched_add+0x48c 1 0% 100% 0.00 1006 sched lock 30 sched_switch+0x81c 1 0% 100% 0.00 340 sched lock 31 (usermode) 1 0% 100% 0.00 1154 sched lock 33 0xc000000002d3cccc 1 0% 100% 0.00 12668 sched lock 35 (usermode) 1 0% 100% 0.00 2640 sched lock 5 sched_add+0x48c 1 0% 100% 0.00 1174 sched lock 45 (usermode) 1 0% 100% 0.00 2002 sched lock 45 0xc000000002d3cccc 1 0% 100% 0.00 314 sched lock 51 (usermode) 1 0% 100% 0.00 1713 sched lock 50 0xc000000002d3cccc 1 0% 100% 0.00 531 sched lock 52 sched_add+0x48c 1 0% 100% 0.00 1212 sched lock 53 sched_add+0x48c 1 0% 100% 0.00 1911 sched lock 10 (usermode) 1 0% 100% 0.00 1640 sched lock 0 (usermode) 1 0% 100% 0.00 6673 sched lock 17 0xc000000002d3cce4 1 0% 100% 0.00 670 sched lock 16 0xc000000002d3cce4 1 0% 100% 0.00 2996 sched lock 15 0xc000000002d3cccc 1 0% 100% 0.00 883 sched lock 16 (usermode) 1 0% 100% 0.00 562 sched lock 12 sched_add+0x48c 1 0% 100% 0.00 39888 sched lock 12 0xc000000002d3cccc 1 0% 100% 0.00 297 sched lock 15 sched_add+0x48c 1 0% 100% 0.00 3419 sched lock 22 (usermode) 1 0% 100% 0.00 1838 sched lock 2 0xc000000002d3cccc 1 0% 100% 0.00 6333 sched lock 2 sched_add+0x48c 1 0% 100% 0.00 1242 sched lock 18 (usermode) 1 0% 100% 0.00 198 sched lock 18 0xc000000002d3cce4 1 0% 100% 0.00 1178 sched lock 19 0xc000000002d3cccc 1 0% 100% 0.00 1498 sched lock 25 0xc000000002d3cccc 1 0% 100% 0.00 3550 sched lock 24 sched_add+0x48c 1 0% 100% 0.00 3347 sched lock 22 0xc000000002d3cce4 1 0% 100% 0.00 572 sched lock 26 0xc000000002d3cccc 1 0% 100% 0.00 494 sched lock 26 sched_add+0x48c 1 0% 100% 0.00 39953 sched lock 3 sched_add+0x48c 1 0% 100% 0.00 699 sched lock 3 0xc000000002d3cccc ------------------------------------------------------------------------------- Thread lock spin: 273 events in 2.265 seconds (121 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 121 44% 44% 0.00 3570 sleepq chain _thread_lock+0xf4 50 18% 63% 0.00 4096 sleepq chain (usermode) 23 8% 71% 0.00 6363 turnstile lock _thread_lock+0xf4 6 2% 73% 0.00 50146 sched lock 70 _thread_lock+0xf4 5 2% 75% 0.00 4826 sched lock 15 _thread_lock+0xf4 4 1% 77% 0.00 8600 sched lock 30 _thread_lock+0xf4 3 1% 78% 0.00 7454 sched lock 8 (usermode) 3 1% 79% 0.00 5481 sched lock 7 _thread_lock+0xf4 3 1% 80% 0.00 7796 sched lock 37 (usermode) 3 1% 81% 0.00 13501 sched lock 30 (usermode) 3 1% 82% 0.00 2644 sched lock 0 _thread_lock+0xf4 2 1% 83% 0.00 6342 sched lock 70 (usermode) 2 1% 84% 0.00 10900 sched lock 69 _thread_lock+0xf4 2 1% 84% 0.00 11624 sched lock 62 _thread_lock+0xf4 2 1% 85% 0.00 521 sched lock 56 _thread_lock+0xf4 2 1% 86% 0.00 5313 sched lock 53 (usermode) 2 1% 86% 0.00 11518 sched lock 49 _thread_lock+0xf4 2 1% 87% 0.00 10603 sched lock 36 _thread_lock+0xf4 2 1% 88% 0.00 5665 sched lock 34 _thread_lock+0xf4 2 1% 89% 0.00 18133 sched lock 26 (usermode) 2 1% 89% 0.00 7339 sched lock 15 (usermode) 2 1% 90% 0.00 2282 sched lock 0 (usermode) 1 0% 90% 0.00 1833 turnstile lock (usermode) 1 0% 91% 0.00 79337 sched lock 8 _thread_lock+0xf4 1 0% 91% 0.00 777 sched lock 71 _thread_lock+0xf4 1 0% 92% 0.00 5847 sched lock 71 (usermode) 1 0% 92% 0.00 2039 sched lock 61 _thread_lock+0xf4 1 0% 92% 0.00 6021 sched lock 61 (usermode) 1 0% 93% 0.00 696 sched lock 69 (usermode) 1 0% 93% 0.00 609 sched lock 52 (usermode) 1 0% 93% 0.00 21042 sched lock 5 _thread_lock+0xf4 1 0% 94% 0.00 3203 sched lock 5 (usermode) 1 0% 94% 0.00 2038 sched lock 46 (usermode) 1 0% 95% 0.00 2324 sched lock 46 _thread_lock+0xf4 1 0% 95% 0.00 3113 sched lock 55 _thread_lock+0xf4 1 0% 95% 0.00 528 sched lock 57 (usermode) 1 0% 96% 0.00 322 sched lock 6 _thread_lock+0xf4 1 0% 96% 0.00 3718 sched lock 60 (usermode) 1 0% 96% 0.00 6310 sched lock 22 _thread_lock+0xf4 1 0% 97% 0.00 1379 sched lock 22 (usermode) 1 0% 97% 0.00 10604 sched lock 12 (usermode) 1 0% 97% 0.00 10235 sched lock 12 _thread_lock+0xf4 1 0% 98% 0.00 22579 sched lock 10 _thread_lock+0xf4 1 0% 98% 0.00 411 sched lock 31 _thread_lock+0xf4 1 0% 99% 0.00 877 sched lock 31 (usermode) 1 0% 99% 0.00 45754 sched lock 26 _thread_lock+0xf4 1 0% 99% 0.00 1902 sched lock 37 _thread_lock+0xf4 1 0% 100% 0.00 1285 sched lock 38 _thread_lock+0xf4 1 0% 100% 0.00 1084 sched lock 45 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 586 events in 2.265 seconds (259 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 115 20% 20% 0.00 67827537 vm object vm_object_deallocate+0x800 106 18% 38% 0.00 67111338 vm object vm_fault_hold+0x1b28 95 16% 54% 0.00 75824644 vm object vm_map_insert+0x598 74 13% 67% 0.00 64072353 vm object vm_object_reference+0x134 46 8% 74% 0.00 64613186 vm object (usermode) 42 7% 82% 0.00 47898796 vm object 0xc0000000030f8558 39 7% 88% 0.00 94092694 vm object vm_object_shadow+0x390 30 5% 93% 0.00 81758743 vm object vm_object_shadow+0x378 19 3% 97% 0.00 70392213 vm object exec_map_first_page+0x780 12 2% 99% 0.00 127825114 vm object vm_imgact_map_page+0x3f0 6 1% 100% 0.00 45553169 vm object vm_object_deallocate+0x814 1 0% 100% 0.00 208831047 vm object vnode_create_vobject+0x18c 1 0% 100% 0.00 441969 kernel vm object kmem_back_domain+0x340 ------------------------------------------------------------------------------- R/W writer blocked by readers: 1028 events in 2.265 seconds (454 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 358 35% 35% 0.00 60701254 vm object vm_fault_hold+0x1b28 127 12% 47% 0.00 47157056 vm object vm_object_deallocate+0x800 120 12% 59% 0.00 46625294 vm object (usermode) 103 10% 69% 0.00 52317975 vm object vm_object_deallocate+0x814 95 9% 78% 0.00 55423253 vm object 0xc0000000030f8558 90 9% 87% 0.00 63664308 vm object vnode_create_vobject+0x18c 54 5% 92% 0.00 79299225 vm object vm_object_reference+0x134 31 3% 95% 0.00 55857291 vm object vm_imgact_map_page+0x3f0 25 2% 98% 0.00 92201717 vm object vm_map_insert+0x598 17 2% 99% 0.00 80531575 vm object vm_object_shadow+0x378 7 1% 100% 0.00 20351102 vm object vm_object_shadow+0x390 1 0% 100% 0.00 91342506 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader blocked by writer: 225 events in 2.265 seconds (99 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 157 70% 70% 0.00 35269134 vm object __rw_rlock_int+0x9c 68 30% 100% 0.00 33012286 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 28 events in 2.265 seconds (12 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 28 100% 100% 0.00 24513232 vm object __rw_rlock_int+0x9c ------------------------------------------------------------------------------- R/W writer spin on writer: 1109 events in 2.265 seconds (490 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 197 18% 18% 0.00 4047437 vm object vm_object_deallocate+0x800 156 14% 32% 0.00 4292234 vm object vm_fault_hold+0x1b28 141 13% 45% 0.00 5093733 vm object vm_map_insert+0x598 122 11% 56% 0.00 4718770 vm object vm_object_reference+0x134 110 10% 65% 0.00 2507362 vm object (usermode) 74 7% 72% 0.00 14558 vm object vm_fault_hold+0x2700 63 6% 78% 0.00 3918872 vm object 0xc0000000030f8558 50 5% 82% 0.00 7950294 vm object vm_object_shadow+0x390 47 4% 87% 0.00 5258832 vm object vm_object_shadow+0x378 33 3% 90% 0.00 3706776 vm object vm_imgact_map_page+0x3f0 23 2% 92% 0.00 5979526 vm object exec_map_first_page+0x780 17 2% 93% 0.00 213461 kernel vm object 0xc0000000030f09c0 17 2% 95% 0.00 238986 kernel vm object kmem_back_domain+0x340 11 1% 96% 0.00 11416 Per-Filesystem Softdep Lock (usermode) 8 1% 96% 0.00 3410066 vm object vm_object_deallocate+0x814 7 1% 97% 0.00 653613 kernel vm object (usermode) 6 1% 98% 0.00 5967 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 4 0% 98% 0.00 12633 Per-Filesystem Softdep Lock 0xc000000003095904 2 0% 98% 0.00 25626 vm object vm_object_collapse+0x710 2 0% 98% 0.00 4525 vm object vm_fault_hold+0x1298 2 0% 98% 0.00 9175 Per-Filesystem Softdep Lock 0xc0000000030adb58 2 0% 99% 0.00 21277 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c 2 0% 99% 0.00 6235 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 2 0% 99% 0.00 25474 Per-Filesystem Softdep Lock 0xc00000000309c6f4 1 0% 99% 0.00 19082518 vm object vnode_create_vobject+0x18c 1 0% 99% 0.00 572 vm object bdwrite+0x2f4 1 0% 99% 0.00 218 tidhash tidhash_add+0x16c 1 0% 99% 0.00 5895 Per-Filesystem Softdep Lock softdep_slowdown+0x374 1 0% 99% 0.00 23417 bufobj interlock bqrelse+0x2a0 1 0% 100% 0.00 11610 Per-Filesystem Softdep Lock softdep_freefile+0x944 1 0% 100% 0.00 2028 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 1 0% 100% 0.00 11431 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c 1 0% 100% 0.00 22348 Per-Filesystem Softdep Lock 0xc00000000309f2b4 1 0% 100% 0.00 5952 Per-Filesystem Softdep Lock 0xc000000002cec288 1 0% 100% 0.00 1699 Per-Filesystem Softdep Lock softdep_setup_blkmapdep+0x294 ------------------------------------------------------------------------------- R/W writer spin on readers: 1114 events in 2.265 seconds (492 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 362 32% 32% 0.00 5886158 vm object vm_fault_hold+0x1b28 139 12% 45% 0.00 4037590 vm object vm_object_deallocate+0x800 129 12% 57% 0.00 5458300 vm object (usermode) 106 10% 66% 0.00 5075452 vm object vm_object_deallocate+0x814 99 9% 75% 0.00 4771047 vm object 0xc0000000030f8558 90 8% 83% 0.00 5461882 vm object vnode_create_vobject+0x18c 55 5% 88% 0.00 8111977 vm object vm_object_reference+0x134 31 3% 91% 0.00 7325308 vm object vm_map_insert+0x598 31 3% 94% 0.00 6589354 vm object vm_imgact_map_page+0x3f0 24 2% 96% 0.00 19539 vm object vm_fault_hold+0x1298 22 2% 98% 0.00 10694 vm object vm_fault_hold+0x2700 17 2% 99% 0.00 8913230 vm object vm_object_shadow+0x378 7 1% 100% 0.00 2133206 vm object vm_object_shadow+0x390 1 0% 100% 0.00 7148053 vm object exec_map_first_page+0x780 1 0% 100% 0.00 391 bufobj interlock bqrelse+0x2a0 ------------------------------------------------------------------------------- R/W reader spin on writer: 401 events in 2.265 seconds (177 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 290 72% 72% 0.00 1139370 vm object __rw_rlock_int+0x9c 110 27% 100% 0.00 1010936 vm object (usermode) 1 0% 100% 0.00 6525 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 28 events in 2.265 seconds (12 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 25 89% 89% 0.00 917363 vm object __rw_rlock_int+0x9c 3 11% 100% 0.00 124400 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 11 events in 2.265 seconds (5 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 9 82% 82% 0.00 55261 vm map (user) (usermode) 1 9% 91% 0.00 27233 vq->vq_lock (usermode) 1 9% 100% 0.00 47760 vm map (user) _vm_map_lock+0x138 ------------------------------------------------------------------------------- SX exclusive spin: 2016 events in 2.265 seconds (890 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 447 22% 22% 0.00 4577 rrl->rr_lock 0xc00000000260abbc 369 18% 40% 0.00 5218 rrl->rr_lock rrw_exit+0x310 225 11% 52% 0.00 38184 vm map (user) (usermode) 143 7% 59% 0.00 315948 vq->vq_lock (usermode) 96 5% 63% 0.00 676580 os->os_userused_lock (usermode) 90 4% 68% 0.00 238970 vq->vq_lock vdev_queue_io+0x268 88 4% 72% 0.00 15181 proctree kern_wait6+0x3e8 61 3% 75% 0.00 5994 rrl->rr_lock (usermode) 47 2% 78% 0.00 31882 vm map (user) _vm_map_lock+0x138 41 2% 80% 0.00 1635 zp->z_acl_lock 0xc0000000026790e0 41 2% 82% 0.00 17138 tm->tm_lock trim_map_free+0x19c 22 1% 83% 0.00 12894 mg->mg_lock metaslab_alloc_dva+0x3a8 21 1% 84% 0.00 9547 ms->ms_lock metaslab_alloc_dva+0x1138 20 1% 85% 0.00 222356 vq->vq_lock 0xc0000000026563c8 20 1% 86% 0.00 1053317 os->os_userused_lock 0xc0000000025b7cdc 20 1% 87% 0.00 20036 mg->mg_lock (usermode) 18 1% 88% 0.00 16215 zio->io_lock 0xc0000000026bfad8 18 1% 89% 0.00 9852 zio->io_lock (usermode) 17 1% 89% 0.00 9387 zio->io_lock 0xc0000000026c1ac0 17 1% 90% 0.00 18705 vm map (user) _vm_map_lock_upgrade+0x180 17 1% 91% 0.00 30731 mg->mg_lock metaslab_alloc_dva+0x7d4 15 1% 92% 0.00 11853 zio->io_lock zio_add_child+0x274 13 1% 93% 0.00 5638 scl->scl_lock spa_config_exit+0x84 10 0% 93% 0.00 8116 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x268 10 0% 94% 0.00 7726 scl->scl_lock spa_config_enter+0x140 9 0% 94% 0.00 126314 vq->vq_lock vdev_queue_io_done+0x12c 9 0% 94% 0.00 11822 vq->vq_lock vdev_queue_io_done+0x278 9 0% 95% 0.00 4217 spa->spa_alloc_locks[i] 0xc0000000026bd584 9 0% 95% 0.00 1518 dd->dd_lock dsl_dir_tempreserve_clear+0x88 8 0% 96% 0.00 22060 dp->dp_lock dsl_pool_undirty_space+0x18c 8 0% 96% 0.00 7708 dd->dd_lock dsl_dir_diduse_space+0x270 7 0% 96% 0.00 1405 zp->z_acl_lock (usermode) 7 0% 97% 0.00 8574 proctree (usermode) 7 0% 97% 0.00 440011 os->os_userused_lock 0xc0000000025b7e74 7 0% 98% 0.00 8370 mc->mc_lock metaslab_class_throttle_unreserve+0x154 6 0% 98% 0.00 22428 proctree exit1+0x1aec 6 0% 98% 0.00 4449 ms->ms_lock (usermode) 5 0% 98% 0.00 219 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 5 0% 99% 0.00 10984 db->db_mtx (usermode) 4 0% 99% 0.00 12888 dd->dd_lock (usermode) 3 0% 99% 0.00 1141 spa->spa_alloc_locks[i] (usermode) 3 0% 99% 0.00 1413 ms->ms_lock metaslab_free_concrete+0x2e8 3 0% 99% 0.00 3189 db->db_mtx dbuf_read+0xa44 2 0% 99% 0.00 1126 allproc exit1+0x1a84 1 0% 99% 0.00 197407 tm->tm_lock (usermode) 1 0% 99% 0.00 20584 proctree killjobc+0x600 1 0% 100% 0.00 1082 proctree fork1+0x28bc 1 0% 100% 0.00 7759 mc->mc_lock (usermode) 1 0% 100% 0.00 556 hdl->sa_lock sa_bulk_lookup+0x154 1 0% 100% 0.00 231 hdl->sa_lock sa_lookup_uio+0x198 1 0% 100% 0.00 152 dp->dp_lock dsl_pool_need_dirty_delay+0x1ac 1 0% 100% 0.00 5601 dp->dp_lock dsl_pool_dirty_space+0x17c 1 0% 100% 0.00 84914 dp->dp_lock (usermode) 1 0% 100% 0.00 4879 dn->dn_struct_rwlock 0xc0000000025a65e0 1 0% 100% 0.00 246 db->db_mtx dmu_buf_will_dirty+0x2e8 1 0% 100% 0.00 1799 dd->dd_lock 0xc0000000025e7c24 1 0% 100% 0.00 826 db->db_mtx dbuf_find+0x12c ------------------------------------------------------------------------------- SX shared spin: 304 events in 2.265 seconds (134 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 234 77% 77% 0.00 23506 vm map (user) _sx_slock_int+0xa0 67 22% 99% 0.00 29834 vm map (user) (usermode) 2 1% 100% 0.00 271 zf->zf_rwlock _sx_slock_int+0xa0 1 0% 100% 0.00 1352 zf->zf_rwlock (usermode) ------------------------------------------------------------------------------- root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 89025 events in 3.212 seconds (27719 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 27527 31% 31% 0.00 14730 page pv moea64_enter+0x86c 25887 29% 60% 0.00 13183 page pv 0xc000000003142640 23845 27% 87% 0.00 14667 page pv (usermode) 2425 3% 90% 0.00 9728 vm active pagequeue 0xc00000000310e854 2135 2% 92% 0.00 11063 vm page vm_page_change_lock+0x160 837 1% 93% 0.00 4435 vnode interlock vget+0x2f8 746 1% 94% 0.00 11520 vm active pagequeue (usermode) 642 1% 94% 0.00 13557 vm page vm_fault_hold+0x2724 480 1% 95% 0.00 8262 vnode interlock (usermode) 370 0% 95% 0.00 16505 vm page (usermode) 300 0% 96% 0.00 10349 vnode interlock 0xc000000002e224a0 295 0% 96% 0.00 168779 zio_write_issue 0xc000000002d7c86c 242 0% 96% 0.00 17344 dp_sync_taskq (usermode) 224 0% 97% 0.00 35579 dp_sync_taskq 0xc000000002d7c86c 208 0% 97% 0.00 12556 dp_sync_taskq 0xc000000002cc30c0 207 0% 97% 0.00 8761 struct mount mtx vfs_unbusy+0x15c 200 0% 97% 0.00 221129 zio_write_issue (usermode) 182 0% 97% 0.00 27030 zio_write_intr (usermode) 153 0% 98% 0.00 154372 zio_write_issue 0xc000000002cc30c0 150 0% 98% 0.00 206083 zio_write_issue taskqueue_enqueue+0x158 142 0% 98% 0.00 6469 select mtxpool selrecord+0x1dc 142 0% 98% 0.00 16772 dp_sync_taskq taskqueue_enqueue+0x158 130 0% 98% 0.00 14290 pmap moea64_enter+0x880 129 0% 98% 0.00 2785 struct mount mtx vfs_busy+0x434 117 0% 99% 0.00 21822 zio_write_intr 0xc000000002d7c86c 113 0% 99% 0.00 4544 struct mount mtx (usermode) 112 0% 99% 0.00 17230 zio_write_intr 0xc000000002cc30c0 102 0% 99% 0.00 6771 page pv moea64_enter+0x910 101 0% 99% 0.00 39804 kernelpmap moea64_kextract+0x174 100 0% 99% 0.00 15846 vnode interlock 0xc000000002e22484 70 0% 99% 0.00 17449 zio_write_intr taskqueue_enqueue+0x158 63 0% 99% 0.00 556126 process lock ast+0x7e0 49 0% 99% 0.00 12158 pmap (usermode) 45 0% 99% 0.00 8252 vnode interlock 0xc0000000030ce5b8 45 0% 99% 0.00 7308 vm active pagequeue vm_page_dequeue+0x1b4 40 0% 99% 0.00 41297 kernelpmap (usermode) 37 0% 100% 0.00 7133 select mtxpool 0xc000000002d93038 32 0% 100% 0.00 10695 umtxql (usermode) 32 0% 100% 0.00 27057 kernelpmap moea64_enter+0x880 24 0% 100% 0.00 956 struct mount vlist mtx _vhold+0x474 24 0% 100% 0.00 73095 process lock kern_thr_exit+0x12c 21 0% 100% 0.00 2813 struct mount vlist mtx _vdrop+0x6f4 21 0% 100% 0.00 95377 process lock kern_thr_exit+0x24c 19 0% 100% 0.00 11161 vm page 0xc0000000030ec28c 16 0% 100% 0.00 18234 vm page free queue 0xc000000003111274 16 0% 100% 0.00 7742 umtxql 0xc000000002d228cc 14 0% 100% 0.00 8172 umtxql 0xc000000002d245d4 14 0% 100% 0.00 759 UPVO entry uma_zalloc_arg+0x98c 13 0% 100% 0.00 843 UPVO entry (usermode) 11 0% 100% 0.00 1070 ttymtx 0xc000000002db7520 10 0% 100% 0.00 6097 struct mount mtx vfs_ref+0x114 10 0% 100% 0.00 70767 nvme qpair lock 0xc000000002a101b0 9 0% 100% 0.00 17352 vm page free queue (usermode) 9 0% 100% 0.00 952 struct mount vlist mtx (usermode) 8 0% 100% 0.00 1090 pipe mutex 0xc000000002d93a60 8 0% 100% 0.00 2116 UPVO entry uma_zfree_arg+0x5c8 7 0% 100% 0.00 8622 umtxql 0xc000000002d22b78 6 0% 100% 0.00 7901 ttymtx (usermode) 5 0% 100% 0.00 30356 vm page vm_object_madvise+0x54c 5 0% 100% 0.00 5963 vm page exec_map_first_page+0x7b8 5 0% 100% 0.00 265265 process lock (usermode) 5 0% 100% 0.00 2340 pipe mutex 0xc000000002d95074 5 0% 100% 0.00 7499 pipe mutex 0xc000000002d9456c 4 0% 100% 0.00 1165 vnode interlock _vhold+0x4a4 4 0% 100% 0.00 1578 vm page vm_object_unwire+0x2d0 4 0% 100% 0.00 6008 vm page vm_imgact_unmap_page+0x184 4 0% 100% 0.00 6630 vm page vm_imgact_map_page+0x43c 4 0% 100% 0.00 2803 umtxql 0xc000000002d253f8 4 0% 100% 0.00 11122 struct mount mtx vfs_rel+0x128 4 0% 100% 0.00 1930 pipe mutex 0xc000000002d93828 4 0% 100% 0.00 5456 pipe mutex (usermode) 4 0% 100% 0.00 99182 kernelpmap moea64_remove+0x200 4 0% 100% 0.00 3189 CAM device lock 0xc000000002524148 3 0% 100% 0.00 6558 vnode interlock vinactive+0x25c 3 0% 100% 0.00 17412 vm page free queue 0xc000000003110fb4 2 0% 100% 0.00 15507 ttymtx ttydisc_write+0x388 2 0% 100% 0.00 1140 ttymtx ttyoutq_read_uio+0x34c 2 0% 100% 0.00 658 ttymtx 0xc000000002db84b4 2 0% 100% 0.00 934 sellck 0xc000000002d92870 2 0% 100% 0.00 152 select mtxpool 0xc000000002d92a68 2 0% 100% 0.00 4492 select mtxpool (usermode) 2 0% 100% 0.00 1196 reseed mutex 0xc000000002a42484 2 0% 100% 0.00 221 pipe mutex 0xc000000002d95914 2 0% 100% 0.00 17482 dp_sync_taskq taskqueue_quiesce+0x2e8 2 0% 100% 0.00 730 CAM device lock 0xc000000002524ac0 1 0% 100% 0.00 221 vnode interlock 0xc000000002e09c84 1 0% 100% 0.00 22904 vm page vm_page_rename+0x224 1 0% 100% 0.00 38472 vm active pagequeue 0xc000000003114894 1 0% 100% 0.00 5892 umtxql 0xc000000002d24294 1 0% 100% 0.00 12417 umtxql 0xc000000002d25428 1 0% 100% 0.00 422 process_exit (usermode) 1 0% 100% 0.00 4267 process lock ast+0x6e0 1 0% 100% 0.00 10278 reseed mutex (usermode) 1 0% 100% 0.00 11228 reseed mutex 0xc000000002a42744 1 0% 100% 0.00 225 CAM device lock 0xc000000002523518 1 0% 100% 0.00 1793 CAM queue lock 0xc000000002511d84 1 0% 100% 0.00 314 SLB table moea64_get_unique_vsid+0x2a8 1 0% 100% 0.00 12434 kernelpmap moea64_kenter_attr+0x3d4 1 0% 100% 0.00 225 kernel arena domain (usermode) 1 0% 100% 0.00 1228 ncneglh (usermode) 1 0% 100% 0.00 40919 metaslab_group_taskq taskqueue_enqueue+0x158 1 0% 100% 0.00 10874 metaslab_group_taskq (usermode) 1 0% 100% 0.00 201 ncneglh cache_lookup+0x1608 ------------------------------------------------------------------------------- Adaptive mutex block: 7 events in 3.212 seconds (2 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2 29% 29% 0.00 33739 zio_write_intr 0xc000000002d7c86c 2 29% 57% 0.00 258472 page pv (usermode) 1 14% 71% 0.00 24165 zio_write_intr taskqueue_enqueue+0x158 1 14% 86% 0.00 21554 zio_write_intr (usermode) 1 14% 100% 0.00 740629 page pv 0xc000000003142640 ------------------------------------------------------------------------------- Spin lock spin: 25483 events in 3.212 seconds (7935 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 13772 54% 54% 0.00 456422 turnstile chain (usermode) 9592 38% 92% 0.00 380969 turnstile chain turnstile_trywait+0x208 929 4% 95% 0.00 10443 sleepq chain sleepq_lock+0xe4 373 1% 97% 0.00 7872 sleepq chain (usermode) 342 1% 98% 0.00 70732 turnstile chain turnstile_chain_lock+0xdc 169 1% 99% 0.00 422 turnstile lock turnstile_trywait+0x150 135 1% 99% 0.00 501 turnstile lock (usermode) 13 0% 99% 0.00 19531 sched lock 55 0xc000000002d3cce4 11 0% 99% 0.00 24085 sched lock 5 0xc000000002d3cccc 9 0% 99% 0.00 5641 sched lock 55 0xc000000002d3cccc 8 0% 99% 0.00 3720 sched lock 0 0xc000000002d3cccc 5 0% 100% 0.00 5658 sched lock 20 0xc000000002d3cccc 4 0% 100% 0.00 3926 sched lock 0 (usermode) 3 0% 100% 0.00 4063 turnstile lock turnstile_lookup+0xd4 3 0% 100% 0.00 416 sched lock 65 0xc000000002d3cce4 3 0% 100% 0.00 4591 sched lock 65 sched_switch+0x81c 3 0% 100% 0.00 868 sched lock 62 sched_add+0x48c 3 0% 100% 0.00 5700 sched lock 55 (usermode) 3 0% 100% 0.00 940 sched lock 25 0xc000000002d3cccc 3 0% 100% 0.00 2147 sched lock 14 sched_add+0x48c 3 0% 100% 0.00 1206 sched lock 10 0xc000000002d3cccc 2 0% 100% 0.00 1102 sched lock 67 0xc000000002d3cce4 2 0% 100% 0.00 437 sched lock 55 sched_add+0x48c 2 0% 100% 0.00 4403 sched lock 55 sched_switch+0x81c 2 0% 100% 0.00 11329 sched lock 53 0xc000000002d3cce4 2 0% 100% 0.00 27303 sched lock 53 0xc000000002d3cccc 2 0% 100% 0.00 411 sched lock 53 sched_add+0x48c 2 0% 100% 0.00 6011 sched lock 45 0xc000000002d3cce4 2 0% 100% 0.00 1884 sched lock 40 0xc000000002d3cccc 2 0% 100% 0.00 1486 sched lock 38 0xc000000002d3cccc 2 0% 100% 0.00 10325 sched lock 33 0xc000000002d3cce4 2 0% 100% 0.00 26224 sched lock 3 (usermode) 2 0% 100% 0.00 373 sched lock 29 sched_add+0x48c 2 0% 100% 0.00 15185 sched lock 28 0xc000000002d3cccc 2 0% 100% 0.00 277 sched lock 28 sched_add+0x48c 2 0% 100% 0.00 675 sched lock 22 sched_add+0x48c 2 0% 100% 0.00 3665 sched lock 16 0xc000000002d3cccc 1 0% 100% 0.00 49408 sched lock 9 (usermode) 1 0% 100% 0.00 1103 sched lock 8 0xc000000002d3cccc 1 0% 100% 0.00 1923 sched lock 8 sched_add+0x48c 1 0% 100% 0.00 311 sched lock 70 0xc000000002d3cce4 1 0% 100% 0.00 990 sched lock 71 sched_add+0x48c 1 0% 100% 0.00 1932 sched lock 68 sched_add+0x48c 1 0% 100% 0.00 771 sched lock 69 0xc000000002d3cce4 1 0% 100% 0.00 522 sched lock 70 sched_add+0x48c 1 0% 100% 0.00 1096 sched lock 63 sched_add+0x48c 1 0% 100% 0.00 1957 sched lock 67 sched_add+0x48c 1 0% 100% 0.00 5035 sched lock 60 sched_switch+0x81c 1 0% 100% 0.00 3251 sched lock 58 0xc000000002d3cccc 1 0% 100% 0.00 5790 sched lock 57 (usermode) 1 0% 100% 0.00 252 sched lock 55 0xc000000002d3ccb4 1 0% 100% 0.00 228 sched lock 54 (usermode) 1 0% 100% 0.00 20177 sched lock 55 sched_switch+0xca0 1 0% 100% 0.00 5923 sched lock 48 0xc000000002d3cccc 1 0% 100% 0.00 382 sched lock 48 (usermode) 1 0% 100% 0.00 2758 sched lock 47 0xc000000002d3cce4 1 0% 100% 0.00 311 sched lock 47 sched_add+0x48c 1 0% 100% 0.00 18937 sched lock 47 0xc000000002d3cccc 1 0% 100% 0.00 959 sched lock 5 (usermode) 1 0% 100% 0.00 457 sched lock 49 0xc000000002d3cce4 1 0% 100% 0.00 4988 sched lock 51 0xc000000002d3cce4 1 0% 100% 0.00 699 sched lock 51 sched_add+0x48c 1 0% 100% 0.00 1748 sched lock 52 sched_add+0x48c 1 0% 100% 0.00 195 sched lock 53 (usermode) 1 0% 100% 0.00 1803 sched lock 45 0xc000000002d3cccc 1 0% 100% 0.00 1786 sched lock 45 sched_add+0x48c 1 0% 100% 0.00 1217 sched lock 44 0xc000000002d3cccc 1 0% 100% 0.00 1199 sched lock 45 (usermode) 1 0% 100% 0.00 5246 sched lock 43 (usermode) 1 0% 100% 0.00 205 sched lock 43 sched_add+0x48c 1 0% 100% 0.00 205 sched lock 44 sched_add+0x48c 1 0% 100% 0.00 1744 sched lock 35 (usermode) 1 0% 100% 0.00 1125 sched lock 40 sched_switch+0x81c 1 0% 100% 0.00 1119 sched lock 38 0xc000000002d3cce4 1 0% 100% 0.00 3433 sched lock 42 0xc000000002d3cccc 1 0% 100% 0.00 248 sched lock 24 (usermode) 1 0% 100% 0.00 260 sched lock 24 sched_add+0x48c 1 0% 100% 0.00 1330 sched lock 24 0xc000000002d3cccc 1 0% 100% 0.00 7052 sched lock 29 0xc000000002d3ccb4 1 0% 100% 0.00 2877 sched lock 30 sched_switch+0x81c 1 0% 100% 0.00 407 sched lock 3 0xc000000002d3cccc 1 0% 100% 0.00 672 sched lock 32 0xc000000002d3cccc 1 0% 100% 0.00 203 sched lock 33 (usermode) 1 0% 100% 0.00 352 sched lock 12 0xc000000002d3cccc 1 0% 100% 0.00 12356 sched lock 11 0xc000000002d3cccc 1 0% 100% 0.00 1107 sched lock 11 sched_add+0x48c 1 0% 100% 0.00 356 sched lock 1 sched_add+0x48c 1 0% 100% 0.00 1883 sched lock 10 (usermode) 1 0% 100% 0.00 231 callout 0xc000000002d1b164 1 0% 100% 0.00 690 callout (usermode) 1 0% 100% 0.00 457 callout 0xc000000002d1be6c 1 0% 100% 0.00 233 et_hw_mtx (usermode) 1 0% 100% 0.00 1345 sched lock 19 0xc000000002d3cccc 1 0% 100% 0.00 373 sched lock 18 0xc000000002d3cccc 1 0% 100% 0.00 3077 sched lock 17 0xc000000002d3cccc 1 0% 100% 0.00 1091 sched lock 15 sched_add+0x48c 1 0% 100% 0.00 1971 sched lock 16 (usermode) 1 0% 100% 0.00 17793 sched lock 20 0xc000000002d3cce4 1 0% 100% 0.00 257 sched lock 20 (usermode) 1 0% 100% 0.00 219 sched lock 22 0xc000000002d3cccc 1 0% 100% 0.00 203 sched lock 23 sched_switch+0x81c 1 0% 100% 0.00 3506 sched lock 23 0xc000000002d3cccc ------------------------------------------------------------------------------- Thread lock spin: 543 events in 3.212 seconds (169 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 277 51% 51% 0.00 3713 sleepq chain _thread_lock+0xf4 110 20% 71% 0.00 3283 sleepq chain (usermode) 33 6% 77% 0.00 15364 turnstile lock _thread_lock+0xf4 6 1% 78% 0.00 3558 sched lock 65 _thread_lock+0xf4 6 1% 80% 0.00 2211 sched lock 45 _thread_lock+0xf4 5 1% 80% 0.00 3915 sched lock 5 _thread_lock+0xf4 5 1% 81% 0.00 15480 sched lock 0 (usermode) 4 1% 82% 0.00 6713 turnstile lock (usermode) 4 1% 83% 0.00 3973 sched lock 7 _thread_lock+0xf4 4 1% 84% 0.00 6673 sched lock 55 _thread_lock+0xf4 4 1% 84% 0.00 4965 sched lock 5 (usermode) 4 1% 85% 0.00 7146 sched lock 20 _thread_lock+0xf4 4 1% 86% 0.00 6825 sched lock 18 _thread_lock+0xf4 3 1% 86% 0.00 42060 sched lock 55 (usermode) 3 1% 87% 0.00 3800 sched lock 32 _thread_lock+0xf4 3 1% 87% 0.00 8235 sched lock 3 _thread_lock+0xf4 3 1% 88% 0.00 4901 sched lock 0 _thread_lock+0xf4 2 0% 88% 0.00 909 sched lock 9 _thread_lock+0xf4 2 0% 89% 0.00 5238 sched lock 70 _thread_lock+0xf4 2 0% 89% 0.00 3487 sched lock 62 (usermode) 2 0% 90% 0.00 11106 sched lock 51 (usermode) 2 0% 90% 0.00 395 sched lock 49 _thread_lock+0xf4 2 0% 90% 0.00 6125 sched lock 47 _thread_lock+0xf4 2 0% 91% 0.00 3153 sched lock 47 (usermode) 2 0% 91% 0.00 1696 sched lock 33 _thread_lock+0xf4 2 0% 91% 0.00 460 sched lock 32 (usermode) 2 0% 92% 0.00 4238 sched lock 28 (usermode) 2 0% 92% 0.00 12546 sched lock 26 _thread_lock+0xf4 2 0% 92% 0.00 1373 sched lock 25 _thread_lock+0xf4 2 0% 93% 0.00 562 sched lock 20 (usermode) 2 0% 93% 0.00 3005 sched lock 17 _thread_lock+0xf4 2 0% 94% 0.00 393 sched lock 16 (usermode) 2 0% 94% 0.00 1792 sched lock 14 _thread_lock+0xf4 2 0% 94% 0.00 6695 sched lock 1 _thread_lock+0xf4 1 0% 94% 0.00 2000 sched lock 71 (usermode) 1 0% 95% 0.00 445 sched lock 71 _thread_lock+0xf4 1 0% 95% 0.00 566 sched lock 66 (usermode) 1 0% 95% 0.00 357 sched lock 6 _thread_lock+0xf4 1 0% 95% 0.00 777 sched lock 60 _thread_lock+0xf4 1 0% 95% 0.00 520 sched lock 56 (usermode) 1 0% 96% 0.00 412 sched lock 57 _thread_lock+0xf4 1 0% 96% 0.00 1967 sched lock 57 (usermode) 1 0% 96% 0.00 6203 sched lock 58 _thread_lock+0xf4 1 0% 96% 0.00 527 sched lock 6 (usermode) 1 0% 96% 0.00 636 sched lock 49 (usermode) 1 0% 97% 0.00 19461 sched lock 48 (usermode) 1 0% 97% 0.00 20566 sched lock 52 _thread_lock+0xf4 1 0% 97% 0.00 904 sched lock 52 (usermode) 1 0% 97% 0.00 10489 sched lock 53 (usermode) 1 0% 97% 0.00 12959 sched lock 53 _thread_lock+0xf4 1 0% 97% 0.00 375 sched lock 22 _thread_lock+0xf4 1 0% 98% 0.00 3476 sched lock 24 _thread_lock+0xf4 1 0% 98% 0.00 547 sched lock 21 (usermode) 1 0% 98% 0.00 10747 sched lock 21 _thread_lock+0xf4 1 0% 98% 0.00 21474 sched lock 22 (usermode) 1 0% 98% 0.00 284 sched lock 15 _thread_lock+0xf4 1 0% 99% 0.00 418 sched lock 28 _thread_lock+0xf4 1 0% 99% 0.00 15819 sched lock 29 (usermode) 1 0% 99% 0.00 228982 sched lock 36 _thread_lock+0xf4 1 0% 99% 0.00 305 sched lock 34 _thread_lock+0xf4 1 0% 99% 0.00 2605 sched lock 43 (usermode) 1 0% 99% 0.00 12708 sched lock 40 _thread_lock+0xf4 1 0% 100% 0.00 7175 sched lock 43 _thread_lock+0xf4 1 0% 100% 0.00 379 sched lock 44 _thread_lock+0xf4 1 0% 100% 0.00 11052 sched lock 11 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 900 events in 3.212 seconds (280 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 204 23% 23% 0.00 68275248 vm object vm_fault_hold+0x1b28 154 17% 40% 0.00 65504136 vm object vm_object_deallocate+0x800 152 17% 57% 0.00 68160025 vm object vm_map_insert+0x598 119 13% 70% 0.00 75705047 vm object vm_object_reference+0x134 62 7% 77% 0.00 57437927 vm object 0xc0000000030f8558 53 6% 83% 0.00 68094312 vm object vm_object_shadow+0x378 51 6% 88% 0.00 72341573 vm object vm_object_shadow+0x390 51 6% 94% 0.00 49334043 vm object (usermode) 33 4% 98% 0.00 68420038 vm object vm_imgact_map_page+0x3f0 14 2% 99% 0.00 79196931 vm object exec_map_first_page+0x780 5 1% 100% 0.00 65593461 vm object vm_object_deallocate+0x814 1 0% 100% 0.00 62054771 vm object vnode_create_vobject+0x18c 1 0% 100% 0.00 639835 kernel vm object kmem_back_domain+0x340 ------------------------------------------------------------------------------- R/W writer blocked by readers: 1528 events in 3.212 seconds (476 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 642 42% 42% 0.00 57972413 vm object vm_fault_hold+0x1b28 167 11% 53% 0.00 53811387 vm object vm_object_deallocate+0x800 142 9% 62% 0.00 50281313 vm object 0xc0000000030f8558 127 8% 71% 0.00 34196275 vm object (usermode) 126 8% 79% 0.00 46563755 vm object vm_object_deallocate+0x814 122 8% 87% 0.00 62907468 vm object vnode_create_vobject+0x18c 65 4% 91% 0.00 67883370 vm object vm_object_reference+0x134 52 3% 94% 0.00 58631748 vm object vm_imgact_map_page+0x3f0 36 2% 97% 0.00 56889907 vm object vm_map_insert+0x598 35 2% 99% 0.00 72284127 vm object vm_object_shadow+0x378 11 1% 100% 0.00 57857520 vm object vm_object_shadow+0x390 3 0% 100% 0.00 22481818 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader blocked by writer: 307 events in 3.212 seconds (96 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 201 65% 65% 0.00 37234257 vm object __rw_rlock_int+0x9c 106 35% 100% 0.00 26885113 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 36 events in 3.212 seconds (11 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 25 69% 69% 0.00 43848532 vm object __rw_rlock_int+0x9c 11 31% 100% 0.00 21717328 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 1496 events in 3.212 seconds (466 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 306 20% 20% 0.00 4640739 vm object vm_fault_hold+0x1b28 228 15% 36% 0.00 4388054 vm object vm_object_deallocate+0x800 219 15% 50% 0.00 5233995 vm object vm_map_insert+0x598 176 12% 62% 0.00 5555584 vm object vm_object_reference+0x134 126 8% 71% 0.00 2612835 vm object (usermode) 79 5% 76% 0.00 4847232 vm object 0xc0000000030f8558 78 5% 81% 0.00 4346261 vm object vm_object_shadow+0x378 78 5% 86% 0.00 13631 vm object vm_fault_hold+0x2700 73 5% 91% 0.00 5330887 vm object vm_object_shadow+0x390 48 3% 94% 0.00 5442199 vm object vm_imgact_map_page+0x3f0 25 2% 96% 0.00 4755179 vm object exec_map_first_page+0x780 17 1% 97% 0.00 140414 kernel vm object kmem_back_domain+0x340 9 1% 98% 0.00 232819 kernel vm object 0xc0000000030f09c0 8 1% 98% 0.00 13306 Per-Filesystem Softdep Lock (usermode) 7 0% 99% 0.00 4192238 vm object vm_object_deallocate+0x814 4 0% 99% 0.00 18527 Per-Filesystem Softdep Lock 0xc000000003095904 2 0% 99% 0.00 2638777 vm object vnode_create_vobject+0x18c 2 0% 99% 0.00 24734 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 2 0% 99% 0.00 13469 Per-Filesystem Softdep Lock 0xc00000000309f2b4 2 0% 100% 0.00 29296 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 1 0% 100% 0.00 202281 kernel vm object (usermode) 1 0% 100% 0.00 382 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 1 0% 100% 0.00 4447 Per-Filesystem Softdep Lock softdep_slowdown+0x374 1 0% 100% 0.00 1833 Per-Filesystem Softdep Lock 0xc0000000030adb58 1 0% 100% 0.00 3259 bufobj interlock bqrelse+0x2a0 1 0% 100% 0.00 1453 Per-Filesystem Softdep Lock 0xc000000002cec288 1 0% 100% 0.00 15313 Per-Filesystem Softdep Lock softdep_setup_create+0x188 ------------------------------------------------------------------------------- R/W writer spin on readers: 1635 events in 3.212 seconds (509 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 658 40% 40% 0.00 5235416 vm object vm_fault_hold+0x1b28 182 11% 51% 0.00 4506783 vm object vm_object_deallocate+0x800 145 9% 60% 0.00 4457533 vm object 0xc0000000030f8558 136 8% 69% 0.00 4621302 vm object (usermode) 126 8% 76% 0.00 4206536 vm object vm_object_deallocate+0x814 122 7% 84% 0.00 7530029 vm object vnode_create_vobject+0x18c 71 4% 88% 0.00 6466791 vm object vm_object_reference+0x134 53 3% 91% 0.00 5127448 vm object vm_imgact_map_page+0x3f0 41 3% 94% 0.00 4367829 vm object vm_map_insert+0x598 38 2% 96% 0.00 8110362 vm object vm_object_shadow+0x378 22 1% 97% 0.00 15859 vm object vm_fault_hold+0x1298 21 1% 99% 0.00 13490 vm object vm_fault_hold+0x2700 12 1% 100% 0.00 4546083 vm object vm_object_shadow+0x390 4 0% 100% 0.00 1708419 vm object exec_map_first_page+0x780 2 0% 100% 0.00 8015 vm object 0xc00000000311f65c 1 0% 100% 0.00 400 vm object vm_object_collapse+0x710 1 0% 100% 0.00 16389 bufobj interlock bqrelse+0x2a0 ------------------------------------------------------------------------------- R/W reader spin on writer: 515 events in 3.212 seconds (160 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 365 71% 71% 0.00 1257102 vm object __rw_rlock_int+0x9c 149 29% 100% 0.00 1324550 vm object (usermode) 1 0% 100% 0.00 783 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 35 events in 3.212 seconds (11 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 25 71% 71% 0.00 2626769 vm object __rw_rlock_int+0x9c 10 29% 100% 0.00 1285498 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 51 events in 3.212 seconds (16 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 26 51% 51% 0.00 682843 vq->vq_lock (usermode) 12 24% 75% 0.00 767719 vq->vq_lock vdev_queue_io+0x268 6 12% 86% 0.00 42765 vm map (user) (usermode) 3 6% 92% 0.00 29977 vm map (user) _vm_map_lock+0x138 1 2% 94% 0.00 28782 vq->vq_lock 0xc0000000026563c8 1 2% 96% 0.00 21646 ms->ms_lock metaslab_alloc_dva+0x1138 1 2% 98% 0.00 40485 ms->ms_lock (usermode) 1 2% 100% 0.00 25591 dn->dn_struct_rwlock 0xc0000000025a65e0 ------------------------------------------------------------------------------- SX exclusive spin: 2428 events in 3.212 seconds (756 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 416 17% 17% 0.00 5162 rrl->rr_lock 0xc00000000260abbc 323 13% 30% 0.00 4679 rrl->rr_lock rrw_exit+0x310 185 8% 38% 0.00 253437 vq->vq_lock (usermode) 158 7% 45% 0.00 35598 vm map (user) (usermode) 134 6% 50% 0.00 12066 zp->z_acl_lock 0xc0000000026790e0 128 5% 55% 0.00 5382 rrl->rr_lock (usermode) 127 5% 61% 0.00 30186 tm->tm_lock trim_map_free+0x19c 109 4% 65% 0.00 205095 vq->vq_lock vdev_queue_io+0x268 105 4% 69% 0.00 27836 vm map (user) _vm_map_lock+0x138 86 4% 73% 0.00 11332 proctree kern_wait6+0x3e8 48 2% 75% 0.00 987800 os->os_userused_lock (usermode) 44 2% 77% 0.00 2379 zp->z_acl_lock (usermode) 43 2% 79% 0.00 493697 os->os_userused_lock 0xc0000000025b7cdc 36 1% 80% 0.00 20976 mg->mg_lock metaslab_alloc_dva+0x3a8 31 1% 81% 0.00 22597 zio->io_lock 0xc0000000026bfad8 27 1% 82% 0.00 14645 ms->ms_lock metaslab_alloc_dva+0x1138 25 1% 83% 0.00 35996 mg->mg_lock metaslab_alloc_dva+0x7d4 23 1% 84% 0.00 11625 zio->io_lock zio_add_child+0x274 23 1% 85% 0.00 8233 zio->io_lock (usermode) 23 1% 86% 0.00 180127 vq->vq_lock 0xc0000000026563c8 23 1% 87% 0.00 32547 vm map (user) _vm_map_lock_upgrade+0x180 22 1% 88% 0.00 8382 dd->dd_lock dsl_dir_tempreserve_clear+0x88 21 1% 89% 0.00 55675 vq->vq_lock vdev_queue_io_done+0x278 20 1% 90% 0.00 10900 proctree (usermode) 19 1% 91% 0.00 15998 zio->io_lock 0xc0000000026c1ac0 19 1% 91% 0.00 23696 mg->mg_lock (usermode) 17 1% 92% 0.00 308641 os->os_userused_lock 0xc0000000025b7e74 16 1% 93% 0.00 6439 mc->mc_lock metaslab_class_throttle_unreserve+0x154 15 1% 93% 0.00 7731 spa->spa_alloc_locks[i] (usermode) 14 1% 94% 0.00 18033 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x268 14 1% 94% 0.00 5627 scl->scl_lock spa_config_exit+0x84 14 1% 95% 0.00 6092 dd->dd_lock (usermode) 12 0% 96% 0.00 1505 dd->dd_lock 0xc0000000025e7c24 11 0% 96% 0.00 17540 spa->spa_alloc_locks[i] 0xc0000000026bd584 11 0% 96% 0.00 1115 scl->scl_lock spa_config_enter+0x140 10 0% 97% 0.00 10548 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 9 0% 97% 0.00 6320 vq->vq_lock vdev_queue_io_done+0x12c 8 0% 98% 0.00 5640 ms->ms_lock (usermode) 6 0% 98% 0.00 7285 dp->dp_lock dsl_pool_undirty_space+0x18c 6 0% 98% 0.00 4303 db->db_mtx dmu_buf_will_dirty+0x2e8 5 0% 98% 0.00 10559 proctree exit1+0x1aec 5 0% 98% 0.00 27801 dn->dn_struct_rwlock 0xc0000000025a65e0 4 0% 99% 0.00 21565 tm->tm_lock (usermode) 3 0% 99% 0.00 48834 proctree fork1+0x28bc 3 0% 99% 0.00 485 mc->mc_lock (usermode) 3 0% 99% 0.00 605 hdl->sa_lock (usermode) 2 0% 99% 0.00 1135 hdl->sa_lock sa_lookup_uio+0x198 2 0% 99% 0.00 792 dp->dp_lock dsl_pool_need_dirty_delay+0x1ac 2 0% 99% 0.00 3031 dp->dp_lock (usermode) 2 0% 99% 0.00 26541 dn->dn_struct_rwlock (usermode) 2 0% 99% 0.00 283 db->db_mtx dbuf_find+0x12c 2 0% 100% 0.00 225 as->as_lock aggsum_add+0x360 1 0% 100% 0.00 1504 zombproc (usermode) 1 0% 100% 0.00 190 zio->io_lock 0xc0000000026c02a0 1 0% 100% 0.00 164 scl->scl_lock (usermode) 1 0% 100% 0.00 20073 proctree killjobc+0x600 1 0% 100% 0.00 289 ms->ms_lock metaslab_free_concrete+0x2e8 1 0% 100% 0.00 199 ds->ds_lock dsl_dataset_block_kill+0x614 1 0% 100% 0.00 274 dd->dd_lock dsl_dir_diduse_space+0x270 1 0% 100% 0.00 14556 db->db_mtx (usermode) 1 0% 100% 0.00 697 buf_hash_table.ht_locks[i].ht_lock 0xc00000000258be64 1 0% 100% 0.00 355 buf_hash_table.ht_locks[i].ht_lock arc_buf_access+0x488 1 0% 100% 0.00 459 db->db_mtx dbuf_dirty+0xcd0 1 0% 100% 0.00 287 db->db_mtx dbuf_rele+0xbc ------------------------------------------------------------------------------- SX shared spin: 289 events in 3.212 seconds (90 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 237 82% 82% 0.00 25118 vm map (user) _sx_slock_int+0xa0 47 16% 98% 0.00 23047 vm map (user) (usermode) 3 1% 99% 0.00 644 zf->zf_rwlock _sx_slock_int+0xa0 1 0% 100% 0.00 379 zf->zf_rwlock (usermode) 1 0% 100% 0.00 10383 dn->dn_struct_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- root@fledgling:~ #