root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 92219 events in 16.400 seconds (5623 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 37642 41% 41% 0.00 217838 page pv (usermode) 25844 28% 69% 0.00 104587 page pv moea64_remove_pages+0x214 19136 21% 90% 0.00 156165 page pv moea64_enter+0x904 1587 2% 91% 0.00 33290 dp_sync_taskq (usermode) 1280 1% 93% 0.00 36617 dp_sync_taskq 0xc000000002d74f5c 801 1% 94% 0.00 36620 dp_sync_taskq 0xc000000002cbb898 600 1% 94% 0.00 15167 vm active pagequeue 0xc0000000031072a8 545 1% 95% 0.00 20315 vm page vm_page_change_lock+0x160 522 1% 95% 0.00 26887 dp_sync_taskq taskqueue_enqueue+0x158 369 0% 96% 0.00 70667 page pv moea64_remove+0x22c 344 0% 96% 0.00 67195 kernelpmap moea64_kextract+0x174 290 0% 96% 0.00 38264 zio_write_intr (usermode) 289 0% 97% 0.00 47499 kernelpmap moea64_enter+0x918 241 0% 97% 0.00 33633 zio_write_intr 0xc000000002d74f5c 232 0% 97% 0.00 35027 pmap moea64_enter+0x918 230 0% 98% 0.00 37941 zio_write_intr 0xc000000002cbb898 212 0% 98% 0.00 54637 kernelpmap (usermode) 194 0% 98% 0.00 51709 zio_write_issue 0xc000000002cbb898 176 0% 98% 0.00 71919 zio_write_issue 0xc000000002d74f5c 144 0% 98% 0.00 3915 vnode interlock vget+0x2f8 143 0% 98% 0.00 15724 vm active pagequeue (usermode) 138 0% 99% 0.00 28929 vm page vm_fault_hold+0x2724 128 0% 99% 0.00 71599 zio_write_issue taskqueue_enqueue+0x158 124 0% 99% 0.00 10013937 pmap (usermode) 112 0% 99% 0.00 32472 zio_write_intr taskqueue_enqueue+0x158 91 0% 99% 0.00 48696 page pv moea64_enter+0x9a8 87 0% 99% 0.00 24522 vm page (usermode) 82 0% 99% 0.00 97702 zio_write_issue (usermode) 59 0% 99% 0.00 9292 vnode interlock (usermode) 46 0% 99% 0.00 790766 process lock (usermode) 45 0% 99% 0.00 21235 umtxql (usermode) 39 0% 100% 0.00 9390 struct mount mtx vfs_unbusy+0x15c 38 0% 100% 0.00 904898 process lock ast+0x7e0 34 0% 100% 0.00 2673 struct mount mtx vfs_busy+0x434 34 0% 100% 0.00 411312 process lock kern_thr_exit+0x24c 29 0% 100% 0.00 15863 vnode interlock 0xc000000002e1ab8c 24 0% 100% 0.00 437699 process lock kern_thr_exit+0x12c 18 0% 100% 0.00 12770 umtxql 0xc000000002d1afb4 18 0% 100% 0.00 6973 struct mount mtx (usermode) 17 0% 100% 0.00 11086 vm active pagequeue vm_page_dequeue+0x1b4 16 0% 100% 0.00 19383 umtxql 0xc000000002d1ccbc 12 0% 100% 0.00 9525 pipe mutex 0xc000000002d8c150 11 0% 100% 0.00 14099 pipe mutex 0xc000000002d8cc5c 9 0% 100% 0.00 12684 vm page 0xc0000000030e442c 9 0% 100% 0.00 249 pipe mutex 0xc000000002d8e004 9 0% 100% 0.00 51004 kernelpmap moea64_remove+0x420 9 0% 100% 0.00 14697 dp_sync_taskq taskqueue_quiesce+0x2e8 8 0% 100% 0.00 7066 pipe mutex 0xc000000002d8d764 8 0% 100% 0.00 40471 nvme qpair lock 0xc000000002a0bae4 7 0% 100% 0.00 20536 vnode interlock 0xc000000002e1ab70 6 0% 100% 0.00 41876 umtxql 0xc000000002d1dae0 6 0% 100% 0.00 19099 umtxql 0xc000000002d1b260 6 0% 100% 0.00 7518 ttymtx (usermode) 6 0% 100% 0.00 12402 metaslab_group_taskq (usermode) 5 0% 100% 0.00 1110 UPVO entry (usermode) 4 0% 100% 0.00 14539 vnode interlock 0xc0000000030c6708 4 0% 100% 0.00 13192 ttymtx ttyoutq_read_uio+0x34c 4 0% 100% 0.00 88289 select mtxpool selrecord+0x1dc 4 0% 100% 0.00 9982 nvme qpair lock nvme_qpair_submit_request+0x120 4 0% 100% 0.00 1355 UPVO entry uma_zfree_arg+0x5c8 3 0% 100% 0.00 6168 vm page free queue 0xc000000003109cc8 3 0% 100% 0.00 131157 vm page vm_page_rename+0x224 3 0% 100% 0.00 19929 vm page vm_object_madvise+0x54c 3 0% 100% 0.00 2007 vm page 0xc0000000030e6b40 3 0% 100% 0.00 4373 vm inactive pagequeue 0xc0000000031072a8 3 0% 100% 0.00 679 ttymtx 0xc000000002db0ba0 3 0% 100% 0.00 2869 struct mount vlist mtx (usermode) 3 0% 100% 0.00 10610 select mtxpool 0xc000000002d8b728 3 0% 100% 0.00 4777 pipe mutex 0xc000000002d8bf18 3 0% 100% 0.00 11567 metaslab_group_taskq 0xc000000002d74f5c 3 0% 100% 0.00 15009 metaslab_group_taskq taskqueue_enqueue+0x158 3 0% 100% 0.00 1109 UPVO entry uma_zalloc_arg+0x98c 3 0% 100% 0.00 3408 CAM device lock 0xc000000002524924 3 0% 100% 0.00 3251 CAM device lock 0xc000000002523fac 3 0% 100% 0.00 8571 CAM device lock (usermode) 2 0% 100% 0.00 958 umtxql 0xc000000002cbb898 2 0% 100% 0.00 713 struct mount vlist mtx _vdrop+0x6f4 2 0% 100% 0.00 1025 struct mount vlist mtx _vhold+0x474 2 0% 100% 0.00 16198 struct mount mtx vfs_rel+0x128 2 0% 100% 0.00 8844 process lock kthread_suspend_check+0x184 2 0% 100% 0.00 23011 pipe mutex (usermode) 2 0% 100% 0.00 12320 page pv moea64_page_wired_mappings+0x1b8 2 0% 100% 0.00 2065 kernel arena domain vmem_xfree+0x39c 1 0% 100% 0.00 975 zio_null_issue taskqueue_enqueue+0x158 1 0% 100% 0.00 280 zio_free_issue_4 0xc000000002cbb898 1 0% 100% 0.00 488 zio_free_issue_3 taskqueue_enqueue+0x158 1 0% 100% 0.00 172 zio_free_issue_3 0xc000000002d74f5c 1 0% 100% 0.00 989 zio_free_issue_0 taskqueue_enqueue+0x158 1 0% 100% 0.00 12338 zio_free_issue_2 0xc000000002cbb898 1 0% 100% 0.00 505 zio_free_issue_2 taskqueue_enqueue+0x158 1 0% 100% 0.00 217 vnode interlock 0xc000000002e02370 1 0% 100% 0.00 246 vnode interlock 0xc0000000030c612c 1 0% 100% 0.00 55365 vm page exec_map_first_page+0x7b8 1 0% 100% 0.00 1217 vm active pagequeue 0xc00000000310d2e8 1 0% 100% 0.00 11917 vm page vm_imgact_unmap_page+0x184 1 0% 100% 0.00 668 ttymtx 0xc000000002dafc0c 1 0% 100% 0.00 232 thread_ctor 0xc000000002cc2318 1 0% 100% 0.00 5363 ttymtx 0xc000000002cbb898 1 0% 100% 0.00 22423 thread_ctor (usermode) 1 0% 100% 0.00 22281 umtxql 0xc000000002d1c97c 1 0% 100% 0.00 959 umtxql 0xc000000002d1b514 1 0% 100% 0.00 23197 umtxql 0xc000000002d1db10 1 0% 100% 0.00 3114 umtxql 0xc000000002d1db54 1 0% 100% 0.00 504 metaslab_group_taskq 0xc000000002cbb898 1 0% 100% 0.00 3042 kernel arena domain vmem_xalloc+0x8c0 1 0% 100% 0.00 316 SLB table moea64_get_unique_vsid+0x2a8 1 0% 100% 0.00 242 CAM queue lock 0xc000000002511be8 1 0% 100% 0.00 303 CAM queue lock (usermode) 1 0% 100% 0.00 436 UMA Slabs (usermode) 1 0% 100% 0.00 90890 pmap moea64_is_prefaultable+0x13c ------------------------------------------------------------------------------- Adaptive mutex block: 11 events in 16.400 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 3 27% 27% 0.00 47037 page pv (usermode) 2 18% 45% 0.00 181955 zio_write_issue 0xc000000002d74f5c 2 18% 64% 0.00 28662 zio_write_intr taskqueue_enqueue+0x158 1 9% 73% 0.00 45345 page pv moea64_remove_pages+0x214 1 9% 82% 0.00 187428 nvme qpair lock 0xc000000002a0bae4 1 9% 91% 0.00 23681 kernelpmap (usermode) 1 9% 100% 0.00 32100 dp_sync_taskq 0xc000000002cbb898 ------------------------------------------------------------------------------- Spin lock spin: 18717 events in 16.400 seconds (1141 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 9076 48% 48% 0.00 1291506 turnstile chain (usermode) 8120 43% 92% 0.00 1111650 turnstile chain turnstile_trywait+0x208 586 3% 95% 0.00 8509 sleepq chain sleepq_lock+0xe4 425 2% 97% 0.00 8159 sleepq chain (usermode) 234 1% 99% 0.00 643007 turnstile chain turnstile_chain_lock+0xdc 82 0% 99% 0.00 1979 turnstile lock turnstile_trywait+0x150 60 0% 99% 0.00 679 turnstile lock (usermode) 12 0% 99% 0.00 20504 sched lock 15 0xc000000002d353b4 8 0% 99% 0.00 3437 sched lock 0 0xc000000002d353b4 7 0% 99% 0.00 24597 sched lock 20 0xc000000002d353b4 6 0% 99% 0.00 16540 sched lock 20 (usermode) 5 0% 99% 0.00 30563 sched lock 45 0xc000000002d353cc 5 0% 100% 0.00 10525 sched lock 35 (usermode) 5 0% 100% 0.00 18825 sched lock 15 (usermode) 5 0% 100% 0.00 52358 sched lock 10 0xc000000002d353b4 4 0% 100% 0.00 32012 sched lock 30 0xc000000002d353cc 4 0% 100% 0.00 14925 sched lock 30 (usermode) 3 0% 100% 0.00 4038 turnstile lock turnstile_lookup+0xd4 3 0% 100% 0.00 10998 sched lock 50 0xc000000002d353cc 3 0% 100% 0.00 11163 sched lock 45 (usermode) 3 0% 100% 0.00 34187 sched lock 35 0xc000000002d353cc 3 0% 100% 0.00 28005 sched lock 30 0xc000000002d353b4 3 0% 100% 0.00 509 callout (usermode) 2 0% 100% 0.00 11740 sched lock 55 0xc000000002d353cc 2 0% 100% 0.00 13603 sched lock 50 0xc000000002d353b4 2 0% 100% 0.00 6002 sched lock 45 sched_add+0x48c 2 0% 100% 0.00 11213 sched lock 40 0xc000000002d353cc 2 0% 100% 0.00 3383 sched lock 35 0xc000000002d353b4 2 0% 100% 0.00 2972 sched lock 30 sched_switch+0x81c 2 0% 100% 0.00 13870 sched lock 30 sched_switch+0xca0 2 0% 100% 0.00 19993 sched lock 20 0xc000000002d353cc 2 0% 100% 0.00 11435 sched lock 20 sched_switch+0x81c 2 0% 100% 0.00 3000 sched lock 10 (usermode) 2 0% 100% 0.00 1151 sched lock 0 (usermode) 2 0% 100% 0.00 11268 callout 0xc000000002d1384c 1 0% 100% 0.00 396 sched lock 66 (usermode) 1 0% 100% 0.00 224 sched lock 71 sched_add+0x48c 1 0% 100% 0.00 10362 sched lock 71 0xc000000002d353cc 1 0% 100% 0.00 2771 sched lock 61 sched_add+0x48c 1 0% 100% 0.00 156 sched lock 60 sched_add+0x48c 1 0% 100% 0.00 179 sched lock 57 sched_add+0x48c 1 0% 100% 0.00 2816 sched lock 62 sched_add+0x48c 1 0% 100% 0.00 82332 sched lock 65 (usermode) 1 0% 100% 0.00 957 sched lock 65 sched_add+0x48c 1 0% 100% 0.00 187 sched lock 50 (usermode) 1 0% 100% 0.00 596 sched lock 45 0xc000000002d353b4 1 0% 100% 0.00 5861 sched lock 45 sched_switch+0x81c 1 0% 100% 0.00 172 sched lock 45 sched_switch+0xca0 1 0% 100% 0.00 3050 sched lock 50 sched_switch+0xca0 1 0% 100% 0.00 5858 sched lock 50 sched_switch+0x81c 1 0% 100% 0.00 331 sched lock 55 (usermode) 1 0% 100% 0.00 1644 sched lock 56 sched_add+0x48c 1 0% 100% 0.00 1734 sched lock 43 sched_add+0x48c 1 0% 100% 0.00 566 sched lock 43 (usermode) 1 0% 100% 0.00 230 sched lock 40 0xc000000002d353b4 1 0% 100% 0.00 609 sched lock 20 sched_add+0x48c 1 0% 100% 0.00 88334 sched lock 15 0xc000000002d353cc 1 0% 100% 0.00 625 sched lock 2 sched_add+0x48c 1 0% 100% 0.00 1008 sched lock 25 0xc000000002d353b4 1 0% 100% 0.00 211 sched lock 22 sched_add+0x48c 1 0% 100% 0.00 136 sched lock 26 0xc000000002d353cc 1 0% 100% 0.00 1049 sched lock 26 sched_add+0x48c 1 0% 100% 0.00 119 sched lock 28 sched_add+0x48c 1 0% 100% 0.00 39573 sched lock 28 0xc000000002d353b4 1 0% 100% 0.00 9300 sched lock 0 sched_add+0x48c 1 0% 100% 0.00 700 sched lock 12 sched_add+0x48c ------------------------------------------------------------------------------- Thread lock spin: 238 events in 16.400 seconds (15 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 85 36% 36% 0.00 4923 sleepq chain _thread_lock+0xf4 18 8% 43% 0.00 5887 sleepq chain (usermode) 13 5% 49% 0.00 4194 sched lock 30 _thread_lock+0xf4 11 5% 53% 0.00 10867 sched lock 15 _thread_lock+0xf4 8 3% 57% 0.00 6973 sched lock 35 _thread_lock+0xf4 8 3% 60% 0.00 4886 sched lock 10 _thread_lock+0xf4 7 3% 63% 0.00 5429 sched lock 45 _thread_lock+0xf4 6 3% 66% 0.00 4828 sched lock 55 _thread_lock+0xf4 6 3% 68% 0.00 6292 sched lock 20 _thread_lock+0xf4 5 2% 70% 0.00 3294 turnstile lock _thread_lock+0xf4 5 2% 72% 0.00 2102 sched lock 50 _thread_lock+0xf4 5 2% 74% 0.00 10444 sched lock 0 _thread_lock+0xf4 4 2% 76% 0.00 4286 sched lock 0 (usermode) 3 1% 77% 0.00 6579 sched lock 60 _thread_lock+0xf4 3 1% 79% 0.00 3873 sched lock 40 _thread_lock+0xf4 2 1% 79% 0.00 1138 sched lock 9 (usermode) 2 1% 80% 0.00 15415 sched lock 71 (usermode) 2 1% 81% 0.00 19898 sched lock 57 (usermode) 2 1% 82% 0.00 21678 sched lock 56 (usermode) 2 1% 83% 0.00 5644 sched lock 5 _thread_lock+0xf4 2 1% 84% 0.00 641 sched lock 43 _thread_lock+0xf4 2 1% 84% 0.00 11147 sched lock 32 _thread_lock+0xf4 2 1% 85% 0.00 22533 sched lock 31 _thread_lock+0xf4 2 1% 86% 0.00 1153 sched lock 3 (usermode) 2 1% 87% 0.00 10381 sched lock 26 _thread_lock+0xf4 2 1% 88% 0.00 4320 sched lock 25 _thread_lock+0xf4 2 1% 89% 0.00 16975 sched lock 20 (usermode) 2 1% 89% 0.00 1955 sched lock 15 (usermode) 2 1% 90% 0.00 11503 sched lock 13 _thread_lock+0xf4 1 0% 91% 0.00 21394 sched lock 66 _thread_lock+0xf4 1 0% 91% 0.00 10490 sched lock 65 (usermode) 1 0% 92% 0.00 44701 sched lock 61 _thread_lock+0xf4 1 0% 92% 0.00 689 sched lock 62 _thread_lock+0xf4 1 0% 92% 0.00 326 sched lock 59 (usermode) 1 0% 93% 0.00 578 sched lock 59 _thread_lock+0xf4 1 0% 93% 0.00 445 sched lock 56 _thread_lock+0xf4 1 0% 94% 0.00 375 sched lock 39 (usermode) 1 0% 94% 0.00 41669 sched lock 34 _thread_lock+0xf4 1 0% 95% 0.00 381 sched lock 34 (usermode) 1 0% 95% 0.00 205 sched lock 42 _thread_lock+0xf4 1 0% 95% 0.00 617 sched lock 44 (usermode) 1 0% 96% 0.00 498 sched lock 47 (usermode) 1 0% 96% 0.00 22515 sched lock 3 _thread_lock+0xf4 1 0% 97% 0.00 47014 sched lock 28 _thread_lock+0xf4 1 0% 97% 0.00 41339 sched lock 26 (usermode) 1 0% 97% 0.00 876 sched lock 28 (usermode) 1 0% 98% 0.00 39672 sched lock 2 (usermode) 1 0% 98% 0.00 766 sched lock 21 (usermode) 1 0% 99% 0.00 511 sched lock 22 (usermode) 1 0% 99% 0.00 462 sched lock 11 _thread_lock+0xf4 1 0% 100% 0.00 518 sched lock 17 _thread_lock+0xf4 1 0% 100% 0.00 537 sched lock 19 _thread_lock+0xf4 ------------------------------------------------------------------------------- R/W writer blocked by writer: 680 events in 16.400 seconds (41 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 132 19% 19% 0.00 546132233 vm object vm_fault_hold+0x1b28 127 19% 38% 0.00 509441143 vm object vm_map_insert+0x758 93 14% 52% 0.00 550821044 vm object vm_object_reference+0x134 91 13% 65% 0.00 446018882 vm object vm_object_deallocate+0x800 79 12% 77% 0.00 425731953 vm object (usermode) 45 7% 83% 0.00 578841281 vm object vm_object_shadow+0x390 42 6% 90% 0.00 472487511 vm object vm_object_shadow+0x378 32 5% 94% 0.00 536791352 vm object 0xc0000000030f0be8 22 3% 98% 0.00 457408741 vm object vm_imgact_map_page+0x3f0 8 1% 99% 0.00 394144652 vm object exec_map_first_page+0x780 6 1% 100% 0.00 401063 vm object 0xc000000003117de8 2 0% 100% 0.00 294693227 vm object vm_object_deallocate+0x814 1 0% 100% 0.00 754913 kernel vm object kmem_back_domain+0x340 ------------------------------------------------------------------------------- R/W writer blocked by readers: 1043 events in 16.400 seconds (64 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 439 42% 42% 0.00 414830728 vm object vm_fault_hold+0x1b28 142 14% 56% 0.00 386479094 vm object (usermode) 99 9% 65% 0.00 392947862 vm object vm_object_deallocate+0x800 81 8% 73% 0.00 331881362 vm object 0xc0000000030f0be8 79 8% 81% 0.00 425782575 vm object vnode_create_vobject+0x18c 72 7% 87% 0.00 473069007 vm object vm_object_deallocate+0x814 49 5% 92% 0.00 455202234 vm object vm_object_reference+0x134 24 2% 94% 0.00 509038307 vm object vm_object_shadow+0x378 24 2% 97% 0.00 681525049 vm object vm_imgact_map_page+0x3f0 23 2% 99% 0.00 515047690 vm object vm_map_insert+0x758 8 1% 100% 0.00 440593199 vm object vm_object_shadow+0x390 2 0% 100% 0.00 495660 vm object 0xc000000003117de8 1 0% 100% 0.00 219178424 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader blocked by writer: 218 events in 16.400 seconds (13 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 179 82% 82% 0.00 272247560 vm object __rw_rlock_int+0x9c 39 18% 100% 0.00 166606684 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 37 events in 16.400 seconds (2 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 33 89% 89% 0.00 311333698 vm object __rw_rlock_int+0x9c 4 11% 100% 0.00 174195925 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 1324 events in 16.400 seconds (81 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 219 17% 17% 0.00 11421608 vm object vm_fault_hold+0x1b28 189 14% 31% 0.00 10814909 vm object vm_map_insert+0x758 142 11% 42% 0.00 9745263 vm object vm_object_deallocate+0x800 139 10% 52% 0.00 9867901 vm object (usermode) 134 10% 62% 0.00 13252336 vm object vm_object_reference+0x134 63 5% 67% 0.00 8831997 vm object vm_object_shadow+0x378 62 5% 72% 0.00 13291636 vm object vm_object_shadow+0x390 49 4% 75% 0.00 13412213 vm object 0xc0000000030f0be8 45 3% 79% 0.00 63434 vm object 0xc000000003117de8 36 3% 81% 0.00 15075 vm object vm_fault_hold+0x2700 33 2% 84% 0.00 8899604 vm object vm_imgact_map_page+0x3f0 31 2% 86% 0.00 480747 kernel vm object kmem_back_domain+0x340 30 2% 89% 0.00 72744 Per-Filesystem Softdep Lock 0xc000000002ce4a60 25 2% 90% 0.00 48615 Per-Filesystem Softdep Lock (usermode) 23 2% 92% 0.00 507608 kernel vm object 0xc0000000030e8b68 23 2% 94% 0.00 1026559 kernel vm object (usermode) 13 1% 95% 0.00 44658 Per-Filesystem Softdep Lock softdep_freefile+0xbb4 10 1% 96% 0.00 12500170 vm object exec_map_first_page+0x780 7 1% 96% 0.00 44764 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c 6 0% 97% 0.00 27496 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 5 0% 97% 0.00 2383380 vm object vm_object_deallocate+0x814 5 0% 97% 0.00 42956 Per-Filesystem Softdep Lock 0xc000000003097404 5 0% 98% 0.00 21682 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 4 0% 98% 0.00 25845 Per-Filesystem Softdep Lock 0xc00000000308da54 3 0% 98% 0.00 20972 Per-Filesystem Softdep Lock 0xc000000003094844 3 0% 98% 0.00 17393 Per-Filesystem Softdep Lock 0xc000000003087ebc 2 0% 99% 0.00 946 vm object bdwrite+0x2f4 2 0% 99% 0.00 11233 bufobj interlock (usermode) 2 0% 99% 0.00 81294 Per-Filesystem Softdep Lock softdep_slowdown+0x374 1 0% 99% 0.00 21818 vm object vm_object_collapse+0x710 1 0% 99% 0.00 44341 vm object bufdone+0x45c 1 0% 99% 0.00 22310 bufobj interlock 0xc0000000030b1b10 1 0% 99% 0.00 5833 bufobj interlock vn_fsync_buf+0x244 1 0% 99% 0.00 1093 bufobj interlock bqrelse+0x2a0 1 0% 99% 0.00 12976 Per-Filesystem Softdep Lock 0xc0000000030a1778 1 0% 99% 0.00 41747 Per-Filesystem Softdep Lock 0xc0000000030a5ca8 1 0% 100% 0.00 1738 vm object vfs_busy_pages+0x2a8 1 0% 100% 0.00 187912 Per-Filesystem Softdep Lock 0xc000000003086e70 1 0% 100% 0.00 7400 Per-Filesystem Softdep Lock 0xc000000003085470 1 0% 100% 0.00 20629 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 1 0% 100% 0.00 25251 Per-Filesystem Softdep Lock 0xc00000000308705c 1 0% 100% 0.00 39580 Per-Filesystem Softdep Lock softdep_freefile+0x58c 1 0% 100% 0.00 15885 Per-Filesystem Softdep Lock 0xc00000000309f9b8 ------------------------------------------------------------------------------- R/W writer spin on readers: 1129 events in 16.400 seconds (69 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 450 40% 40% 0.00 13424291 vm object vm_fault_hold+0x1b28 155 14% 54% 0.00 11241646 vm object (usermode) 108 10% 63% 0.00 12571036 vm object vm_object_deallocate+0x800 84 7% 71% 0.00 11946721 vm object 0xc0000000030f0be8 83 7% 78% 0.00 13238596 vm object vnode_create_vobject+0x18c 75 7% 85% 0.00 14024989 vm object vm_object_deallocate+0x814 52 5% 89% 0.00 15812548 vm object vm_object_reference+0x134 29 3% 92% 0.00 14416003 vm object vm_map_insert+0x758 26 2% 94% 0.00 12436745 vm object vm_object_shadow+0x378 24 2% 96% 0.00 24570819 vm object vm_imgact_map_page+0x3f0 13 1% 97% 0.00 39282 vm object vm_fault_hold+0x1298 9 1% 98% 0.00 13270148 vm object vm_object_shadow+0x390 9 1% 99% 0.00 27919 vm object vm_fault_hold+0x2700 4 0% 99% 0.00 191983 vm object 0xc000000003117de8 2 0% 99% 0.00 6548 bufobj interlock reassignbuf+0x350 2 0% 100% 0.00 12530 bufobj interlock (usermode) 1 0% 100% 0.00 28501 vm object vmspace_fork+0x2d4 1 0% 100% 0.00 13568 bufobj interlock bqrelse+0x2a0 1 0% 100% 0.00 28735 bufobj interlock 0xc0000000030b1b10 1 0% 100% 0.00 5484508 vm object exec_map_first_page+0x780 ------------------------------------------------------------------------------- R/W reader spin on writer: 392 events in 16.400 seconds (24 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 305 78% 78% 0.00 3256182 vm object __rw_rlock_int+0x9c 84 21% 99% 0.00 1702592 vm object (usermode) 2 1% 100% 0.00 42399 bufobj interlock (usermode) 1 0% 100% 0.00 26433 bufobj interlock __rw_rlock_int+0x9c ------------------------------------------------------------------------------- R/W reader spin on write wanted: 39 events in 16.400 seconds (2 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 33 85% 85% 0.00 5889453 vm object __rw_rlock_int+0x9c 5 13% 97% 0.00 1353772 vm object (usermode) 1 3% 100% 0.00 46000 bufobj interlock __rw_rlock_int+0x9c ------------------------------------------------------------------------------- SX exclusive block: 40 events in 16.400 seconds (2 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 24 60% 60% 0.00 130621 vm map (user) (usermode) 6 15% 75% 0.00 65840 vm map (user) _vm_map_lock+0x138 3 8% 82% 0.00 77651 vq->vq_lock vdev_queue_io+0x268 3 8% 90% 0.00 146883 vm map (user) _vm_map_lock_upgrade+0x180 2 5% 95% 0.00 31626 vq->vq_lock (usermode) 1 2% 98% 0.00 33204 vq->vq_lock vdev_queue_io_done+0x278 1 2% 100% 0.00 20393 as->as_lock aggsum_compare+0x3dc ------------------------------------------------------------------------------- SX shared block: 6 events in 16.400 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 5 83% 83% 0.00 144659 vm map (user) _sx_slock_int+0xa0 1 17% 100% 0.00 65584 vm map (user) (usermode) ------------------------------------------------------------------------------- SX exclusive spin: 3303 events in 16.400 seconds (201 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 534 16% 16% 0.00 234509 tm->tm_lock trim_map_free+0x19c 276 8% 25% 0.00 217786 vq->vq_lock vdev_queue_io+0x268 240 7% 32% 0.00 395651 os->os_userused_lock (usermode) 218 7% 38% 0.00 271292 vq->vq_lock (usermode) 157 5% 43% 0.00 105454 vm map (user) (usermode) 146 4% 48% 0.00 51538 zio->io_lock (usermode) 135 4% 52% 0.00 249892 tm->tm_lock (usermode) 122 4% 55% 0.00 61909 vm map (user) _vm_map_lock+0x138 113 3% 59% 0.00 8285 rrl->rr_lock 0xc00000000260a56c 112 3% 62% 0.00 50280 zio->io_lock 0xc0000000026bf498 97 3% 65% 0.00 62940 ms->ms_lock metaslab_alloc_dva+0x1138 80 2% 68% 0.00 55507 mg->mg_lock metaslab_alloc_dva+0x3a8 78 2% 70% 0.00 36922 vq->vq_lock vdev_queue_io_done+0x278 67 2% 72% 0.00 423903 os->os_userused_lock 0xc0000000025b7700 66 2% 74% 0.00 71249 mg->mg_lock metaslab_alloc_dva+0x7d4 66 2% 76% 0.00 42647 as->as_lock aggsum_add+0x360 65 2% 78% 0.00 31410 vq->vq_lock vdev_queue_io_done+0x12c 61 2% 80% 0.00 8840 rrl->rr_lock rrw_exit+0x310 55 2% 81% 0.00 32253 zio->io_lock zio_add_child+0x274 54 2% 83% 0.00 66575 mg->mg_lock (usermode) 51 2% 85% 0.00 67103 scl->scl_lock spa_config_enter+0x140 47 1% 86% 0.00 2251 zp->z_acl_lock 0xc000000002678cac 44 1% 87% 0.00 46560 scl->scl_lock spa_config_exit+0x84 44 1% 89% 0.00 472024 os->os_userused_lock 0xc0000000025b7898 43 1% 90% 0.00 54351 zio->io_lock 0xc0000000026c1474 42 1% 91% 0.00 83816 ms->ms_lock (usermode) 30 1% 92% 0.00 51794 spa->spa_alloc_locks[i] 0xc0000000026bcf70 30 1% 93% 0.00 22419 db->db_mtx (usermode) 28 1% 94% 0.00 38682 proctree kern_wait6+0x3e8 26 1% 95% 0.00 8165 rrl->rr_lock (usermode) 24 1% 95% 0.00 499076 scl->scl_lock (usermode) 23 1% 96% 0.00 25934 db->db_mtx dmu_buf_will_dirty+0x2e8 17 1% 97% 0.00 85334 vm map (user) _vm_map_lock_upgrade+0x180 17 1% 97% 0.00 11007 mc->mc_lock metaslab_class_throttle_unreserve+0x154 15 0% 98% 0.00 126850 vq->vq_lock 0xc000000002655fb4 12 0% 98% 0.00 39170 spa->spa_alloc_locks[i] (usermode) 9 0% 98% 0.00 17627 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 7 0% 98% 0.00 2372 dn->dn_struct_rwlock 0xc0000000025a6044 7 0% 99% 0.00 70146 as->as_lock aggsum_compare+0x3dc 6 0% 99% 0.00 9667 zp->z_acl_lock (usermode) 4 0% 99% 0.00 955 dp->dp_lock dsl_pool_undirty_space+0x18c 3 0% 99% 0.00 1431 ms->ms_lock metaslab_free_concrete+0x2e8 3 0% 99% 0.00 14619 h->hash_mutexes[i] dbuf_find+0x114 3 0% 99% 0.00 8013 dd->dd_lock dsl_dir_tempreserve_clear+0x88 3 0% 99% 0.00 1620 bpl->bpl_lock (usermode) 3 0% 99% 0.00 14827 as->as_lock (usermode) 2 0% 99% 0.00 5785 zio->io_lock 0xc0000000026bda18 2 0% 100% 0.00 2931 mc->mc_lock (usermode) 2 0% 100% 0.00 3368 ds->ds_lock dsl_dataset_block_kill+0x614 2 0% 100% 0.00 11334 dn->dn_mtx 0xc0000000025a605c 1 0% 100% 0.00 45032 zio->io_lock 0xc0000000026be2dc 1 0% 100% 0.00 552 zio->io_lock 0xc0000000026bda48 1 0% 100% 0.00 6007 tm->tm_lock trim_map_write_start+0x3f8 1 0% 100% 0.00 409968 proctree (usermode) 1 0% 100% 0.00 1553 hdl->sa_lock (usermode) 1 0% 100% 0.00 22376 ds->ds_lock dsl_dataset_block_kill+0x5e4 1 0% 100% 0.00 932 h->hash_mutexes[i] (usermode) 1 0% 100% 0.00 20672 ds->ds_lock dsl_dataset_block_born+0x3d4 1 0% 100% 0.00 21988 ds->ds_lock (usermode) 1 0% 100% 0.00 443 dp->dp_lock (usermode) 1 0% 100% 0.00 44733 dn->dn_struct_rwlock (usermode) 1 0% 100% 0.00 593 as->as_buckets[i].asc_lock aggsum_add+0x330 ------------------------------------------------------------------------------- SX shared spin: 325 events in 16.400 seconds (20 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 275 85% 85% 0.00 57027 vm map (user) _sx_slock_int+0xa0 44 14% 98% 0.00 75634 vm map (user) (usermode) 3 1% 99% 0.00 15512 dn->dn_struct_rwlock (usermode) 2 1% 100% 0.00 603 dn->dn_struct_rwlock _sx_slock_int+0xa0 1 0% 100% 0.00 1384 zf->zf_rwlock _sx_slock_int+0xa0 -------------------------------------------------------------------------------