-------------------------------------------------------------- >>> Kernel build for GENERIC64 completed on Tue May 28 22:31:23 CDT 2019 -------------------------------------------------------------- >>> Kernel(s) GENERIC64 built in 461 seconds, ncpu: 72, make -j72 -------------------------------------------------------------- Adaptive mutex spin: 14825458 events in 462.353 seconds (32065 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 5024410 34% 34% 0.00 24138 page pv moea64_enter+0x904 4723272 32% 66% 0.00 25246 page pv moea64_remove_pages+0x214 3240434 22% 88% 0.00 28038 page pv (usermode) 251213 2% 89% 0.00 40786 pmap moea64_enter+0x918 159516 1% 90% 0.00 26062 page pv moea64_remove+0x22c 159227 1% 91% 0.00 9904 vm active pagequeue 0xc000000003109348 116624 1% 92% 0.00 11355 vm page vm_page_change_lock+0x160 74649 1% 93% 0.00 739385 process lock ast+0x7e0 71878 0% 93% 0.00 61048 pmap (usermode) 61291 0% 94% 0.00 19957 umtxql (usermode) 50868 0% 94% 0.00 19594 page pv moea64_enter+0x9a8 45087 0% 94% 0.00 53728 kernelpmap moea64_enter+0x918 41760 0% 95% 0.00 70811 kernelpmap moea64_kextract+0x174 35812 0% 95% 0.00 13535 vm page vm_fault_hold+0x2724 32743 0% 95% 0.00 70215 kernelpmap (usermode) 32364 0% 95% 0.00 131934 pmap moea64_is_prefaultable+0x13c 31578 0% 95% 0.00 361476 process lock kern_thr_exit+0x12c 30851 0% 96% 0.00 274927 process lock kern_thr_exit+0x24c 30086 0% 96% 0.00 11049 vm active pagequeue (usermode) 29121 0% 96% 0.00 21249 umtxql 0xc000000002d1cc8c 28927 0% 96% 0.00 117271 zio_write_issue 0xc000000002d74ef8 27746 0% 96% 0.00 12963 vm page (usermode) 26972 0% 97% 0.00 7933 vnode interlock vget+0x2f8 26906 0% 97% 0.00 52202 kernelpmap moea64_remove+0x420 26362 0% 97% 0.00 38749 dp_sync_taskq (usermode) 26187 0% 97% 0.00 43701 struct mount mtx (usermode) 25224 0% 97% 0.00 44122 dp_sync_taskq 0xc000000002d74ef8 21989 0% 97% 0.00 63272 struct mount mtx vfs_unbusy+0x15c 21791 0% 98% 0.00 35306 struct mount vlist mtx _vdrop+0x6f4 21545 0% 98% 0.00 12978 umtxql 0xc000000002d1af84 21072 0% 98% 0.00 95882 zio_write_issue 0xc000000002cbb944 20032 0% 98% 0.00 33160 dp_sync_taskq 0xc000000002cbb944 19650 0% 98% 0.00 34683 struct mount vlist mtx _vhold+0x474 19173 0% 98% 0.00 50101 struct mount mtx vfs_busy+0x434 19162 0% 98% 0.00 173755 zio_write_issue taskqueue_enqueue+0x158 18904 0% 99% 0.00 63223 kernelpmap moea64_kenter_attr+0x550 16666 0% 99% 0.00 166036 zio_write_issue (usermode) 16557 0% 99% 0.00 28250 zio_write_intr (usermode) 15825 0% 99% 0.00 959888 process lock (usermode) 15400 0% 99% 0.00 14078 vnode interlock (usermode) 13047 0% 99% 0.00 27482 umtxql 0xc000000002d1dab0 11589 0% 99% 0.00 30618 dp_sync_taskq taskqueue_enqueue+0x158 9660 0% 99% 0.00 30060 zio_write_intr 0xc000000002cbb944 9325 0% 99% 0.00 20108 struct mount mtx vfs_rel+0x128 9067 0% 99% 0.00 15643 umtxql 0xc000000002d1b230 9020 0% 99% 0.00 16653 vnode interlock 0xc000000002e1ab28 8919 0% 99% 0.00 108891 struct mount mtx vfs_ref+0x114 7174 0% 100% 0.00 32774 zio_write_intr 0xc000000002d74ef8 6664 0% 100% 0.00 9690 vm active pagequeue vm_page_dequeue+0x1b4 5999 0% 100% 0.00 24407 zio_write_intr taskqueue_enqueue+0x158 5832 0% 100% 0.00 15576 umtxql 0xc000000002d1dae0 3367 0% 100% 0.00 9426 select mtxpool selrecord+0x1dc 3317 0% 100% 0.00 23753 umtxql 0xc000000002cbb944 2821 0% 100% 0.00 15486 vnode interlock 0xc0000000030c87a8 2744 0% 100% 0.00 1648 UPVO entry uma_zfree_arg+0x5c8 2476 0% 100% 0.00 22086 page pv moea64_page_wired_mappings+0x1b8 2232 0% 100% 0.00 22562 vnode interlock 0xc000000002e1ab0c 2110 0% 100% 0.00 19734 umtxql 0xc000000002d1db24 2090 0% 100% 0.00 10676 vm page 0xc0000000030e64cc 1761 0% 100% 0.00 3286 UPVO entry uma_zalloc_arg+0x98c 1569 0% 100% 0.00 24164 page pv 0xc00000000313df54 1225 0% 100% 0.00 63377 struct mount vlist mtx (usermode) 1121 0% 100% 0.00 14789 struct mount vlist mtx vfs_msync+0x1d4 1044 0% 100% 0.00 6887 vnode interlock vref+0x1e8 917 0% 100% 0.00 11253 pipe mutex 0xc000000002d8cbf8 915 0% 100% 0.00 5946 ttymtx 0xc000000002db0b3c 852 0% 100% 0.00 13673 vm page free queue 0xc00000000310bd68 850 0% 100% 0.00 3317 umtxql 0xc000000002d1c94c 849 0% 100% 0.00 10662 pipe mutex 0xc000000002d8d700 841 0% 100% 0.00 10091 pipe mutex 0xc000000002d8c0ec 787 0% 100% 0.00 9441 ttymtx (usermode) 706 0% 100% 0.00 8526 vm page 0xc0000000030e8be0 681 0% 100% 0.00 15383 select mtxpool (usermode) 679 0% 100% 0.00 6741 select mtxpool 0xc000000002d8b6c4 624 0% 100% 0.00 8619 ttymtx 0xc000000002dafba8 542 0% 100% 0.00 6804 pipe mutex 0xc000000002d8beb4 437 0% 100% 0.00 2324 pipe mutex 0xc000000002d8dfa0 374 0% 100% 0.00 8752 pipe mutex (usermode) 361 0% 100% 0.00 130467 nvme qpair lock 0xc000000002a0bb90 341 0% 100% 0.00 10970 ttymtx ttyoutq_read_uio+0x34c 339 0% 100% 0.00 8949 vm page dmu_read_pages+0x7b8 336 0% 100% 0.00 14289 vm page free queue 0xc00000000310baa8 331 0% 100% 0.00 10850 ttymtx ttydisc_write+0x388 322 0% 100% 0.00 3540 UPVO entry (usermode) 318 0% 100% 0.00 4353 kernel arena vmem_xalloc+0x8c0 313 0% 100% 0.00 7880 vm page dmu_read_pages+0x360 310 0% 100% 0.00 16192 vm page free queue (usermode) 293 0% 100% 0.00 13794 vnode interlock vinactive+0x25c 281 0% 100% 0.00 21885 umtxql 0xc000000002d1b4e4 278 0% 100% 0.00 11349 struct mount mtx vn_finished_write+0x174 273 0% 100% 0.00 10136 vm active pagequeue 0xc00000000310f388 257 0% 100% 0.00 10597 vm page vm_imgact_unmap_page+0x184 247 0% 100% 0.00 16554 kernel arena domain vmem_xalloc+0x8c0 215 0% 100% 0.00 5964 vnode interlock 0xc0000000030c93f8 208 0% 100% 0.00 14406 struct mount mtx vn_finished_secondary_write+0x174 195 0% 100% 0.00 7939 vm page vm_page_rename+0x224 182 0% 100% 0.00 8035 umtxql kern_umtx_wake+0x324 171 0% 100% 0.00 9451 CAM device lock 0xc000000002524924 166 0% 100% 0.00 32323 dp_sync_taskq taskqueue_quiesce+0x2e8 165 0% 100% 0.00 9044 vm page 0xc000000002df2368 157 0% 100% 0.00 2267 vm inactive pagequeue 0xc000000003109348 155 0% 100% 0.00 279001 vm page exec_unmap_first_page+0x188 145 0% 100% 0.00 9238 vm page exec_map_first_page+0x7b8 141 0% 100% 0.00 11057 CAM device lock (usermode) 128 0% 100% 0.00 15204 vm page vm_fault_hold+0x27ec 127 0% 100% 0.00 18428 metaslab_group_taskq (usermode) 123 0% 100% 0.00 6010 vm page vm_pageout_flush+0x280 122 0% 100% 0.00 6690 vm page vm_imgact_map_page+0x43c 122 0% 100% 0.00 5995 FFS 0xc0000000030c202c 122 0% 100% 0.00 3756 CAM device lock 0xc000000002523fac 121 0% 100% 0.00 6235 vnode interlock 0xc000000002e0230c 119 0% 100% 0.00 4717 vnode interlock 0xc0000000030c81cc 118 0% 100% 0.00 3253 TID lock (usermode) 113 0% 100% 0.00 6384 ttymtx 0xc000000002da9f60 110 0% 100% 0.00 2387 TID lock alloc_unr+0x13c 108 0% 100% 0.00 4477 sellck 0xc000000002d8aefc 105 0% 100% 0.00 1529 pipe mutex 0xc000000002cbb944 102 0% 100% 0.00 2758 UMA Slabs uma_large_malloc_domain+0x2ac 98 0% 100% 0.00 3853 kernel arena (usermode) 93 0% 100% 0.00 9635 struct mount mtx vn_start_write+0x1a0 91 0% 100% 0.00 16647 ncvn (usermode) 87 0% 100% 0.00 8879 FFS (usermode) 86 0% 100% 0.00 4678 vnode interlock _vhold+0x4a4 83 0% 100% 0.00 4665 umtxql 0xc000000002d1c7c0 81 0% 100% 0.00 14001 vm page vm_object_madvise+0x54c 69 0% 100% 0.00 10159 struct mount mtx vfs_notify_upper+0x460 68 0% 100% 0.00 6729 select mtxpool 0xc000000002d8b0f4 67 0% 100% 0.00 1722 vm inactive pagequeue (usermode) 66 0% 100% 0.00 6414 vm page vm_object_unwire+0x2d0 66 0% 100% 0.00 16097 metaslab_group_taskq 0xc000000002d74ef8 66 0% 100% 0.00 19404 kernel arena domain (usermode) 64 0% 100% 0.00 3461 kernel arena domain vmem_xfree+0x39c 60 0% 100% 0.00 19605 vm page free queue vm_page_alloc_domain_after+0x250 60 0% 100% 0.00 9182 metaslab_group_taskq taskqueue_enqueue+0x158 57 0% 100% 0.00 1894 vnode interlock vop_stdset_text+0x130 56 0% 100% 0.00 10434 vnode interlock rangelock_unlock+0xd0 56 0% 100% 0.00 303 pipe mutex 0xc000000002d8f0a8 54 0% 100% 0.00 9657 vm page 0xc0000000030ff640 54 0% 100% 0.00 5626 vm active pagequeue 0xc00000000310f6cc 54 0% 100% 0.00 4858 sellck (usermode) 54 0% 100% 0.00 10105 ncneglh cache_lookup+0x1608 53 0% 100% 0.00 6405 struct mount mtx vn_start_secondary_write+0x1c4 53 0% 100% 0.00 14544 UPVO entry 0xc0000000030ddb58 52 0% 100% 0.00 11639 ncvn 0xc000000002dfd010 51 0% 100% 0.00 1167 struct mount vlist mtx insmntque1+0x420 48 0% 100% 0.00 1078 zio_free_issue_0 taskqueue_enqueue+0x158 47 0% 100% 0.00 8365 struct mount mtx insmntque1+0x3f8 45 0% 100% 0.00 6050 nvme qpair lock nvme_qpair_submit_request+0x120 44 0% 100% 0.00 1585 zio_free_issue_3 taskqueue_enqueue+0x158 44 0% 100% 0.00 23971 pipe mutex 0xc000000002d8f0e0 43 0% 100% 0.00 3756 vnode interlock 0xc000000002cd9228 43 0% 100% 0.00 8234 struct mount mtx 0xc000000002e1c264 41 0% 100% 0.00 1142 zio_free_issue_6 taskqueue_enqueue+0x158 41 0% 100% 0.00 18554 vm page vm_page_sleep_if_busy+0x1c8 41 0% 100% 0.00 13804 sellck 0xc000000002cbb944 41 0% 100% 0.00 6734 bge0 (usermode) 41 0% 100% 0.00 2362 UMA Slabs (usermode) 37 0% 100% 0.00 10455 SLB table moea64_release_vsid+0x144 35 0% 100% 0.00 3146 Global Softdep Lock 0xc00000000308e234 34 0% 100% 0.00 2221 zio_free_issue_7 taskqueue_enqueue+0x158 34 0% 100% 0.00 15538 process lock 0xc000000002c86e34 33 0% 100% 0.00 9392 vm page vm_fault_hold+0x27a8 32 0% 100% 0.00 1090 zio_free_issue_2 taskqueue_enqueue+0x158 32 0% 100% 0.00 1706 UMA Slabs 0xc0000000030e184c 31 0% 100% 0.00 1648 zio_free_issue_1 taskqueue_enqueue+0x158 31 0% 100% 0.00 30776 struct mount vlist mtx __mnt_vnode_next_active+0xdc 31 0% 100% 0.00 526 SLB table moea64_get_unique_vsid+0x2a8 31 0% 100% 0.00 13078 FFS softdep_freefile+0x920 30 0% 100% 0.00 2210 zio_free_issue_4 taskqueue_enqueue+0x158 30 0% 100% 0.00 2587 zio_free_issue_3 0xc000000002cbb944 29 0% 100% 0.00 1254 zio_free_issue_5 taskqueue_enqueue+0x158 28 0% 100% 0.00 39356 umtxql 0xc000000002d1bce8 27 0% 100% 0.00 1501 zio_free_issue_7 0xc000000002cbb944 26 0% 100% 0.00 31017 process lock ast+0x6e0 26 0% 100% 0.00 6938 FFS 0xc0000000030983ac 25 0% 100% 0.00 974 vm inactive pagequeue vm_page_dequeue+0x1b4 24 0% 100% 0.00 518 sleep mtxpool foffset_lock+0x1c0 23 0% 100% 0.00 9699 FFS ffs_freefile+0x358 22 0% 100% 0.00 2291 vnode_free_list _vhold+0x4c0 21 0% 100% 0.00 1610 zio_free_issue_2 0xc000000002cbb944 21 0% 100% 0.00 14414 vm page free queue vm_page_free_toq+0x208 20 0% 100% 0.00 3264 zio_free_issue_6 0xc000000002d74ef8 20 0% 100% 0.00 2443 zio_free_issue_4 0xc000000002cbb944 20 0% 100% 0.00 19869 metaslab_group_taskq 0xc000000002cbb944 20 0% 100% 0.00 15006 FFS 0xc000000003073b1c 20 0% 100% 0.00 1953 CAM queue lock 0xc000000002511be8 19 0% 100% 0.00 1913 zio_free_issue_1 0xc000000002cbb944 19 0% 100% 0.00 8819 umtxql 0xc000000002d246e8 19 0% 100% 0.00 9165 ncvn vn_vptocnp+0x4a4 19 0% 100% 0.00 7330 bge0 0xc000000002824dac 18 0% 100% 0.00 2615 zio_free_issue_7 0xc000000002d74ef8 18 0% 100% 0.00 638 zio_free_issue_6 (usermode) 18 0% 100% 0.00 8619 vm active pagequeue 0xc00000000310f6e0 17 0% 100% 0.00 3076 zio_free_issue_6 0xc000000002cbb944 17 0% 100% 0.00 2257 zio_free_issue_4 0xc000000002d74ef8 17 0% 100% 0.00 4347 vm page 0xc0000000030e5df4 17 0% 100% 0.00 241818 nvme qpair lock (usermode) 17 0% 100% 0.00 1559 ncneglh cache_lookup+0x1638 16 0% 100% 0.00 1577 zio_free_issue_7 (usermode) 16 0% 100% 0.00 955 struct mount vlist mtx 0xc000000002e1c2c0 15 0% 100% 0.00 1959 zio_free_issue_5 0xc000000002d74ef8 14 0% 100% 0.00 4281 zio_free_issue_3 0xc000000002d74ef8 14 0% 100% 0.00 4970 zio_free_issue_1 0xc000000002d74ef8 14 0% 100% 0.00 2533 zio_free_issue_0 0xc000000002cbb944 14 0% 100% 0.00 257562 xhci0 xhci_interrupt+0x428 13 0% 100% 0.00 1657 zio_free_issue_0 0xc000000002d74ef8 13 0% 100% 0.00 14293 umtxql 0xc000000002d28034 13 0% 100% 0.00 11265 so_snd tcp_do_segment+0x3ca0 13 0% 100% 0.00 567 sleep mtxpool foffset_unlock+0x1b0 13 0% 100% 0.00 5051 reseed mutex 0xc000000002a3c9c0 12 0% 100% 0.00 1019 zio_free_issue_2 0xc000000002d74ef8 12 0% 100% 0.00 1836 zio_free_issue_2 (usermode) 12 0% 100% 0.00 2683 zio_free_issue_1 (usermode) 12 0% 100% 0.00 1729 vm page 0xc0000000026a4258 12 0% 100% 0.00 4173 vm page 0xc0000000026a4214 12 0% 100% 0.00 5120 ttymtx 0xc000000002cc4f18 12 0% 100% 0.00 2653 bufq dirty lock (usermode) 12 0% 100% 0.00 57950 UPVO entry 0xc0000000030e0e78 12 0% 100% 0.00 12962 TID lock clean_unrhdrl+0x94 12 0% 100% 0.00 2635 TID lock 0xc000000002d08b1c 12 0% 100% 0.00 6861 Global Softdep Lock (usermode) 11 0% 100% 0.00 21212 vnode interlock 0xc0000000030b496c 11 0% 100% 0.00 558 vm pgcache uma_zfree_arg+0x5c8 11 0% 100% 0.00 6773 umtxql 0xc000000002d197fc 11 0% 100% 0.00 678 so_snd sopoll_generic+0x470 11 0% 100% 0.00 12161 FFS 0xc000000003074f38 10 0% 100% 0.00 604 zio_null_issue taskqueue_enqueue+0x158 10 0% 100% 0.00 2848 zio_free_issue_5 0xc000000002cbb944 10 0% 100% 0.00 3959 zio_free_issue_3 (usermode) 10 0% 100% 0.00 1033 vnode_free_list (usermode) 10 0% 100% 0.00 1296 ncneglh (usermode) 10 0% 100% 0.00 46427 kernelpmap 0xc00000000313dd2c 10 0% 100% 0.00 2755 kernel arena vmem_xfree+0x39c 9 0% 100% 0.00 1260 zio_free_issue_5 (usermode) 9 0% 100% 0.00 778 vnode_free_list getnewvnode_reserve+0x480 9 0% 100% 0.00 22389 vm page vm_page_pa_tryrelock+0x298 9 0% 100% 0.00 6764 ttymtx 0xc000000002cbb944 9 0% 100% 0.00 6130 so_snd sosend_generic+0x3bc 9 0% 100% 0.00 18532 reseed mutex 0xc000000002a3cef8 9 0% 100% 0.00 6881 process_exec exec_new_vmspace+0x138 9 0% 100% 0.00 12600 process_ctor 0xc000000002cc21b4 9 0% 100% 0.00 2977 USB device mutex 0xc000000002b36774 8 0% 100% 0.00 708 zio_null_issue (usermode) 8 0% 100% 0.00 11093 xhci0 0xc000000002cbb944 8 0% 100% 0.00 117729 xhci0 (usermode) 8 0% 100% 0.00 15977 thread_dtor 0xc000000002cc26f8 8 0% 100% 0.00 8089 process lock kthread_suspend_check+0x184 8 0% 100% 0.00 18310 FFS ffs_alloc+0x474 8 0% 100% 0.00 10603 CAM queue lock 0xc00000000250c3e8 7 0% 100% 0.00 14159 umtxql 0xc000000002d280ac 7 0% 100% 0.00 3700 thread_ctor 0xc000000002d08668 7 0% 100% 0.00 19510 reseed mutex (usermode) 7 0% 100% 0.00 1759 process lock exit1+0x1424 7 0% 100% 0.00 11128118 pmap 0xc00000000313dd2c 7 0% 100% 0.00 1623 ncneglh 0xc000000002df91b8 7 0% 100% 0.00 18647 bufq dirty lock 0xc000000002dea138 7 0% 100% 0.00 13860 UMA Slabs 0xc0000000030e0e78 7 0% 100% 0.00 7835 FFS softdep_setup_freeblocks+0xc28 7 0% 100% 0.00 332 CAM queue lock (usermode) 6 0% 100% 0.00 1451 zio_free_issue_4 (usermode) 6 0% 100% 0.00 2131 zio_free_issue_0 (usermode) 6 0% 100% 0.00 7599 vm pgcache uma_zalloc_arg+0x9e4 6 0% 100% 0.00 716 vm pgcache uma_zalloc_arg+0x98c 6 0% 100% 0.00 465 vm map (system) vm_map_lookup+0x15c 6 0% 100% 0.00 722 thread_dtor 0xc000000002cc2aa4 6 0% 100% 0.00 21238 reseed mutex 0xc000000002a3cc80 6 0% 100% 0.00 5272 process lock kern_sigprocmask+0x378 6 0% 100% 0.00 7751 TID lock free_unr+0x45c 6 0% 100% 0.00 3272 SLB table (usermode) 6 0% 100% 0.00 5063 FFS 0xc0000000030a8c14 6 0% 100% 0.00 11525 FFS 0xc0000000030704b4 5 0% 100% 0.00 540 vnode_free_list getnewvnode+0x3cc 5 0% 100% 0.00 16375 vnode interlock zfs_zget+0x37c 5 0% 100% 0.00 682 vm pgcache (usermode) 5 0% 100% 0.00 8588 thread_ctor 0xc000000002d084d4 5 0% 100% 0.00 294 so_snd (usermode) 5 0% 100% 0.00 9522 process_exit (usermode) 5 0% 100% 0.00 1162 ncvn 0xc000000002dfcffc 4 0% 100% 0.00 1090 zio_null_intr taskqueue_enqueue+0x158 4 0% 100% 0.00 5768 umtxql 0xc000000002d19838 4 0% 100% 0.00 5431 thread_dtor 0xc000000002d0884c 4 0% 100% 0.00 1012 thread_ctor 0xc000000002cc256c 4 0% 100% 0.00 6227 thread_ctor 0xc000000002cc23c4 4 0% 100% 0.00 234 procid (usermode) 4 0% 100% 0.00 11588 process_exit exit1+0x564 4 0% 100% 0.00 254 process_ctor 0xc000000002cc2540 4 0% 100% 0.00 15154 process lock kern_sigaction+0x67c 4 0% 100% 0.00 14586 process lock 0xc000000002cbb944 4 0% 100% 0.00 198 bufq dirty lock 0xc000000002de9384 4 0% 100% 0.00 108370 bufq clean lock 0xc000000002df1ba4 3 0% 100% 0.00 386 xhci0 usbd_transfer_pending+0x288 3 0% 100% 0.00 7290 vm page vm_page_xunbusy_hard+0x15c 3 0% 100% 0.00 15021 umtxql 0xc000000002d28098 3 0% 100% 0.00 14738 umtxql 0xc000000002d24694 3 0% 100% 0.00 10807 umtxql 0xc000000002d23f00 3 0% 100% 0.00 2525 so_snd tcp_output+0x1de8 3 0% 100% 0.00 59989 UPVO entry uma_zalloc_arg+0x9e4 3 0% 100% 0.00 10557 FFS 0xc000000003076338 2 0% 100% 0.00 6477 xhci0 usbd_transfer_power_ref+0x364 2 0% 100% 0.00 10672 vnode interlock 0xc0000000030a77c0 2 0% 100% 0.00 4085 vm page vm_fault_hold+0x1098 2 0% 100% 0.00 15083 umtxql 0xc000000002d1bd4c 2 0% 100% 0.00 15319 thread_dtor (usermode) 2 0% 100% 0.00 198 sellck 0xc000000002d8a15c 2 0% 100% 0.00 624 procid fork1+0x279c 2 0% 100% 0.00 14525 process lock sendsig+0x5ac 2 0% 100% 0.00 229655 process lock thread_create+0x6f0 2 0% 100% 0.00 4042191 process lock 0xc000000002cca068 2 0% 100% 0.00 8201 pmap trap+0x145c 2 0% 100% 0.00 1026 ncnegl cache_enter_time+0x1a00 2 0% 100% 0.00 710 ncnegl (usermode) 2 0% 100% 0.00 668 g_disk_done 0xc000000002c20e3c 2 0% 100% 0.00 23045 exec args kva 0xc000000002c82614 2 0% 100% 0.00 6364 Global Softdep Lock 0xc00000000308fd94 2 0% 100% 0.00 265 FFS ffs_balloc_ufs2+0xb8c 2 0% 100% 0.00 934 FFS ffs_balloc_ufs2+0x1b98 2 0% 100% 0.00 1425 256 (usermode) 2 0% 100% 0.00 43622 CAM device lock 0xc00000000252337c 2 0% 100% 0.00 130 CAM queue lock xpt_action_default+0x1254 1 0% 100% 0.00 3082 zio_null_intr 0xc000000002cbb944 1 0% 100% 0.00 12469 xhci0 0xc000000002b0a4fc 1 0% 100% 0.00 11341 xhci0 0xc000000002b08fc0 1 0% 100% 0.00 5830 xhci0 usbd_pipe_enter+0x248 1 0% 100% 0.00 40009 xhci0 0xc000000002b3678c 1 0% 100% 0.00 627 zio_cache (usermode) 1 0% 100% 0.00 230 vm page vm_page_grab_pages+0x48c 1 0% 100% 0.00 1090 thread_dtor 0xc000000002d08b58 1 0% 100% 0.00 21277 umtxql 0xc000000002d23ebc 1 0% 100% 0.00 11296 umtxql 0xc000000002d25bfc 1 0% 100% 0.00 40240 umtxql 0xc000000002d280d4 1 0% 100% 0.00 20130 struct mount vlist mtx __mnt_vnode_first_active+0x298 1 0% 100% 0.00 421 so_snd sbdrop+0x130 1 0% 100% 0.00 217 process_exec exec_new_vmspace+0x5ec 1 0% 100% 0.00 169 process_exit exit1+0x1b10 1 0% 100% 0.00 213 sleep mtxpool g_io_request+0x414 1 0% 100% 0.00 342 sleep mtxpool (usermode) 1 0% 100% 0.00 299 sleep mtxpool g_io_deliver+0x310 1 0% 100% 0.00 432 pipe mutex 0xc000000002d8e5e0 1 0% 100% 0.00 1898 pipe mutex 0xc000000002b6a5f8 1 0% 100% 0.00 2791 pbuf uma_zfree_arg+0x5c8 1 0% 100% 0.00 48419 process lock fork_rfppwait+0x1f0 1 0% 100% 0.00 3048 process lock proc_reap+0xc38 1 0% 100% 0.00 2895 process lock proc_reap+0xda8 1 0% 100% 0.00 23171 process lock exit1+0x1a54 1 0% 100% 0.00 213 process_dtor (usermode) 1 0% 100% 0.00 430 RADIX NODE uma_zfree_arg+0x5c8 1 0% 100% 0.00 371 RADIX NODE (usermode) 1 0% 100% 0.00 183 Global Softdep Lock 0xc0000000030a3ed0 1 0% 100% 0.00 403 Syncer mtx brelvp+0x300 1 0% 100% 0.00 1707 Syncer mtx 0xc000000002e19c78 1 0% 100% 0.00 306 g_disk_start (usermode) 1 0% 100% 0.00 999 exec args kva (usermode) 1 0% 100% 0.00 193 XPT topology lock xpt_lock_buses+0xac 1 0% 100% 0.00 20037 USB device mutex (usermode) 1 0% 100% 0.00 149 cdev devvn_refthread+0x228 1 0% 100% 0.00 24982 ncnegl 0xc000000002df919c 1 0% 100% 0.00 1023 ncnegl cache_lookup+0x15b0 ------------------------------------------------------------------------------- Adaptive mutex block: 2674 events in 462.353 seconds (6 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 687 26% 26% 0.00 490913 page pv moea64_enter+0x904 458 17% 43% 0.00 587603 page pv moea64_remove_pages+0x214 303 11% 54% 0.00 470078 page pv (usermode) 253 9% 64% 0.00 768455 page pv moea64_remove+0x22c 111 4% 68% 0.00 45820 zio_write_intr taskqueue_enqueue+0x158 83 3% 71% 0.00 87976 kernelpmap moea64_enter+0x918 71 3% 74% 0.00 131947 kernelpmap moea64_kextract+0x174 60 2% 76% 0.00 483543 process lock ast+0x7e0 49 2% 78% 0.00 1380185 zio_write_issue 0xc000000002d74ef8 43 2% 79% 0.00 200163 pmap moea64_enter+0x918 41 2% 81% 0.00 319544 vm page vm_page_change_lock+0x160 36 1% 82% 0.00 196702 zio_write_issue taskqueue_enqueue+0x158 36 1% 83% 0.00 933155 zio_write_issue 0xc000000002cbb944 35 1% 85% 0.00 56238 zio_write_intr (usermode) 34 1% 86% 0.00 141877 nvme qpair lock 0xc000000002a0bb90 33 1% 87% 0.00 92164 kernelpmap (usermode) 23 1% 88% 0.00 317652 vm page vm_fault_hold+0x2724 22 1% 89% 0.00 66021 zio_write_intr 0xc000000002cbb944 22 1% 90% 0.00 2462995 umtxql 0xc000000002d1cc8c 20 1% 91% 0.00 320069 zio_write_issue (usermode) 19 1% 91% 0.00 401437 page pv moea64_enter+0x9a8 18 1% 92% 0.00 269753 vm active pagequeue 0xc000000003109348 17 1% 93% 0.00 73144 zio_write_intr 0xc000000002d74ef8 17 1% 93% 0.00 175516 kernelpmap moea64_remove+0x420 16 1% 94% 0.00 858985 process lock (usermode) 13 0% 94% 0.00 239133 vm page (usermode) 12 0% 95% 0.00 1308709 umtxql (usermode) 12 0% 95% 0.00 96865 dp_sync_taskq 0xc000000002d74ef8 12 0% 96% 0.00 73212 dp_sync_taskq 0xc000000002cbb944 9 0% 96% 0.00 27189 xhci0 (usermode) 9 0% 96% 0.00 523437 process lock kern_thr_exit+0x12c 7 0% 97% 0.00 281045 process lock kern_thr_exit+0x24c 6 0% 97% 0.00 154016 pmap moea64_is_prefaultable+0x13c 6 0% 97% 0.00 461902 pmap (usermode) 6 0% 97% 0.00 88255 kernelpmap moea64_kenter_attr+0x550 6 0% 97% 0.00 122637 dp_sync_taskq (usermode) 5 0% 98% 0.00 23662 xhci0 0xc000000002cbb944 5 0% 98% 0.00 253263 vm page 0xc0000000030e64cc 5 0% 98% 0.00 51700 sellck 0xc000000002cbb944 5 0% 98% 0.00 60944 dp_sync_taskq taskqueue_enqueue+0x158 4 0% 98% 0.00 291201 vnode interlock vget+0x2f8 4 0% 98% 0.00 3179649 umtxql 0xc000000002d1af84 4 0% 99% 0.00 51247 ttymtx 0xc000000002cbb944 4 0% 99% 0.00 34791 CAM device lock 0xc000000002524924 3 0% 99% 0.00 23848 umass0 0xc000000002b36774 3 0% 99% 0.00 47261 struct mount mtx (usermode) 3 0% 99% 0.00 51298 CAM queue lock 0xc000000002511be8 2 0% 99% 0.00 45158 vnode interlock (usermode) 2 0% 99% 0.00 94139 vm page 0xc0000000030e8be0 2 0% 99% 0.00 48894 struct mount mtx vfs_busy+0x434 1 0% 99% 0.00 72523 vnode interlock 0xc0000000030b496c 1 0% 99% 0.00 29552 vm page vm_imgact_map_page+0x43c 1 0% 99% 0.00 183428 vm active pagequeue vm_page_dequeue+0x1b4 1 0% 99% 0.00 79073 umtxql 0xc000000002d1b230 1 0% 100% 0.00 444997 umtxql 0xc000000002d1dab0 1 0% 100% 0.00 55425 umtxql 0xc000000002d1dae0 1 0% 100% 0.00 246972 umtxql 0xc000000002d1db24 1 0% 100% 0.00 27099 sleep mtxpool (usermode) 1 0% 100% 0.00 194720 process lock ast+0x6e0 1 0% 100% 0.00 156685 select mtxpool selrecord+0x1dc 1 0% 100% 0.00 59901 struct mount mtx vfs_ref+0x114 1 0% 100% 0.00 3347561 umtxql kern_umtx_wake+0x324 1 0% 100% 0.00 220879 pipe mutex 0xc000000002cbb944 1 0% 100% 0.00 112362 page pv 0xc00000000313df54 1 0% 100% 0.00 15377 kernel arena vmem_xalloc+0x8c0 1 0% 100% 0.00 72566 UMA Slabs uma_large_malloc_domain+0x2ac 1 0% 100% 0.00 39749 CAM device lock (usermode) 1 0% 100% 0.00 46000 FFS 0xc0000000030983ac ------------------------------------------------------------------------------- Spin lock spin: 3020232 events in 462.353 seconds (6532 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1727357 57% 57% 0.00 558008 turnstile chain turnstile_trywait+0x208 903953 30% 87% 0.00 605666 turnstile chain (usermode) 170771 6% 93% 0.00 73154 sleepq chain sleepq_lock+0xe4 105758 4% 96% 0.00 62062 sleepq chain (usermode) 55847 2% 98% 0.00 230035 turnstile chain turnstile_chain_lock+0xdc 19379 1% 99% 0.00 1813 turnstile lock turnstile_trywait+0x150 8474 0% 99% 0.00 1299 turnstile lock (usermode) 1428 0% 99% 0.00 20256 sched lock 0 0xc000000002d35384 751 0% 99% 0.00 6821 turnstile lock turnstile_lookup+0xd4 659 0% 99% 0.00 51970 sched lock 10 0xc000000002d35384 594 0% 99% 0.00 38929 sched lock 5 0xc000000002d35384 549 0% 99% 0.00 33104 sched lock 15 0xc000000002d35384 468 0% 99% 0.00 4161 callout 0xc000000002d1381c 407 0% 99% 0.00 5894 td_contested turnstile_wait+0x298 407 0% 99% 0.00 31618 sched lock 30 0xc000000002d35384 394 0% 99% 0.00 33864 sched lock 25 0xc000000002d35384 369 0% 99% 0.00 31838 sched lock 20 0xc000000002d35384 335 0% 99% 0.00 18920 sched lock 0 (usermode) 318 0% 99% 0.00 31579 sched lock 65 0xc000000002d3539c 308 0% 99% 0.00 38207 sched lock 35 0xc000000002d35384 304 0% 99% 0.00 27135 sched lock 55 0xc000000002d3539c 296 0% 99% 0.00 26975 sched lock 60 0xc000000002d3539c 293 0% 99% 0.00 61238 zombie lock thread_stash+0x170 282 0% 99% 0.00 47092 sched lock 35 0xc000000002d3539c 270 0% 99% 0.00 27891 sched lock 50 0xc000000002d3539c 251 0% 99% 0.00 23210 sched lock 45 0xc000000002d3539c 223 0% 99% 0.00 32365 sched lock 30 0xc000000002d3539c 212 0% 99% 0.00 38113 sched lock 40 0xc000000002d3539c 207 0% 99% 0.00 33128 sched lock 25 0xc000000002d3539c 196 0% 99% 0.00 24377 sched lock 45 0xc000000002d35384 193 0% 99% 0.00 36139 sched lock 40 0xc000000002d35384 188 0% 99% 0.00 18627 sched lock 70 0xc000000002d3539c 167 0% 99% 0.00 29464 sched lock 15 (usermode) 162 0% 99% 0.00 42513 sched lock 15 0xc000000002d3539c 156 0% 99% 0.00 56293 zombie lock (usermode) 155 0% 99% 0.00 45742 sched lock 10 (usermode) 154 0% 99% 0.00 33804 sched lock 30 (usermode) 149 0% 99% 0.00 11027 sched lock 0 sched_add+0x48c 147 0% 99% 0.00 23415 sched lock 50 0xc000000002d35384 146 0% 99% 0.00 10218 td_contested turnstile_unpend+0x428 145 0% 99% 0.00 35618 sched lock 25 (usermode) 137 0% 99% 0.00 29174 sched lock 20 (usermode) 136 0% 99% 0.00 20910 sched lock 55 (usermode) 132 0% 99% 0.00 24844 sched lock 20 0xc000000002d3539c 130 0% 99% 0.00 4313 sched lock 60 sched_add+0x48c 129 0% 99% 0.00 29317 sched lock 50 (usermode) 126 0% 99% 0.00 58759 sched lock 35 (usermode) 124 0% 99% 0.00 18408 sched lock 60 (usermode) 119 0% 99% 0.00 29056 sched lock 65 (usermode) 118 0% 99% 0.00 9361 sched lock 65 sched_add+0x48c 117 0% 99% 0.00 13552 sched lock 10 sched_add+0x48c 116 0% 99% 0.00 9284 callout (usermode) 114 0% 99% 0.00 4907 sched lock 40 sched_add+0x48c 112 0% 99% 0.00 25661 sched lock 55 0xc000000002d35384 112 0% 99% 0.00 4564 sched lock 33 sched_add+0x48c 111 0% 99% 0.00 40158 sched lock 5 (usermode) 109 0% 99% 0.00 19018 sched lock 8 0xc000000002d35384 109 0% 99% 0.00 20862 sched lock 63 0xc000000002d3539c 109 0% 99% 0.00 25761 sched lock 45 (usermode) 109 0% 99% 0.00 34731 sched lock 40 (usermode) 107 0% 100% 0.00 20289 sched lock 62 0xc000000002d3539c 105 0% 100% 0.00 18075 sched lock 60 0xc000000002d35384 104 0% 100% 0.00 3351 sched lock 67 sched_add+0x48c 102 0% 100% 0.00 3324 sched lock 66 sched_add+0x48c 101 0% 100% 0.00 62538 sched lock 10 0xc000000002d3539c 100 0% 100% 0.00 5106 sched lock 55 sched_add+0x48c 100 0% 100% 0.00 5206 sched lock 25 sched_add+0x48c 100 0% 100% 0.00 14542 sched lock 1 0xc000000002d35384 98 0% 100% 0.00 3180 sched lock 71 sched_add+0x48c 98 0% 100% 0.00 3799 sched lock 61 sched_add+0x48c 98 0% 100% 0.00 5251 sched lock 45 sched_add+0x48c 97 0% 100% 0.00 18138 sched lock 67 0xc000000002d3539c 97 0% 100% 0.00 6441 sched lock 20 sched_add+0x48c 96 0% 100% 0.00 3380 sched lock 46 sched_add+0x48c 96 0% 100% 0.00 4050 sched lock 30 sched_add+0x48c 96 0% 100% 0.00 8637 sched lock 23 sched_add+0x48c 95 0% 100% 0.00 6535 sched lock 13 sched_add+0x48c 94 0% 100% 0.00 4980 sched lock 49 sched_add+0x48c 94 0% 100% 0.00 14010 sched lock 14 sched_add+0x48c 93 0% 100% 0.00 4935 sched lock 63 sched_add+0x48c 92 0% 100% 0.00 13824 sched lock 8 sched_add+0x48c 92 0% 100% 0.00 6115 sched lock 6 sched_add+0x48c 92 0% 100% 0.00 5188 sched lock 35 sched_add+0x48c 91 0% 100% 0.00 18041 sched lock 70 (usermode) 91 0% 100% 0.00 2146 sched lock 64 sched_add+0x48c 89 0% 100% 0.00 3334 sched lock 68 sched_add+0x48c 89 0% 100% 0.00 3748 sched lock 54 sched_add+0x48c 89 0% 100% 0.00 6371 sched lock 50 sched_add+0x48c 89 0% 100% 0.00 4965 sched lock 43 sched_add+0x48c 89 0% 100% 0.00 9125 sched lock 15 sched_add+0x48c 89 0% 100% 0.00 7218 sched lock 11 sched_add+0x48c 88 0% 100% 0.00 15786 sched lock 71 0xc000000002d3539c 88 0% 100% 0.00 2351 sched lock 70 sched_add+0x48c 88 0% 100% 0.00 18797 sched lock 42 sched_add+0x48c 88 0% 100% 0.00 12555 sched lock 32 sched_add+0x48c 87 0% 100% 0.00 15668 sched lock 6 0xc000000002d35384 87 0% 100% 0.00 3683 sched lock 53 sched_add+0x48c 86 0% 100% 0.00 3318 sched lock 69 sched_add+0x48c 85 0% 100% 0.00 8274 sched lock 9 sched_add+0x48c 85 0% 100% 0.00 17068 sched lock 68 0xc000000002d3539c 85 0% 100% 0.00 6376 sched lock 55 sched_switch+0x81c 85 0% 100% 0.00 11150 sched lock 5 sched_add+0x48c 84 0% 100% 0.00 7780 sched lock 7 sched_add+0x48c 84 0% 100% 0.00 3824 sched lock 28 sched_add+0x48c 84 0% 100% 0.00 51293 sched lock 2 0xc000000002d35384 83 0% 100% 0.00 5733 sched lock 27 sched_add+0x48c 82 0% 100% 0.00 6430 sched lock 48 sched_add+0x48c 82 0% 100% 0.00 7864 sched lock 38 sched_add+0x48c 82 0% 100% 0.00 13002 sched lock 31 sched_add+0x48c 80 0% 100% 0.00 33546 sched lock 61 0xc000000002d3539c 80 0% 100% 0.00 5825 sched lock 51 sched_add+0x48c 80 0% 100% 0.00 17708 sched lock 14 0xc000000002d35384 80 0% 100% 0.00 11447 sched lock 1 sched_add+0x48c 79 0% 100% 0.00 3728 sched lock 62 sched_add+0x48c 79 0% 100% 0.00 103756 sched lock 46 0xc000000002d3539c 79 0% 100% 0.00 14393 sched lock 24 sched_add+0x48c 78 0% 100% 0.00 21695 sched lock 7 0xc000000002d35384 78 0% 100% 0.00 7659 sched lock 4 sched_add+0x48c 78 0% 100% 0.00 9876 sched lock 37 sched_add+0x48c 78 0% 100% 0.00 20737 sched lock 34 sched_add+0x48c 78 0% 100% 0.00 3578 sched lock 29 sched_add+0x48c 78 0% 100% 0.00 6403 sched lock 21 sched_add+0x48c 77 0% 100% 0.00 3350 sched lock 39 sched_add+0x48c 77 0% 100% 0.00 8458 sched lock 2 sched_add+0x48c 76 0% 100% 0.00 8226 td_contested (usermode) 76 0% 100% 0.00 6221 sched lock 41 sched_add+0x48c 76 0% 100% 0.00 23803 sched lock 3 sched_add+0x48c 76 0% 100% 0.00 4881 sched lock 16 sched_add+0x48c 75 0% 100% 0.00 5271 sched lock 17 sched_add+0x48c 75 0% 100% 0.00 25397 sched lock 11 0xc000000002d35384 74 0% 100% 0.00 29636 sched lock 16 0xc000000002d35384 73 0% 100% 0.00 15146 sched lock 9 0xc000000002d35384 73 0% 100% 0.00 7402 sched lock 26 sched_add+0x48c 73 0% 100% 0.00 19546 sched lock 19 sched_add+0x48c 73 0% 100% 0.00 16294 sched lock 18 sched_add+0x48c 71 0% 100% 0.00 5577 sched lock 58 sched_add+0x48c 71 0% 100% 0.00 4066 sched lock 57 sched_add+0x48c 71 0% 100% 0.00 2569 sched lock 56 sched_add+0x48c 71 0% 100% 0.00 15167 sched lock 51 0xc000000002d3539c 71 0% 100% 0.00 24784 sched lock 13 0xc000000002d35384 70 0% 100% 0.00 4697 sched lock 47 sched_add+0x48c 70 0% 100% 0.00 3178 sched lock 44 sched_add+0x48c 70 0% 100% 0.00 3235 sched lock 36 sched_add+0x48c 70 0% 100% 0.00 9315 sched lock 15 sched_switch+0x81c 69 0% 100% 0.00 19483 sched lock 66 0xc000000002d3539c 69 0% 100% 0.00 13867 sched lock 64 0xc000000002d3539c 69 0% 100% 0.00 7058 sched lock 50 sched_switch+0x81c 69 0% 100% 0.00 5750 sched lock 22 sched_add+0x48c 68 0% 100% 0.00 14913 sched lock 21 0xc000000002d35384 66 0% 100% 0.00 10002 sched lock 59 sched_add+0x48c 66 0% 100% 0.00 43328 sched lock 47 0xc000000002d3539c 65 0% 100% 0.00 13777 sched lock 65 0xc000000002d35384 65 0% 100% 0.00 21261 sched lock 53 0xc000000002d3539c 65 0% 100% 0.00 5262 sched lock 52 sched_add+0x48c 65 0% 100% 0.00 18298 sched lock 4 0xc000000002d35384 65 0% 100% 0.00 13683 sched lock 28 0xc000000002d35384 65 0% 100% 0.00 8345 sched lock 12 sched_add+0x48c 65 0% 100% 0.00 8277 sched lock 10 sched_switch+0x81c 63 0% 100% 0.00 11017 sched lock 29 0xc000000002d35384 62 0% 100% 0.00 79638 sched lock 46 (usermode) 62 0% 100% 0.00 16732 sched lock 3 0xc000000002d35384 61 0% 100% 0.00 10585 sched lock 12 0xc000000002d35384 60 0% 100% 0.00 11579 sched lock 69 0xc000000002d3539c 60 0% 100% 0.00 36263 sched lock 42 (usermode) 60 0% 100% 0.00 7704 sched lock 35 sched_switch+0x81c 59 0% 100% 0.00 13928 sched lock 62 (usermode) 58 0% 100% 0.00 9599 sched lock 24 0xc000000002d35384 58 0% 100% 0.00 15961 sched lock 14 (usermode) 58 0% 100% 0.00 11266 sched lock 0 sched_switch+0x81c 57 0% 100% 0.00 5849 sched lock 65 sched_switch+0x81c 57 0% 100% 0.00 8411 sched lock 45 sched_switch+0x81c 56 0% 100% 0.00 16727 sched lock 18 0xc000000002d35384 55 0% 100% 0.00 9132 sched lock 59 0xc000000002d3539c 55 0% 100% 0.00 19395 sched lock 39 0xc000000002d3539c 54 0% 100% 0.00 17237 sched lock 8 (usermode) 54 0% 100% 0.00 8991 sched lock 5 sched_switch+0x81c 54 0% 100% 0.00 30198 sched lock 33 0xc000000002d3539c 54 0% 100% 0.00 18651 sched lock 32 (usermode) 54 0% 100% 0.00 447 callout 0xc000000002d14524 53 0% 100% 0.00 9407 sched lock 58 0xc000000002d3539c 53 0% 100% 0.00 14821 sched lock 44 0xc000000002d3539c 53 0% 100% 0.00 8498 sched lock 30 sched_switch+0x81c 52 0% 100% 0.00 21508 sched lock 63 (usermode) 52 0% 100% 0.00 31680 sched lock 42 0xc000000002d3539c 52 0% 100% 0.00 10749 sched lock 25 sched_switch+0x81c 51 0% 100% 0.00 24721 sched lock 54 0xc000000002d3539c 51 0% 100% 0.00 12484 sched lock 37 (usermode) 51 0% 100% 0.00 28811 sched lock 22 0xc000000002d35384 50 0% 100% 0.00 12450 sched lock 68 (usermode) 50 0% 100% 0.00 5712 sched lock 40 sched_switch+0x81c 50 0% 100% 0.00 15387 sched lock 22 (usermode) 50 0% 100% 0.00 11664 sched lock 21 (usermode) 50 0% 100% 0.00 35054 sched lock 19 0xc000000002d35384 50 0% 100% 0.00 16509 sched lock 17 0xc000000002d35384 50 0% 100% 0.00 17851 sched lock 0 sched_switch+0xca0 49 0% 100% 0.00 27092 sched lock 34 0xc000000002d3539c 48 0% 100% 0.00 9371 sched lock 6 (usermode) 48 0% 100% 0.00 8354 sched lock 26 0xc000000002d35384 47 0% 100% 0.00 15474 sched lock 7 (usermode) 47 0% 100% 0.00 13946 sched lock 59 (usermode) 47 0% 100% 0.00 13003 sched lock 56 0xc000000002d3539c 47 0% 100% 0.00 20831 sched lock 51 (usermode) 47 0% 100% 0.00 42186 sched lock 33 0xc000000002d35384 47 0% 100% 0.00 19735 sched lock 28 (usermode) 47 0% 100% 0.00 9512 sched lock 11 (usermode) 46 0% 100% 0.00 2820 td_contested turnstile_broadcast+0x1bc 46 0% 100% 0.00 11214 sched lock 57 0xc000000002d3539c 46 0% 100% 0.00 21501 sched lock 27 0xc000000002d35384 46 0% 100% 0.00 23029 sched lock 2 (usermode) 45 0% 100% 0.00 9539 sched lock 71 (usermode) 45 0% 100% 0.00 11780 sched lock 44 0xc000000002d35384 45 0% 100% 0.00 51872 sched lock 3 (usermode) 45 0% 100% 0.00 29835 sched lock 16 (usermode) 44 0% 100% 0.00 15392 sched lock 38 0xc000000002d3539c 44 0% 100% 0.00 11940 sched lock 36 (usermode) 43 0% 100% 0.00 12878 sched lock 9 (usermode) 43 0% 100% 0.00 6875 sched lock 60 sched_switch+0x81c 43 0% 100% 0.00 6973 sched lock 41 (usermode) 43 0% 100% 0.00 10255 sched lock 38 (usermode) 43 0% 100% 0.00 16546 sched lock 37 0xc000000002d3539c 43 0% 100% 0.00 19334 sched lock 31 0xc000000002d3539c 43 0% 100% 0.00 18222 sched lock 27 0xc000000002d3539c 42 0% 100% 0.00 14099 sched lock 66 (usermode) 42 0% 100% 0.00 8205 sched lock 64 (usermode) 42 0% 100% 0.00 70807 sched lock 61 (usermode) 42 0% 100% 0.00 13151 sched lock 54 (usermode) 42 0% 100% 0.00 30614 sched lock 5 0xc000000002d3539c 42 0% 100% 0.00 23282 sched lock 33 (usermode) 42 0% 100% 0.00 29334 sched lock 13 (usermode) 41 0% 100% 0.00 9505 sched lock 49 0xc000000002d3539c 41 0% 100% 0.00 17262 sched lock 27 (usermode) 41 0% 100% 0.00 26192 sched lock 22 0xc000000002d3539c 40 0% 100% 0.00 11637 sched lock 69 (usermode) 40 0% 100% 0.00 30651 sched lock 67 (usermode) 40 0% 100% 0.00 6693 sched lock 48 (usermode) 40 0% 100% 0.00 20485 sched lock 44 (usermode) 40 0% 100% 0.00 11643 sched lock 43 0xc000000002d3539c 40 0% 100% 0.00 33175 sched lock 42 0xc000000002d35384 40 0% 100% 0.00 19128 sched lock 32 0xc000000002d35384 40 0% 100% 0.00 20448 sched lock 28 0xc000000002d3539c 40 0% 100% 0.00 9852 sched lock 20 sched_switch+0x81c 40 0% 100% 0.00 15836 sched lock 19 (usermode) 39 0% 100% 0.00 19104 sched lock 52 0xc000000002d3539c 39 0% 100% 0.00 14592 sched lock 34 (usermode) 38 0% 100% 0.00 1702 td_contested turnstile_wait+0x584 38 0% 100% 0.00 8043 sched lock 49 (usermode) 38 0% 100% 0.00 21586 sched lock 47 (usermode) 38 0% 100% 0.00 6032 sched lock 43 (usermode) 38 0% 100% 0.00 19549 sched lock 29 (usermode) 37 0% 100% 0.00 21932 sched lock 36 0xc000000002d3539c 37 0% 100% 0.00 7708 sched lock 36 0xc000000002d35384 37 0% 100% 0.00 20636 sched lock 23 0xc000000002d35384 37 0% 100% 0.00 12790 sched lock 19 0xc000000002d3539c 37 0% 100% 0.00 19361 sched lock 1 (usermode) 36 0% 100% 0.00 11027 sched lock 58 (usermode) 36 0% 100% 0.00 9388 sched lock 38 0xc000000002d35384 36 0% 100% 0.00 16777 sched lock 37 0xc000000002d35384 35 0% 100% 0.00 9213 sched lock 48 0xc000000002d35384 35 0% 100% 0.00 26351 sched lock 34 0xc000000002d35384 35 0% 100% 0.00 16105 sched lock 29 0xc000000002d3539c 34 0% 100% 0.00 10472 sched lock 56 (usermode) 34 0% 100% 0.00 17149 sched lock 4 (usermode) 34 0% 100% 0.00 19991 sched lock 31 0xc000000002d35384 34 0% 100% 0.00 22278 sched lock 31 (usermode) 34 0% 100% 0.00 26866 sched lock 12 (usermode) 33 0% 100% 0.00 9710 sched lock 18 (usermode) 32 0% 100% 0.00 23785 sched lock 18 0xc000000002d3539c 32 0% 100% 0.00 12288 callout softclock+0x154 31 0% 100% 0.00 5795 sched lock 57 (usermode) 31 0% 100% 0.00 10203 sched lock 48 0xc000000002d3539c 31 0% 100% 0.00 104346 sched lock 46 0xc000000002d35384 31 0% 100% 0.00 8274 sched lock 41 0xc000000002d3539c 30 0% 100% 0.00 9192 sched lock 53 (usermode) 30 0% 100% 0.00 12149 sched lock 52 (usermode) 30 0% 100% 0.00 48403 sched lock 47 0xc000000002d35384 30 0% 100% 0.00 22150 sched lock 23 (usermode) 30 0% 100% 0.00 14831 callout 0xc000000002d12b30 29 0% 100% 0.00 7632 sched lock 70 sched_switch+0x81c 29 0% 100% 0.00 10437 sched lock 26 (usermode) 29 0% 100% 0.00 5344 sched lock 17 (usermode) 28 0% 100% 0.00 19271 sched lock 43 0xc000000002d35384 28 0% 100% 0.00 12423 process slock kern_thr_exit+0x304 27 0% 100% 0.00 8017 sched lock 52 0xc000000002d35384 27 0% 100% 0.00 13700 sched lock 51 0xc000000002d35384 27 0% 100% 0.00 18038 sched lock 26 0xc000000002d3539c 27 0% 100% 0.00 19063 sched lock 24 (usermode) 27 0% 100% 0.00 9479 sched lock 11 0xc000000002d3539c 26 0% 100% 0.00 9632 sched lock 41 0xc000000002d35384 25 0% 100% 0.00 73576 sched lock 61 0xc000000002d35384 25 0% 100% 0.00 14989 sched lock 39 (usermode) 25 0% 100% 0.00 3871 sched lock 23 0xc000000002d3539c 25 0% 100% 0.00 224 et_hw_mtx cpu_new_callout+0x370 24 0% 100% 0.00 9947 sched lock 70 0xc000000002d35384 24 0% 100% 0.00 18050 sched lock 62 0xc000000002d35384 24 0% 100% 0.00 12724 sched lock 49 0xc000000002d35384 24 0% 100% 0.00 14408 sched lock 32 0xc000000002d3539c 24 0% 100% 0.00 9497 sched lock 21 0xc000000002d3539c 23 0% 100% 0.00 17531 sched lock 57 0xc000000002d35384 23 0% 100% 0.00 10863 sched lock 56 0xc000000002d35384 23 0% 100% 0.00 36126 sched lock 54 0xc000000002d35384 23 0% 100% 0.00 14853 sched lock 14 0xc000000002d3539c 23 0% 100% 0.00 273 et_hw_mtx (usermode) 22 0% 100% 0.00 24638 sched lock 39 0xc000000002d35384 22 0% 100% 0.00 624 pstatl (usermode) 21 0% 100% 0.00 13179 sched lock 3 0xc000000002d3539c 20 0% 100% 0.00 25662 sched lock 55 sched_switch+0xca0 20 0% 100% 0.00 28465 sched lock 53 0xc000000002d35384 20 0% 100% 0.00 21050 sched lock 16 0xc000000002d3539c 19 0% 100% 0.00 17333 sched lock 7 0xc000000002d3539c 19 0% 100% 0.00 9989 sched lock 24 0xc000000002d3539c 18 0% 100% 0.00 15017 sched lock 64 0xc000000002d35384 18 0% 100% 0.00 27931 sched lock 50 sched_switch+0xca0 18 0% 100% 0.00 4079 sched lock 2 0xc000000002d3539c 18 0% 100% 0.00 13077 sched lock 17 0xc000000002d3539c 17 0% 100% 0.00 42933 sched lock 30 sched_switch+0xca0 17 0% 100% 0.00 3580 sched lock 13 0xc000000002d3539c 16 0% 100% 0.00 9285 sched lock 68 0xc000000002d35384 16 0% 100% 0.00 11904 sched lock 6 0xc000000002d3539c 16 0% 100% 0.00 30105 sched lock 45 sched_switch+0xca0 16 0% 100% 0.00 20809 sched lock 35 sched_switch+0xca0 16 0% 100% 0.00 53004 sched lock 15 sched_switch+0xca0 16 0% 100% 0.00 13890 sched lock 12 0xc000000002d3539c 15 0% 100% 0.00 27038 sched lock 20 sched_switch+0xca0 15 0% 100% 0.00 27279 sched lock 10 sched_switch+0xca0 14 0% 100% 0.00 21850 sched lock 70 sched_switch+0xca0 14 0% 100% 0.00 43006 sched lock 5 sched_switch+0xca0 13 0% 100% 0.00 42130 sched lock 65 sched_switch+0xca0 13 0% 100% 0.00 7669 sched lock 0 0xc000000002d3536c 12 0% 100% 0.00 8112 sched lock 9 0xc000000002d3539c 12 0% 100% 0.00 15949 sched lock 8 0xc000000002d3539c 12 0% 100% 0.00 2469 sched lock 69 0xc000000002d35384 12 0% 100% 0.00 8960 sched lock 66 0xc000000002d35384 11 0% 100% 0.00 40988 sched lock 60 sched_switch+0xca0 11 0% 100% 0.00 10956 sched lock 59 0xc000000002d35384 11 0% 100% 0.00 10348 sched lock 58 0xc000000002d35384 11 0% 100% 0.00 4866 sched lock 39 sched_switch+0x81c 11 0% 100% 0.00 58086 sched lock 25 sched_switch+0xca0 10 0% 100% 0.00 5661 sched lock 8 sched_switch+0x81c 10 0% 100% 0.00 6985 sched lock 63 0xc000000002d35384 10 0% 100% 0.00 7129 sched lock 63 sched_switch+0x81c 9 0% 100% 0.00 36971 sched lock 40 sched_switch+0xca0 9 0% 100% 0.00 8087 sched lock 34 sched_switch+0x81c 9 0% 100% 0.00 29654 sched lock 28 sched_switch+0xca0 9 0% 100% 0.00 3326 sched lock 11 sched_switch+0x81c 9 0% 100% 0.00 794 pstatl thread_exit+0x3dc 8 0% 100% 0.00 5105 td_contested turnstile_claim+0x290 8 0% 100% 0.00 4511 sched lock 7 sched_switch+0x81c 8 0% 100% 0.00 1894 sched lock 47 sched_switch+0x81c 8 0% 100% 0.00 2076 sched lock 42 sched_switch+0x81c 8 0% 100% 0.00 7325 sched lock 3 sched_switch+0x81c 8 0% 100% 0.00 2100 sched lock 28 sched_switch+0x81c 8 0% 100% 0.00 6941 sched lock 16 sched_switch+0x81c 8 0% 100% 0.00 2330 sched lock 1 0xc000000002d3539c 7 0% 100% 0.00 8339 sched lock 8 0xc000000002d3536c 7 0% 100% 0.00 8349 sched lock 5 0xc000000002d3536c 7 0% 100% 0.00 4768 sched lock 41 sched_switch+0x81c 7 0% 100% 0.00 784 sched lock 38 sched_switch+0x81c 7 0% 100% 0.00 1840 sched lock 37 sched_switch+0x81c 7 0% 100% 0.00 7294 sched lock 33 sched_switch+0x81c 7 0% 100% 0.00 8874 sched lock 31 sched_switch+0x81c 7 0% 100% 0.00 2544 sched lock 27 sched_switch+0x81c 7 0% 100% 0.00 7156 sched lock 21 sched_switch+0x81c 7 0% 100% 0.00 4336 sched lock 14 sched_switch+0x81c 6 0% 100% 0.00 80667 sched lock 22 sched_switch+0x81c 6 0% 100% 0.00 1193 sched lock 19 sched_switch+0x81c 6 0% 100% 0.00 5010 sched lock 18 sched_switch+0x81c 6 0% 100% 0.00 9039 sched lock 15 0xc000000002d3536c 5 0% 100% 0.00 953 sched lock 9 sched_switch+0x81c 5 0% 100% 0.00 5346 sched lock 9 0xc000000002d3536c 5 0% 100% 0.00 8821 sched lock 67 sched_switch+0x81c 5 0% 100% 0.00 3728 sched lock 64 sched_switch+0x81c 5 0% 100% 0.00 6735 sched lock 63 sched_switch+0xca0 5 0% 100% 0.00 10818 sched lock 62 sched_switch+0x81c 5 0% 100% 0.00 3697 sched lock 6 sched_switch+0x81c 5 0% 100% 0.00 898 sched lock 59 sched_switch+0x81c 5 0% 100% 0.00 2828 sched lock 53 sched_switch+0x81c 5 0% 100% 0.00 370 sched lock 48 sched_switch+0x81c 5 0% 100% 0.00 13266 sched lock 47 sched_switch+0xca0 5 0% 100% 0.00 3350 sched lock 46 sched_switch+0x81c 5 0% 100% 0.00 2618 sched lock 43 sched_switch+0x81c 5 0% 100% 0.00 19093 sched lock 4 0xc000000002d3536c 5 0% 100% 0.00 3505 sched lock 36 sched_switch+0x81c 5 0% 100% 0.00 3921 sched lock 32 sched_switch+0xca0 5 0% 100% 0.00 678 sched lock 26 sched_switch+0x81c 5 0% 100% 0.00 6321 sched lock 2 0xc000000002d3536c 5 0% 100% 0.00 2054 sched lock 17 sched_switch+0x81c 5 0% 100% 0.00 10723 sched lock 13 0xc000000002d3536c 5 0% 100% 0.00 5444 sched lock 12 sched_switch+0x81c 5 0% 100% 0.00 20333 sched lock 10 0xc000000002d3536c 5 0% 100% 0.00 4662 sched lock 1 0xc000000002d3536c 5 0% 100% 0.00 805 sched lock 1 sched_switch+0x81c 4 0% 100% 0.00 1486 zombie lock thread_reap+0x218 4 0% 100% 0.00 6807 sched lock 8 sched_switch+0xca0 4 0% 100% 0.00 2401 sched lock 67 sched_switch+0xca0 4 0% 100% 0.00 8763 sched lock 6 0xc000000002d3536c 4 0% 100% 0.00 8566 sched lock 57 sched_switch+0x81c 4 0% 100% 0.00 14286 sched lock 54 sched_switch+0xca0 4 0% 100% 0.00 3982 sched lock 51 sched_switch+0xca0 4 0% 100% 0.00 28070 sched lock 49 sched_switch+0xca0 4 0% 100% 0.00 2398 sched lock 46 sched_switch+0xca0 4 0% 100% 0.00 15626 sched lock 36 sched_switch+0xca0 4 0% 100% 0.00 1441 sched lock 32 sched_switch+0x81c 4 0% 100% 0.00 2313 sched lock 29 sched_switch+0x81c 4 0% 100% 0.00 2879 sched lock 26 sched_switch+0xca0 4 0% 100% 0.00 7595 sched lock 25 0xc000000002d3536c 4 0% 100% 0.00 38538 sched lock 20 0xc000000002d3536c 4 0% 100% 0.00 56136 sched lock 13 sched_switch+0xca0 4 0% 100% 0.00 2735 sched lock 12 0xc000000002d3536c 3 0% 100% 0.00 8867 sched lock 71 sched_switch+0xca0 3 0% 100% 0.00 17316 sched lock 69 sched_switch+0xca0 3 0% 100% 0.00 2559 sched lock 66 sched_switch+0x81c 3 0% 100% 0.00 12778 sched lock 62 sched_switch+0xca0 3 0% 100% 0.00 21651 sched lock 61 sched_switch+0xca0 3 0% 100% 0.00 250 sched lock 6 sched_switch+0xca0 3 0% 100% 0.00 16156 sched lock 59 sched_switch+0xca0 3 0% 100% 0.00 18316 sched lock 58 sched_switch+0xca0 3 0% 100% 0.00 1331 sched lock 58 sched_switch+0x81c 3 0% 100% 0.00 2103 sched lock 53 sched_switch+0xca0 3 0% 100% 0.00 8827 sched lock 51 sched_switch+0x81c 3 0% 100% 0.00 4020 sched lock 49 sched_switch+0x81c 3 0% 100% 0.00 24512 sched lock 48 sched_switch+0xca0 3 0% 100% 0.00 8537 sched lock 48 0xc000000002d3536c 3 0% 100% 0.00 565 sched lock 44 sched_switch+0x81c 3 0% 100% 0.00 5704 sched lock 41 sched_switch+0xca0 3 0% 100% 0.00 24870 sched lock 4 0xc000000002d3539c 3 0% 100% 0.00 583 sched lock 4 sched_switch+0x81c 3 0% 100% 0.00 16556 sched lock 39 sched_switch+0xca0 3 0% 100% 0.00 4702 sched lock 38 sched_switch+0xca0 3 0% 100% 0.00 9401 sched lock 3 0xc000000002d3536c 3 0% 100% 0.00 2304 sched lock 27 0xc000000002d3536c 3 0% 100% 0.00 14610 sched lock 24 sched_switch+0x81c 3 0% 100% 0.00 5068 sched lock 22 sched_switch+0xca0 3 0% 100% 0.00 652 sched lock 2 sched_switch+0x81c 3 0% 100% 0.00 219 sched lock 18 0xc000000002d3536c 3 0% 100% 0.00 17604 sched lock 16 0xc000000002d3536c 3 0% 100% 0.00 16709 sched lock 16 sched_switch+0xca0 3 0% 100% 0.00 32123 sched lock 14 sched_switch+0xca0 3 0% 100% 0.00 7660 sched lock 13 sched_switch+0x81c 3 0% 100% 0.00 9658 sched lock 12 sched_switch+0xca0 3 0% 100% 0.00 10199 sched lock 11 0xc000000002d3536c 3 0% 100% 0.00 13945 process slock (usermode) 3 0% 100% 0.00 7000 callout callout_process+0x388 2 0% 100% 0.00 46922 sched lock 7 sched_switch+0xca0 2 0% 100% 0.00 11572 sched lock 64 sched_switch+0xca0 2 0% 100% 0.00 35865 sched lock 64 0xc000000002d3536c 2 0% 100% 0.00 11821 sched lock 61 0xc000000002d3536c 2 0% 100% 0.00 11605 sched lock 61 sched_switch+0x81c 2 0% 100% 0.00 388 sched lock 57 sched_switch+0xca0 2 0% 100% 0.00 216 sched lock 56 sched_switch+0x81c 2 0% 100% 0.00 54335 sched lock 56 sched_switch+0xca0 2 0% 100% 0.00 3968 sched lock 56 0xc000000002d3536c 2 0% 100% 0.00 2742 sched lock 52 0xc000000002d3536c 2 0% 100% 0.00 20757 sched lock 52 sched_switch+0x81c 2 0% 100% 0.00 8099 sched lock 45 0xc000000002d3536c 2 0% 100% 0.00 7273 sched lock 37 sched_switch+0xca0 2 0% 100% 0.00 1874 sched lock 34 sched_switch+0xca0 2 0% 100% 0.00 21018 sched lock 33 0xc000000002d3536c 2 0% 100% 0.00 1016 sched lock 33 sched_switch+0xca0 2 0% 100% 0.00 9059 sched lock 32 0xc000000002d3536c 2 0% 100% 0.00 7016 sched lock 31 sched_switch+0xca0 2 0% 100% 0.00 15137 sched lock 29 0xc000000002d3536c 2 0% 100% 0.00 7637 sched lock 24 0xc000000002d3536c 2 0% 100% 0.00 1183 sched lock 23 sched_switch+0x81c 2 0% 100% 0.00 22490 sched lock 22 0xc000000002d3536c 2 0% 100% 0.00 1880 sched lock 17 0xc000000002d3536c 2 0% 100% 0.00 1070 sched lock 14 0xc000000002d3536c 2 0% 100% 0.00 1199 sched lock 11 sched_switch+0xca0 2 0% 100% 0.00 1318 sched lock 1 sched_switch+0xca0 1 0% 100% 0.00 344 sched lock 71 sched_switch+0x81c 1 0% 100% 0.00 73604 sched lock 68 sched_switch+0x81c 1 0% 100% 0.00 20546 sched lock 68 sched_switch+0xca0 1 0% 100% 0.00 20863 sched lock 66 sched_switch+0xca0 1 0% 100% 0.00 52959 sched lock 66 0xc000000002d3536c 1 0% 100% 0.00 324 sched lock 63 0xc000000002d3536c 1 0% 100% 0.00 1672 sched lock 54 sched_switch+0x81c 1 0% 100% 0.00 172 sched lock 57 0xc000000002d3536c 1 0% 100% 0.00 1160 sched lock 52 sched_switch+0xca0 1 0% 100% 0.00 10608 sched lock 50 0xc000000002d3536c 1 0% 100% 0.00 322 sched lock 47 0xc000000002d3536c 1 0% 100% 0.00 11766 sched lock 43 0xc000000002d3536c 1 0% 100% 0.00 24894 sched lock 42 sched_switch+0xca0 1 0% 100% 0.00 6068 sched lock 40 0xc000000002d3536c 1 0% 100% 0.00 24505 sched lock 41 0xc000000002d3536c 1 0% 100% 0.00 6929 sched lock 44 sched_switch+0xca0 1 0% 100% 0.00 562 sched lock 44 0xc000000002d3536c 1 0% 100% 0.00 185 sched lock 30 0xc000000002d3536c 1 0% 100% 0.00 23550 sched lock 31 0xc000000002d3536c 1 0% 100% 0.00 52129 sched lock 29 sched_switch+0xca0 1 0% 100% 0.00 1167 sched lock 28 0xc000000002d3536c 1 0% 100% 0.00 193 sched lock 36 0xc000000002d3536c 1 0% 100% 0.00 10538 sched lock 35 0xc000000002d3536c 1 0% 100% 0.00 20816 sched lock 38 0xc000000002d3536c 1 0% 100% 0.00 2091 sched lock 37 0xc000000002d3536c 1 0% 100% 0.00 23289 sched lock 4 sched_switch+0xca0 1 0% 100% 0.00 342 et_hw_mtx cpu_idleclock+0x37c 1 0% 100% 0.00 49369 sched lock 18 sched_switch+0xca0 1 0% 100% 0.00 12327 sched lock 19 0xc000000002d3536c 1 0% 100% 0.00 338 sched lock 17 sched_switch+0xca0 1 0% 100% 0.00 4950 sched lock 23 sched_switch+0xca0 1 0% 100% 0.00 47403 sched lock 21 sched_switch+0xca0 1 0% 100% 0.00 226 sched lock 21 0xc000000002d3536c 1 0% 100% 0.00 859 sched lock 27 sched_switch+0xca0 ------------------------------------------------------------------------------- Thread lock spin: 73052 events in 462.353 seconds (158 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 34722 48% 48% 0.00 5016 sleepq chain _thread_lock+0xf4 12791 18% 65% 0.00 4816 sleepq chain (usermode) 2278 3% 68% 0.00 7376 turnstile lock _thread_lock+0xf4 839 1% 69% 0.00 11065 sched lock 0 (usermode) 747 1% 70% 0.00 27074 sched lock 0 _thread_lock+0xf4 464 1% 71% 0.00 15825 sched lock 30 _thread_lock+0xf4 424 1% 72% 0.00 11563 sched lock 60 _thread_lock+0xf4 423 1% 72% 0.00 14895 sched lock 15 _thread_lock+0xf4 422 1% 73% 0.00 18052 sched lock 10 _thread_lock+0xf4 386 1% 73% 0.00 13292 sched lock 45 _thread_lock+0xf4 380 1% 74% 0.00 37140 sched lock 25 _thread_lock+0xf4 375 1% 74% 0.00 12365 sched lock 65 _thread_lock+0xf4 354 0% 75% 0.00 10907 sched lock 55 _thread_lock+0xf4 352 0% 75% 0.00 14904 sched lock 35 _thread_lock+0xf4 346 0% 76% 0.00 11808 sched lock 50 _thread_lock+0xf4 336 0% 76% 0.00 10588 sched lock 70 _thread_lock+0xf4 333 0% 77% 0.00 13421 sched lock 5 _thread_lock+0xf4 317 0% 77% 0.00 11822 sched lock 40 _thread_lock+0xf4 311 0% 77% 0.00 13759 sched lock 20 _thread_lock+0xf4 263 0% 78% 0.00 8947 turnstile lock (usermode) 235 0% 78% 0.00 12435 sched lock 69 _thread_lock+0xf4 221 0% 78% 0.00 16394 sched lock 33 _thread_lock+0xf4 218 0% 79% 0.00 8129 sched lock 71 (usermode) 217 0% 79% 0.00 12368 sched lock 6 _thread_lock+0xf4 217 0% 79% 0.00 11935 sched lock 22 _thread_lock+0xf4 209 0% 80% 0.00 17196 sched lock 10 (usermode) 204 0% 80% 0.00 10860 sched lock 4 _thread_lock+0xf4 200 0% 80% 0.00 10578 sched lock 65 (usermode) 197 0% 80% 0.00 16389 sched lock 63 _thread_lock+0xf4 196 0% 81% 0.00 11926 sched lock 40 (usermode) 185 0% 81% 0.00 14237 sched lock 64 _thread_lock+0xf4 181 0% 81% 0.00 16491 sched lock 5 (usermode) 178 0% 81% 0.00 18331 sched lock 50 (usermode) 176 0% 82% 0.00 12653 sched lock 67 _thread_lock+0xf4 173 0% 82% 0.00 19722 sched lock 71 _thread_lock+0xf4 173 0% 82% 0.00 14363 sched lock 23 _thread_lock+0xf4 170 0% 82% 0.00 12317 sched lock 9 _thread_lock+0xf4 170 0% 83% 0.00 20132 sched lock 42 _thread_lock+0xf4 169 0% 83% 0.00 17504 sched lock 8 _thread_lock+0xf4 169 0% 83% 0.00 13336 sched lock 35 (usermode) 168 0% 83% 0.00 15815 sched lock 61 _thread_lock+0xf4 165 0% 84% 0.00 20771 sched lock 15 (usermode) 164 0% 84% 0.00 112678 sched lock 59 _thread_lock+0xf4 163 0% 84% 0.00 17939 sched lock 12 _thread_lock+0xf4 162 0% 84% 0.00 18406 sched lock 11 _thread_lock+0xf4 157 0% 84% 0.00 15220 sched lock 68 _thread_lock+0xf4 156 0% 85% 0.00 15912 sched lock 20 (usermode) 154 0% 85% 0.00 13283 sched lock 66 _thread_lock+0xf4 154 0% 85% 0.00 14730 sched lock 55 (usermode) 152 0% 85% 0.00 11578 sched lock 21 _thread_lock+0xf4 152 0% 86% 0.00 15864 sched lock 2 _thread_lock+0xf4 151 0% 86% 0.00 11652 sched lock 62 _thread_lock+0xf4 151 0% 86% 0.00 784533 sched lock 46 _thread_lock+0xf4 149 0% 86% 0.00 14355 sched lock 28 _thread_lock+0xf4 148 0% 86% 0.00 12301 sched lock 49 _thread_lock+0xf4 147 0% 87% 0.00 16404 sched lock 44 _thread_lock+0xf4 147 0% 87% 0.00 14418 sched lock 43 _thread_lock+0xf4 147 0% 87% 0.00 16650 sched lock 13 _thread_lock+0xf4 144 0% 87% 0.00 12648 sched lock 14 _thread_lock+0xf4 143 0% 87% 0.00 18639 sched lock 31 _thread_lock+0xf4 143 0% 88% 0.00 13022 sched lock 1 _thread_lock+0xf4 141 0% 88% 0.00 13563 sched lock 27 _thread_lock+0xf4 139 0% 88% 0.00 13493 sched lock 34 _thread_lock+0xf4 137 0% 88% 0.00 14971 sched lock 25 (usermode) 136 0% 88% 0.00 15543 sched lock 47 _thread_lock+0xf4 136 0% 88% 0.00 13108 sched lock 37 _thread_lock+0xf4 136 0% 89% 0.00 24060 sched lock 26 _thread_lock+0xf4 135 0% 89% 0.00 13818 sched lock 7 _thread_lock+0xf4 135 0% 89% 0.00 70204 sched lock 58 _thread_lock+0xf4 135 0% 89% 0.00 46743 sched lock 53 _thread_lock+0xf4 134 0% 89% 0.00 14662 sched lock 52 _thread_lock+0xf4 133 0% 90% 0.00 12244 sched lock 67 (usermode) 132 0% 90% 0.00 14981 sched lock 48 _thread_lock+0xf4 131 0% 90% 0.00 14872 sched lock 66 (usermode) 131 0% 90% 0.00 17823 sched lock 54 _thread_lock+0xf4 130 0% 90% 0.00 12052 sched lock 3 _thread_lock+0xf4 129 0% 90% 0.00 12124 sched lock 62 (usermode) 126 0% 91% 0.00 17187 sched lock 60 (usermode) 126 0% 91% 0.00 14844 sched lock 51 _thread_lock+0xf4 126 0% 91% 0.00 11232 sched lock 51 (usermode) 125 0% 91% 0.00 12523 sched lock 11 (usermode) 124 0% 91% 0.00 19506 sched lock 18 _thread_lock+0xf4 122 0% 91% 0.00 15778 sched lock 45 (usermode) 121 0% 92% 0.00 20503 sched lock 19 _thread_lock+0xf4 119 0% 92% 0.00 15019 sched lock 3 (usermode) 118 0% 92% 0.00 14027 sched lock 63 (usermode) 118 0% 92% 0.00 425042 sched lock 61 (usermode) 118 0% 92% 0.00 15071 sched lock 16 _thread_lock+0xf4 116 0% 92% 0.00 14131 sched lock 39 _thread_lock+0xf4 116 0% 93% 0.00 15367 sched lock 14 (usermode) 115 0% 93% 0.00 14403 sched lock 49 (usermode) 115 0% 93% 0.00 20520 sched lock 46 (usermode) 115 0% 93% 0.00 17165 sched lock 32 _thread_lock+0xf4 112 0% 93% 0.00 108598 sched lock 13 (usermode) 111 0% 93% 0.00 13674 sched lock 53 (usermode) 110 0% 94% 0.00 11495 sched lock 54 (usermode) 110 0% 94% 0.00 12827 sched lock 28 (usermode) 107 0% 94% 0.00 14914 sched lock 56 _thread_lock+0xf4 106 0% 94% 0.00 13942 sched lock 8 (usermode) 106 0% 94% 0.00 9215 sched lock 42 (usermode) 106 0% 94% 0.00 16019 sched lock 38 _thread_lock+0xf4 105 0% 94% 0.00 11249 sched lock 36 _thread_lock+0xf4 104 0% 95% 0.00 13170 sched lock 68 (usermode) 104 0% 95% 0.00 13832 sched lock 57 _thread_lock+0xf4 103 0% 95% 0.00 13132 sched lock 9 (usermode) 102 0% 95% 0.00 14535 sched lock 7 (usermode) 102 0% 95% 0.00 16099 sched lock 30 (usermode) 102 0% 95% 0.00 15448 sched lock 24 _thread_lock+0xf4 102 0% 95% 0.00 14036 sched lock 17 _thread_lock+0xf4 101 0% 96% 0.00 11226 sched lock 64 (usermode) 101 0% 96% 0.00 11301 sched lock 58 (usermode) 100 0% 96% 0.00 9800 sched lock 52 (usermode) 100 0% 96% 0.00 15336 sched lock 47 (usermode) 100 0% 96% 0.00 20734 sched lock 29 _thread_lock+0xf4 99 0% 96% 0.00 21490 sched lock 43 (usermode) 99 0% 96% 0.00 10920 sched lock 44 (usermode) 99 0% 97% 0.00 12545 sched lock 41 _thread_lock+0xf4 99 0% 97% 0.00 9747 sched lock 32 (usermode) 98 0% 97% 0.00 14443 sched lock 27 (usermode) 98 0% 97% 0.00 12894 sched lock 1 (usermode) 97 0% 97% 0.00 11595 sched lock 59 (usermode) 97 0% 97% 0.00 13635 sched lock 34 (usermode) 95 0% 97% 0.00 8733 sched lock 56 (usermode) 95 0% 97% 0.00 30849 sched lock 38 (usermode) 95 0% 98% 0.00 15387 sched lock 12 (usermode) 94 0% 98% 0.00 12060 sched lock 24 (usermode) 93 0% 98% 0.00 9304 sched lock 57 (usermode) 93 0% 98% 0.00 17353 sched lock 36 (usermode) 93 0% 98% 0.00 11619 sched lock 31 (usermode) 92 0% 98% 0.00 11032 sched lock 23 (usermode) 88 0% 98% 0.00 14717 sched lock 17 (usermode) 87 0% 98% 0.00 9806 sched lock 48 (usermode) 87 0% 99% 0.00 13649 sched lock 29 (usermode) 87 0% 99% 0.00 23363 sched lock 21 (usermode) 86 0% 99% 0.00 12958 sched lock 70 (usermode) 86 0% 99% 0.00 11283 sched lock 19 (usermode) 84 0% 99% 0.00 12263 sched lock 41 (usermode) 84 0% 99% 0.00 11947 sched lock 39 (usermode) 83 0% 99% 0.00 16555 sched lock 18 (usermode) 80 0% 99% 0.00 9699 sched lock 26 (usermode) 79 0% 99% 0.00 17765 sched lock 2 (usermode) 76 0% 100% 0.00 16678 sched lock 16 (usermode) 75 0% 100% 0.00 11624 sched lock 37 (usermode) 63 0% 100% 0.00 12442 sched lock 22 (usermode) 56 0% 100% 0.00 66909 sched lock 69 (usermode) 42 0% 100% 0.00 28970 sched lock 6 (usermode) 39 0% 100% 0.00 14360 sched lock 33 (usermode) 32 0% 100% 0.00 14493 sched lock 4 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 170731 events in 462.353 seconds (369 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 50512 30% 30% 0.00 36752429 vm object vm_fault_hold+0x1b28 23945 14% 44% 0.00 70840121 vm object vm_map_insert+0x758 19347 11% 55% 0.00 62552708 vm object vm_object_deallocate+0x800 14215 8% 63% 0.00 24715282 vm object (usermode) 10961 6% 70% 0.00 2120778 vm object 0xc000000003119e88 10766 6% 76% 0.00 45005365 vm object 0xc0000000030f2c88 8037 5% 81% 0.00 132654070 vm object vm_object_reference+0x134 6847 4% 85% 0.00 88472940 vm object vm_object_shadow+0x390 6397 4% 88% 0.00 81516209 vm object vm_object_shadow+0x378 5560 3% 92% 0.00 2508024 vm object vm_object_collapse+0x710 2478 1% 93% 0.00 21520049 vm object vm_object_deallocate+0x814 2098 1% 94% 0.00 2581831 vm object zfs_vmobject_wlock+0x9c 1841 1% 95% 0.00 1827432 vm object vm_page_sleep_if_busy+0x1f8 1821 1% 97% 0.00 105692572 vm object vm_imgact_map_page+0x3f0 1262 1% 97% 0.00 2643985 vm object 0xc00000000311a4c8 1260 1% 98% 0.00 18237783 vm object vnode_create_vobject+0x18c 918 1% 99% 0.00 2344672 vm object vm_fault_hold+0x2700 718 0% 99% 0.00 1453313 kernel vm object kmem_back_domain+0x340 693 0% 99% 0.00 147167282 vm object exec_map_first_page+0x780 404 0% 100% 0.00 1512813 kernel vm object (usermode) 243 0% 100% 0.00 1127256 kernel vm object 0xc0000000030eac08 148 0% 100% 0.00 2270006 vm object vmspace_fork+0x2d4 82 0% 100% 0.00 31127 Per-Filesystem Softdep Lock 0xc00000000308837c 77 0% 100% 0.00 1736723 vm object vm_object_split+0xbd0 29 0% 100% 0.00 49631 Per-Filesystem Softdep Lock (usermode) 25 0% 100% 0.00 42144 vm object bufdone+0x45c 8 0% 100% 0.00 1222782 vm object vm_fault_hold+0x1298 7 0% 100% 0.00 180609 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x5dc 4 0% 100% 0.00 42021 bufobj interlock (usermode) 4 0% 100% 0.00 355249 Per-Filesystem Softdep Lock 0xc000000002ce4a40 3 0% 100% 0.00 556460 bufobj interlock bqrelse+0x2a0 2 0% 100% 0.00 30378 bufobj interlock bufobj_wdrop+0x13c 2 0% 100% 0.00 28661 Per-Filesystem Softdep Lock 0xc0000000030983d8 2 0% 100% 0.00 205461 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc84 2 0% 100% 0.00 66683 Per-Filesystem Softdep Lock 0xc00000000308e270 2 0% 100% 0.00 42443 Per-Filesystem Softdep Lock softdep_setup_create+0x188 1 0% 100% 0.00 30399 tidhash tidhash_remove+0x13c 1 0% 100% 0.00 21910 tcpinp 0xc000000002eb6814 1 0% 100% 0.00 27282 vm object bdwrite+0x2f4 1 0% 100% 0.00 66723 Per-Filesystem Softdep Lock 0xc0000000030a7ea4 1 0% 100% 0.00 64606 Per-Filesystem Softdep Lock 0xc0000000030a3420 1 0% 100% 0.00 197215 Per-Filesystem Softdep Lock 0xc000000003098368 1 0% 100% 0.00 29141 Per-Filesystem Softdep Lock 0xc0000000030a33c8 1 0% 100% 0.00 100795 bufobj interlock 0xc0000000030b3bb0 1 0% 100% 0.00 49514 Per-Filesystem Softdep Lock 0xc000000003087414 1 0% 100% 0.00 44220 Per-Filesystem Softdep Lock 0xc0000000030954dc 1 0% 100% 0.00 63155 Per-Filesystem Softdep Lock softdep_freefile+0xbf4 ------------------------------------------------------------------------------- R/W writer blocked by readers: 153898 events in 462.353 seconds (333 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 49390 32% 32% 0.00 80448967 vm object vm_fault_hold+0x1b28 17678 11% 44% 0.00 26294465 vm object (usermode) 16165 11% 54% 0.00 52312464 vm object 0xc0000000030f2c88 13132 9% 63% 0.00 86094649 vm object vm_object_deallocate+0x800 11236 7% 70% 0.00 83635875 vm object vnode_create_vobject+0x18c 8898 6% 76% 0.00 79253800 vm object vm_object_deallocate+0x814 5592 4% 79% 0.00 2523087 vm object vm_fault_hold+0x1298 4893 3% 83% 0.00 65261399 vm object vm_map_insert+0x758 4120 3% 85% 0.00 2629586 vm object zfs_vmobject_wlock+0x9c 4090 3% 88% 0.00 2055835 vm object 0xc000000003119e88 3830 2% 90% 0.00 117204686 vm object vm_object_reference+0x134 3233 2% 92% 0.00 68584983 vm object vm_object_shadow+0x378 3191 2% 95% 0.00 2797843 vm object vm_fault_hold+0x2700 2270 1% 96% 0.00 119953597 vm object vm_imgact_map_page+0x3f0 1975 1% 97% 0.00 2709454 vm object 0xc00000000311a4c8 1296 1% 98% 0.00 1997053 vm object vm_object_collapse+0x710 1241 1% 99% 0.00 88083475 vm object vm_object_shadow+0x390 567 0% 99% 0.00 2392846 vm object vm_page_sleep_if_busy+0x1f8 505 0% 100% 0.00 1010991 vm object vmspace_fork+0x2d4 191 0% 100% 0.00 252887 bufobj interlock bqrelse+0x2a0 190 0% 100% 0.00 213019 bufobj interlock (usermode) 128 0% 100% 0.00 135223247 vm object exec_map_first_page+0x780 31 0% 100% 0.00 221929 bufobj interlock 0xc0000000030b3bb0 26 0% 100% 0.00 868362 vm object vm_object_split+0xbd0 17 0% 100% 0.00 68816 bufobj interlock bufobj_wdrop+0x13c 11 0% 100% 0.00 195406 bufobj interlock reassignbuf+0x350 2 0% 100% 0.00 140767 bufobj interlock bufobj_wref+0x114 ------------------------------------------------------------------------------- R/W reader blocked by writer: 36050 events in 462.353 seconds (78 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 31224 87% 87% 0.00 52614701 vm object __rw_rlock_int+0x9c 4822 13% 100% 0.00 43325400 vm object (usermode) 4 0% 100% 0.00 232657 bufobj interlock __rw_rlock_int+0x9c ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 38476 events in 462.353 seconds (83 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 33666 87% 87% 0.00 9484875 vm object __rw_rlock_int+0x9c 3205 8% 96% 0.00 9451553 vm object (usermode) 1554 4% 100% 0.00 328253 bufobj interlock __rw_rlock_int+0x9c 51 0% 100% 0.00 327323 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 596481 events in 462.353 seconds (1290 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 123906 21% 21% 0.00 3478888 vm object vm_fault_hold+0x1b28 95461 16% 37% 0.00 101452 vm object vm_fault_hold+0x2700 83958 14% 51% 0.00 1147407 vm object (usermode) 49712 8% 59% 0.00 4625255 vm object vm_map_insert+0x758 46317 8% 67% 0.00 1423065 vm object 0xc000000003119e88 33937 6% 73% 0.00 5007867 vm object vm_object_deallocate+0x800 22005 4% 76% 0.00 4153549 vm object 0xc0000000030f2c88 19825 3% 80% 0.00 2774130 vm object vm_object_shadow+0x378 18606 3% 83% 0.00 4741306 vm object vm_object_reference+0x134 12654 2% 85% 0.00 5347116 vm object vm_object_shadow+0x390 12061 2% 87% 0.00 3403348 vm object vm_object_collapse+0x710 9868 2% 89% 0.00 2637030 kernel vm object kmem_back_domain+0x340 7943 1% 90% 0.00 3805880 kernel vm object (usermode) 7714 1% 91% 0.00 1515784 kernel vm object 0xc0000000030eac08 7168 1% 92% 0.00 2274119 vm object vm_imgact_map_page+0x3f0 5679 1% 93% 0.00 3512880 vm object vm_object_deallocate+0x814 5443 1% 94% 0.00 3210183 vm object zfs_vmobject_wlock+0x9c 3832 1% 95% 0.00 3974192 vm object vm_page_sleep_if_busy+0x1f8 3012 1% 95% 0.00 40765 Per-Filesystem Softdep Lock (usermode) 2714 0% 96% 0.00 3723597 vm object 0xc00000000311a4c8 2439 0% 96% 0.00 70719 Per-Filesystem Softdep Lock 0xc000000002ce4a40 2358 0% 97% 0.00 206686 tidhash (usermode) 2009 0% 97% 0.00 5746459 vm object vnode_create_vobject+0x18c 1881 0% 97% 0.00 29911 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x5dc 1502 0% 98% 0.00 232147 tidhash tidhash_remove+0x13c 1297 0% 98% 0.00 30137 Per-Filesystem Softdep Lock softdep_freefile+0xbbc 1169 0% 98% 0.00 31834 Per-Filesystem Softdep Lock 0xc00000000308e270 1077 0% 98% 0.00 26593 vm object bufdone+0x45c 986 0% 98% 0.00 8526653 vm object exec_map_first_page+0x780 933 0% 98% 0.00 30650 Per-Filesystem Softdep Lock 0xc00000000308837c 925 0% 99% 0.00 17428 vm object bdwrite+0x2f4 643 0% 99% 0.00 35602 Per-Filesystem Softdep Lock softdep_setup_inofree+0x250 531 0% 99% 0.00 44096 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x258 497 0% 99% 0.00 37662 Per-Filesystem Softdep Lock 0xc0000000030983d8 477 0% 99% 0.00 1345904 vm object vmspace_fork+0x2d4 409 0% 99% 0.00 29093 bufobj interlock bqrelse+0x2a0 386 0% 99% 0.00 25616 Per-Filesystem Softdep Lock 0xc0000000030a3420 377 0% 99% 0.00 17574 Per-Filesystem Softdep Lock softdep_prealloc+0x460 343 0% 99% 0.00 42290 Per-Filesystem Softdep Lock 0xc000000003087600 315 0% 99% 0.00 187467 vm object vm_fault_hold+0x1298 311 0% 99% 0.00 15744 bufobj interlock bufobj_wdrop+0x13c 281 0% 99% 0.00 32866 Per-Filesystem Softdep Lock 0xc0000000030a33c8 258 0% 99% 0.00 38105 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc84 256 0% 99% 0.00 1143313 vm object vm_object_split+0xbd0 256 0% 100% 0.00 43519 Per-Filesystem Softdep Lock 0xc0000000030a7ea4 250 0% 100% 0.00 19268 bufobj interlock (usermode) 224 0% 100% 0.00 11022 vm object allocbuf+0xa84 178 0% 100% 0.00 20119 Per-Filesystem Softdep Lock 0xc000000003090bdc 174 0% 100% 0.00 51751 Per-Filesystem Softdep Lock 0xc0000000030954dc 170 0% 100% 0.00 23698 Per-Filesystem Softdep Lock softdep_slowdown+0x374 148 0% 100% 0.00 50279 Per-Filesystem Softdep Lock 0xc000000003098368 135 0% 100% 0.00 31786 Per-Filesystem Softdep Lock softdep_load_inodeblock+0x1d8 134 0% 100% 0.00 29833 Per-Filesystem Softdep Lock softdep_setup_allocdirect+0x170 128 0% 100% 0.00 57917 Per-Filesystem Softdep Lock 0xc0000000030a1440 126 0% 100% 0.00 27523 Per-Filesystem Softdep Lock softdep_freefile+0x58c 126 0% 100% 0.00 31645 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc54 126 0% 100% 0.00 16560 Per-Filesystem Softdep Lock softdep_setup_unlink+0x1b8 126 0% 100% 0.00 49911 Per-Filesystem Softdep Lock 0xc000000003085a14 103 0% 100% 0.00 41745 Per-Filesystem Softdep Lock softdep_setup_create+0x188 97 0% 100% 0.00 74348 Per-Filesystem Softdep Lock 0xc000000003087414 68 0% 100% 0.00 34278 Per-Filesystem Softdep Lock softdep_freefile+0x94c 65 0% 100% 0.00 6463 tcpinp (usermode) 55 0% 100% 0.00 36943 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 41 0% 100% 0.00 13645 bufobj interlock getblkx+0x6a4 30 0% 100% 0.00 41402 tidhash tidhash_add+0x16c 26 0% 100% 0.00 11142 tcpinp 0xc000000002eb6814 23 0% 100% 0.00 13500 bufobj interlock reassignbuf+0x350 21 0% 100% 0.00 36336 Per-Filesystem Softdep Lock 0xc00000000309b558 17 0% 100% 0.00 48419 bufobj interlock 0xc0000000030b3bb0 17 0% 100% 0.00 44921 Per-Filesystem Softdep Lock softdep_freefile+0xbf4 16 0% 100% 0.00 41913 Per-Filesystem Softdep Lock 0xc000000003085938 15 0% 100% 0.00 16189 vm object vnode_pager_setsize+0x2bc 14 0% 100% 0.00 11937 vm object vfs_busy_pages+0x2a8 12 0% 100% 0.00 34357 bufobj interlock bufobj_wref+0x114 12 0% 100% 0.00 36128 Per-Filesystem Softdep Lock softdep_setup_directory_add+0xcdc 12 0% 100% 0.00 46873 Per-Filesystem Softdep Lock 0xc000000003093024 12 0% 100% 0.00 49100 Per-Filesystem Softdep Lock 0xc00000000308a450 10 0% 100% 0.00 14425 Per-Filesystem Softdep Lock softdep_setup_blkmapdep+0x294 9 0% 100% 0.00 48996 Per-Filesystem Softdep Lock 0xc000000003095510 7 0% 100% 0.00 17992 vm object cluster_wbuild+0x52c 5 0% 100% 0.00 8651 Per-Filesystem Softdep Lock softdep_sync_buf+0x133c 4 0% 100% 0.00 11018 Per-Filesystem Softdep Lock 0xc0000000030a3774 3 0% 100% 0.00 91400 Per-Filesystem Softdep Lock softdep_freefile+0x5cc 3 0% 100% 0.00 1612 Per-Filesystem Softdep Lock 0xc00000000308ef48 2 0% 100% 0.00 5213 bufobj interlock vn_fsync_buf+0x244 2 0% 100% 0.00 1199 Per-Filesystem Softdep Lock softdep_setup_inomapdep+0x31c 1 0% 100% 0.00 5376 vm object vm_object_coalesce+0x3c0 1 0% 100% 0.00 6128 tcpinp 0xc000000002f872a8 1 0% 100% 0.00 22186 vfs hash vfs_hash_insert+0x214 1 0% 100% 0.00 211 Per-Filesystem Softdep Lock softdep_move_dependencies+0x264 ------------------------------------------------------------------------------- R/W writer spin on readers: 284908 events in 462.353 seconds (616 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 59349 21% 21% 0.00 5888696 vm object vm_fault_hold+0x1b28 44207 16% 36% 0.00 1228739 vm object (usermode) 40038 14% 50% 0.00 622190 vm object vm_fault_hold+0x2700 36083 13% 63% 0.00 1211475 vm object vm_fault_hold+0x1298 21148 7% 70% 0.00 3996618 vm object 0xc0000000030f2c88 15644 5% 76% 0.00 6586005 vm object vm_object_deallocate+0x800 14463 5% 81% 0.00 5945566 vm object vnode_create_vobject+0x18c 11510 4% 85% 0.00 5167263 vm object vm_object_deallocate+0x814 7017 2% 88% 0.00 4417703 vm object zfs_vmobject_wlock+0x9c 6459 2% 90% 0.00 3089321 vm object 0xc000000003119e88 6346 2% 92% 0.00 6297364 vm object vm_map_insert+0x758 4692 2% 94% 0.00 7703602 vm object vm_object_reference+0x134 4511 2% 95% 0.00 5313089 vm object vm_object_shadow+0x378 2809 1% 96% 0.00 5056567 vm object 0xc00000000311a4c8 2622 1% 97% 0.00 7792620 vm object vm_imgact_map_page+0x3f0 2298 1% 98% 0.00 2709904 vm object vm_object_collapse+0x710 1586 1% 99% 0.00 7084191 vm object vm_object_shadow+0x390 1519 1% 99% 0.00 376077 vm object vmspace_fork+0x2d4 830 0% 99% 0.00 6329434 vm object vm_page_sleep_if_busy+0x1f8 627 0% 100% 0.00 75411 bufobj interlock bqrelse+0x2a0 623 0% 100% 0.00 66410 bufobj interlock (usermode) 150 0% 100% 0.00 9043029 vm object exec_map_first_page+0x780 116 0% 100% 0.00 50265 bufobj interlock 0xc0000000030b3bb0 104 0% 100% 0.00 44029 bufobj interlock bufobj_wdrop+0x13c 61 0% 100% 0.00 868707 vm object vm_object_split+0xbd0 29 0% 100% 0.00 3863 ncbuc 0xc000000002dfd764 28 0% 100% 0.00 59694 bufobj interlock reassignbuf+0x350 25 0% 100% 0.00 36471 bufobj interlock bufobj_wref+0x114 9 0% 100% 0.00 3202 ncbuc (usermode) 3 0% 100% 0.00 1394 vfs hash vfs_hash_remove+0x13c 1 0% 100% 0.00 350 vfs hash (usermode) 1 0% 100% 0.00 332 bufobj interlock vn_fsync_buf+0x244 ------------------------------------------------------------------------------- R/W reader spin on writer: 156277 events in 462.353 seconds (338 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 117144 75% 75% 0.00 1183731 vm object __rw_rlock_int+0x9c 38465 25% 100% 0.00 456060 vm object (usermode) 510 0% 100% 0.00 23546 bufobj interlock __rw_rlock_int+0x9c 107 0% 100% 0.00 24923 bufobj interlock (usermode) 29 0% 100% 0.00 4398 ncbuc __rw_rlock_int+0x9c 11 0% 100% 0.00 1202 vfs hash __rw_rlock_int+0x9c 8 0% 100% 0.00 10829 ncbuc (usermode) 3 0% 100% 0.00 675 vfs hash (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 66213 events in 462.353 seconds (143 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 57117 86% 86% 0.00 1529118 vm object __rw_rlock_int+0x9c 6881 10% 97% 0.00 890954 vm object (usermode) 2098 3% 100% 0.00 174546 bufobj interlock __rw_rlock_int+0x9c 113 0% 100% 0.00 114905 bufobj interlock (usermode) 4 0% 100% 0.00 43032 ncbuc __rw_rlock_int+0x9c ------------------------------------------------------------------------------- SX exclusive block: 44944 events in 462.353 seconds (97 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 30077 67% 67% 0.00 268318 vm map (user) (usermode) 9158 20% 87% 0.00 279628 vm map (user) _vm_map_lock+0x138 2164 5% 92% 0.00 93602 vm map (user) _vm_map_lock_upgrade+0x180 785 2% 94% 0.00 6600754 vq->vq_lock (usermode) 655 1% 95% 0.00 6523463 vq->vq_lock vdev_queue_io+0x268 508 1% 96% 0.00 1620659 dd->dd_lock (usermode) 331 1% 97% 0.00 921852 dn->dn_struct_rwlock 0xc0000000025a6044 291 1% 98% 0.00 646390 dn->dn_struct_rwlock (usermode) 252 1% 98% 0.00 1222721 dd->dd_lock 0xc0000000025e75ec 134 0% 99% 0.00 2196084 dd->dd_lock dsl_dir_tempreserve_clear+0x88 93 0% 99% 0.00 1716054 dd->dd_lock dsl_dir_diduse_space+0x270 91 0% 99% 0.00 2619410 tx->tx_cpu[c].tc_open_lock txg_hold_open+0x1b8 63 0% 99% 0.00 3046365 h->hash_mutexes[i] dbuf_find+0x114 27 0% 99% 0.00 2269279 zilog->zl_itxg[i].itxg_lock zil_itx_assign+0x5cc 24 0% 99% 0.00 57243 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 24 0% 99% 0.00 3555794 h->hash_mutexes[i] (usermode) 24 0% 99% 0.00 3716880 db->db_mtx dbuf_find+0x12c 23 0% 100% 0.00 6877944 db->db_mtx (usermode) 21 0% 100% 0.00 4551761 dd->dd_lock dsl_dir_willuse_space+0x204 20 0% 100% 0.00 59260 spa->spa_alloc_locks[i] 0xc0000000026bcf70 16 0% 100% 0.00 227546 vq->vq_lock vdev_queue_io_done+0x278 14 0% 100% 0.00 94439 mc->mc_lock metaslab_class_throttle_unreserve+0x154 14 0% 100% 0.00 10918599 db->db_mtx 0xc0000000025a42b4 9 0% 100% 0.00 72972 spa->spa_alloc_locks[i] (usermode) 8 0% 100% 0.00 1694762 dp->dp_lock dsl_pool_undirty_space+0x18c 7 0% 100% 0.00 95656 ms->ms_lock metaslab_alloc_dva+0x1138 7 0% 100% 0.00 88219 ms->ms_lock (usermode) 7 0% 100% 0.00 68939 mg->mg_lock metaslab_alloc_dva+0x3a8 6 0% 100% 0.00 309091 vq->vq_lock 0xc000000002655fb4 6 0% 100% 0.00 7119261 tx->tx_cpu[c].tc_open_lock (usermode) 6 0% 100% 0.00 81445 mg->mg_lock (usermode) 6 0% 100% 0.00 2582229 h->hash_mutexes[i] dbuf_destroy+0x6dc 5 0% 100% 0.00 103343 zp->z_acl_lock 0xc000000002678cac 5 0% 100% 0.00 62113 mg->mg_lock metaslab_alloc_dva+0x7d4 5 0% 100% 0.00 2165411 dp->dp_lock (usermode) 5 0% 100% 0.00 166262 db->db_mtx dbuf_read+0xa44 4 0% 100% 0.00 49491 scl->scl_lock spa_config_exit+0x84 4 0% 100% 0.00 97320 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 4 0% 100% 0.00 118922 dp->dp_lock dsl_pool_need_dirty_delay+0x194 3 0% 100% 0.00 3191064 vq->vq_lock vdev_queue_io_done+0x12c 3 0% 100% 0.00 32537 scl->scl_lock spa_config_enter+0x140 3 0% 100% 0.00 36279 mc->mc_lock (usermode) 3 0% 100% 0.00 6169434 db->db_mtx 0xc0000000025a6010 3 0% 100% 0.00 105154 db->db_mtx dbuf_rele+0xbc 3 0% 100% 0.00 27228 db->db_mtx dmu_buf_will_dirty+0x2e8 2 0% 100% 0.00 25388 zio->io_lock 0xc0000000026c1474 2 0% 100% 0.00 53419 zilog->zl_itxg[i].itxg_lock (usermode) 2 0% 100% 0.00 38028 proctree fork1+0x28bc 2 0% 100% 0.00 48102 proctree kern_wait6+0x3e8 2 0% 100% 0.00 1810008 buf_hash_table.ht_locks[i].ht_lock arc_buf_access+0x488 2 0% 100% 0.00 2171571 buf_hash_table.ht_locks[i].ht_lock (usermode) 1 0% 100% 0.00 1284170 zs->zs_lock dmu_zfetch+0xf0 1 0% 100% 0.00 20654 zio->io_lock 0xc0000000026bfc60 1 0% 100% 0.00 39353 zio->io_lock zio_add_child+0x274 1 0% 100% 0.00 49033 zio->io_lock 0xc0000000026bf498 1 0% 100% 0.00 50420 zio->io_lock (usermode) 1 0% 100% 0.00 32918 rrl->rr_lock 0xc00000000260a56c 1 0% 100% 0.00 215136 tm->tm_lock trim_map_free+0x19c 1 0% 100% 0.00 8438050 dn->dn_struct_rwlock dnode_sync+0xd38 1 0% 100% 0.00 25078 hdl->sa_lock (usermode) 1 0% 100% 0.00 2280704 buf_hash_table.ht_locks[i].ht_lock 0xc00000000258bc0c 1 0% 100% 0.00 23475 as->as_lock aggsum_add+0x360 ------------------------------------------------------------------------------- SX shared block: 15902 events in 462.353 seconds (34 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 9687 61% 61% 0.00 934474 vm map (user) _sx_slock_int+0xa0 6193 39% 100% 0.00 2324599 vm map (user) (usermode) 17 0% 100% 0.00 3714863 dn->dn_struct_rwlock _sx_slock_int+0xa0 5 0% 100% 0.00 538062 dn->dn_struct_rwlock (usermode) ------------------------------------------------------------------------------- SX exclusive spin: 5073010 events in 462.353 seconds (10972 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2134374 42% 42% 0.00 357974 dd->dd_lock (usermode) 1372052 27% 69% 0.00 308002 dd->dd_lock 0xc0000000025e75ec 230690 5% 74% 0.00 122303 vm map (user) (usermode) 200037 4% 78% 0.00 67387 vm map (user) _vm_map_lock+0x138 179483 4% 81% 0.00 458086 dd->dd_lock dsl_dir_tempreserve_clear+0x88 130642 3% 84% 0.00 9555 zilog->zl_itxg[i].itxg_lock zil_itx_assign+0x5cc 95015 2% 86% 0.00 53096 h->hash_mutexes[i] dbuf_find+0x114 87847 2% 87% 0.00 23016 db->db_mtx dbuf_read+0xa44 86157 2% 89% 0.00 15451 db->db_mtx dbuf_rele+0xbc 65502 1% 90% 0.00 25998 db->db_mtx dbuf_find+0x12c 50426 1% 91% 0.00 38353 db->db_mtx (usermode) 50195 1% 92% 0.00 194587 zp->z_acl_lock 0xc000000002678cac 34865 1% 93% 0.00 216192 zp->z_acl_lock (usermode) 32164 1% 94% 0.00 371804 dd->dd_lock dsl_dir_willuse_space+0x204 25562 1% 94% 0.00 189365 tm->tm_lock trim_map_free+0x19c 24466 0% 95% 0.00 12628 rrl->rr_lock 0xc00000000260a56c 23262 0% 95% 0.00 1031805 vq->vq_lock (usermode) 20539 0% 95% 0.00 1064317 vq->vq_lock vdev_queue_io+0x268 19831 0% 96% 0.00 70973 vm map (user) _vm_map_lock_upgrade+0x180 15541 0% 96% 0.00 10901 zilog->zl_itxg[i].itxg_lock (usermode) 14900 0% 96% 0.00 11410 rrl->rr_lock rrw_exit+0x310 12093 0% 97% 0.00 41352 zio->io_lock 0xc0000000026bf498 11653 0% 97% 0.00 126411 h->hash_mutexes[i] (usermode) 11462 0% 97% 0.00 12736 dp->dp_lock dsl_pool_need_dirty_delay+0x194 11166 0% 97% 0.00 45242 zio->io_lock (usermode) 9092 0% 98% 0.00 283381 spa->spa_alloc_locks[i] 0xc0000000026bcf70 8893 0% 98% 0.00 361938 dd->dd_lock dsl_dir_diduse_space+0x270 8800 0% 98% 0.00 87508 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 7303 0% 98% 0.00 1033918 os->os_userused_lock (usermode) 6554 0% 98% 0.00 31766 zio->io_lock zio_add_child+0x274 6512 0% 98% 0.00 56479 zio->io_lock 0xc0000000026c1474 6150 0% 98% 0.00 12146 rrl->rr_lock (usermode) 6114 0% 99% 0.00 87845 mg->mg_lock metaslab_alloc_dva+0x3a8 5056 0% 99% 0.00 43710 ms->ms_lock metaslab_alloc_dva+0x1138 5008 0% 99% 0.00 114752 mg->mg_lock metaslab_alloc_dva+0x7d4 4870 0% 99% 0.00 1121788 os->os_userused_lock 0xc0000000025b7700 4275 0% 99% 0.00 18697 proctree kern_wait6+0x3e8 4270 0% 99% 0.00 277513 spa->spa_alloc_locks[i] (usermode) 3840 0% 99% 0.00 21091 mc->mc_lock metaslab_class_throttle_unreserve+0x154 3828 0% 99% 0.00 75858 vq->vq_lock vdev_queue_io_done+0x278 3609 0% 99% 0.00 1030356 os->os_userused_lock 0xc0000000025b7898 3430 0% 99% 0.00 351958 tm->tm_lock (usermode) 3126 0% 99% 0.00 124686 scl->scl_lock spa_config_exit+0x84 3112 0% 99% 0.00 96874 vq->vq_lock vdev_queue_io_done+0x12c 3110 0% 99% 0.00 129536 scl->scl_lock spa_config_enter+0x140 2990 0% 100% 0.00 112321 mg->mg_lock (usermode) 2191 0% 100% 0.00 16318 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 1732 0% 100% 0.00 77170 dn->dn_struct_rwlock 0xc0000000025a6044 1704 0% 100% 0.00 28763 db->db_mtx dmu_buf_will_dirty+0x2e8 1611 0% 100% 0.00 28567 dp->dp_lock (usermode) 1347 0% 100% 0.00 59995 ms->ms_lock (usermode) 1210 0% 100% 0.00 478649 vq->vq_lock 0xc000000002655fb4 1105 0% 100% 0.00 44662 dp->dp_lock dsl_pool_undirty_space+0x18c 894 0% 100% 0.00 45786 ms->ms_lock metaslab_free_concrete+0x2e8 890 0% 100% 0.00 19773 mc->mc_lock (usermode) 674 0% 100% 0.00 176622 as->as_lock aggsum_add+0x360 648 0% 100% 0.00 178924 dd->dd_lock dsl_dir_space_available+0x284 577 0% 100% 0.00 240743 dn->dn_struct_rwlock (usermode) 499 0% 100% 0.00 192136 h->hash_mutexes[i] dbuf_destroy+0x6dc 427 0% 100% 0.00 26774 db->db_mtx dmu_buf_hold_array_by_dnode+0x6d0 411 0% 100% 0.00 7246 zs->zs_lock dmu_zfetch+0xf0 408 0% 100% 0.00 138134 buf_hash_table.ht_locks[i].ht_lock arc_buf_access+0x488 360 0% 100% 0.00 18173 db->db_mtx dmu_buf_rele+0xbc 329 0% 100% 0.00 25831 proctree (usermode) 320 0% 100% 0.00 16480 dp->dp_lock dsl_pool_dirty_space+0x17c 307 0% 100% 0.00 1145382 tx->tx_cpu[c].tc_open_lock txg_hold_open+0x1b8 301 0% 100% 0.00 39554 proctree fork1+0x28bc 269 0% 100% 0.00 10472 dn->dn_mtx dnode_block_freed+0x320 246 0% 100% 0.00 109748 scl->scl_lock (usermode) 215 0% 100% 0.00 27381 db->db_mtx dmu_buf_replace_user+0x150 198 0% 100% 0.00 35247 zfsvfs->z_hold_mtx[i] zfs_zget+0x344 196 0% 100% 0.00 10965 dn->dn_mtx (usermode) 196 0% 100% 0.00 50372 db->db_mtx dbuf_dirty+0xcd0 190 0% 100% 0.00 29035 ds->ds_lock dsl_dataset_block_kill+0x614 175 0% 100% 0.00 5571 hdl->sa_lock sa_lookup_uio+0x198 157 0% 100% 0.00 25686 ds->ds_lock dsl_dataset_block_born+0x3d4 153 0% 100% 0.00 18730 proctree exit1+0x1aec 151 0% 100% 0.00 129003 h->hash_mutexes[i] 0xc0000000025a2988 151 0% 100% 0.00 15280 db->db_mtx 0xc0000000025a2414 142 0% 100% 0.00 7364 bpl->bpl_lock bplist_append+0x158 141 0% 100% 0.00 3340 hdl->sa_lock (usermode) 134 0% 100% 0.00 672133 buf_hash_table.ht_locks[i].ht_lock (usermode) 130 0% 100% 0.00 9436 hdl->sa_lock sa_bulk_lookup+0x154 124 0% 100% 0.00 6978 zilog->zl_itxg[i].itxg_lock zil_async_to_sync+0xd0 118 0% 100% 0.00 25434 ds->ds_lock dsl_dataset_block_kill+0x5e4 112 0% 100% 0.00 410909 tx->tx_cpu[c].tc_open_lock 0xc000000002637e00 105 0% 100% 0.00 6867 zs->zs_lock (usermode) 100 0% 100% 0.00 1557091 dn->dn_dbufs_mtx 0xc0000000025a2970 85 0% 100% 0.00 1955266 dn->dn_dbufs_mtx (usermode) 80 0% 100% 0.00 5666 db->db_mtx 0xc0000000025a492c 77 0% 100% 0.00 2920163 tx->tx_cpu[c].tc_open_lock (usermode) 67 0% 100% 0.00 199870 as->as_lock (usermode) 64 0% 100% 0.00 7879 dn->dn_mtx dnode_hold_impl+0x97c 61 0% 100% 0.00 49022 db->db_mtx 0xc000000002cf97a8 60 0% 100% 0.00 9158 ds->ds_lock dsl_dataset_check_quota+0x2f0 58 0% 100% 0.00 769711 dn->dn_dbufs_mtx dbuf_destroy+0x724 57 0% 100% 0.00 11495 proctree killjobc+0x600 57 0% 100% 0.00 781168 db->db_mtx 0xc0000000025a29a0 55 0% 100% 0.00 10931 bpl->bpl_lock (usermode) 54 0% 100% 0.00 37365 zfsvfs->z_hold_mtx[i] (usermode) 54 0% 100% 0.00 12325 proctree sys_setpgid+0x5c8 54 0% 100% 0.00 16585 dn->dn_mtx 0xc0000000025a605c 52 0% 100% 0.00 10732 mls->mls_lock multilist_insert+0x1d4 51 0% 100% 0.00 21684 ds->ds_lock (usermode) 51 0% 100% 0.00 11387 dn->dn_mtx dnode_diduse_space+0x184 45 0% 100% 0.00 5189 zp->z_range_lock zfs_range_unlock+0x3b4 41 0% 100% 0.00 117365 os->os_obj_lock (usermode) 40 0% 100% 0.00 14282 zio->io_lock 0xc0000000026bda18 37 0% 100% 0.00 9670 zilog->zl_itxg[i].itxg_lock zil_itx_assign+0xfc 37 0% 100% 0.00 11697 dn->dn_mtx dnode_rele+0xbc 33 0% 100% 0.00 8569 tl->tl_lock txg_list_add+0x16c 32 0% 100% 0.00 50840 dp->dp_lock dsl_pool_mos_diduse_space+0x160 31 0% 100% 0.00 2607 zp->z_range_lock zfs_range_lock+0x854 27 0% 100% 0.00 16798 db->db_mtx dbuf_prefetch+0x5ec 25 0% 100% 0.00 5525 vd->vdev_stat_lock vdev_stat_update+0x348 22 0% 100% 0.00 1512452 db->db_mtx 0xc0000000025a42b4 21 0% 100% 0.00 1713 mls->mls_lock multilist_remove+0x1d4 20 0% 100% 0.00 1933 allproc fork1+0x26bc 19 0% 100% 0.00 4359 mls->mls_lock (usermode) 17 0% 100% 0.00 18710 dp->dp_lock dmu_free_long_range+0x54c 17 0% 100% 0.00 309590 buf_hash_table.ht_locks[i].ht_lock 0xc00000000258b984 16 0% 100% 0.00 10303 tl->tl_lock (usermode) 14 0% 100% 0.00 5739 dn->dn_mtx dbuf_dirty+0xce8 13 0% 100% 0.00 3557 zp->z_range_lock (usermode) 13 0% 100% 0.00 1206 zombproc (usermode) 13 0% 100% 0.00 194947 dd->dd_lock dsl_dir_sync+0x148 13 0% 100% 0.00 2324393 db->db_mtx 0xc0000000025a6010 12 0% 100% 0.00 11814 db->db_mtx 0xc0000000025a4e8c 12 0% 100% 0.00 2205 allproc (usermode) 11 0% 100% 0.00 17096 zio->io_lock 0xc0000000026be2dc 11 0% 100% 0.00 5412 tm->tm_lock trim_map_write_done+0x188 11 0% 100% 0.00 290477 os->os_obj_lock 0xc0000000025b3054 11 0% 100% 0.00 1348 hdl->sa_lock sa_lookup+0x158 9 0% 100% 0.00 13206 dn->dn_mtx dmu_object_info_from_dnode+0x1d0 8 0% 100% 0.00 47666 db->db_mtx dbuf_dirty+0xdc0 8 0% 100% 0.00 1942173 buf_hash_table.ht_locks[i].ht_lock 0xc00000000258bc0c 8 0% 100% 0.00 984285 buf_hash_table.ht_locks[i].ht_lock arc_release+0x928 7 0% 100% 0.00 796 zio->io_lock 0xc0000000026bfc60 7 0% 100% 0.00 13209 dn->dn_mtx dbuf_dirty+0xd38 7 0% 100% 0.00 1527 allproc exit1+0x1a84 6 0% 100% 0.00 5134 zio->io_lock 0xc0000000026bb8e0 6 0% 100% 0.00 32256 db->db_mtx dbuf_read+0xdf4 5 0% 100% 0.00 10114 zio->io_lock 0xc0000000026bda48 5 0% 100% 0.00 3942 sa->sa_lock 0xc00000000260d738 5 0% 100% 0.00 26948 dp->dp_lock dmu_free_long_range+0x384 5 0% 100% 0.00 1213 dn->dn_mtx dbuf_destroy+0x6c0 4 0% 100% 0.00 5370 vd->vdev_stat_lock (usermode) 4 0% 100% 0.00 1066 db->db_mtx dnode_rele_and_unlock+0x18c 3 0% 100% 0.00 8753 zombproc exit1+0x1aa0 3 0% 100% 0.00 2606 tm->tm_lock trim_map_write_start+0x3f8 3 0% 100% 0.00 838 dn->dn_mtx dmu_tx_commit+0x80 3 0% 100% 0.00 436998 db->db_mtx 0xc0000000025a3cc4 3 0% 100% 0.00 110667 db->db_mtx 0xc0000000025a62f0 2 0% 100% 0.00 270 vc->vc_lock vdev_cache_write+0x27c 2 0% 100% 0.00 2889 sa->sa_lock 0xc00000000260e244 2 0% 100% 0.00 8063 sa->sa_lock 0xc000000002610a34 2 0% 100% 0.00 17403 sa->sa_lock (usermode) 2 0% 100% 0.00 193622 dn->dn_struct_rwlock dnode_sync+0xd38 2 0% 100% 0.00 11440 dn->dn_mtx dnode_add_ref+0x1b8 2 0% 100% 0.00 10760 dn->dn_mtx dmu_tx_assign+0x1b0 2 0% 100% 0.00 27424 as->as_lock aggsum_value+0x3a8 2 0% 100% 0.00 1168 as->as_buckets[i].asc_lock aggsum_add+0x330 1 0% 100% 0.00 350 zombproc proc_reap+0xca8 1 0% 100% 0.00 21194 zio->io_lock 0xc0000000026bf808 1 0% 100% 0.00 5683 zio->io_lock zio_wait+0x1e4 1 0% 100% 0.00 41870 zfsvfs->z_hold_mtx[i] zfs_znode_delete+0x224 1 0% 100% 0.00 39944 zfsvfs->z_hold_mtx[i] zfs_zinactive+0x1d4 1 0% 100% 0.00 807014 zap->zap_rwlock 0xc000000002668334 1 0% 100% 0.00 22259 tx->tx_sync_lock 0xc000000002cf97a8 1 0% 100% 0.00 1099 mls->mls_lock multilist_sublist_lock+0xbc 1 0% 100% 0.00 4746 pidhash fork1+0x27d0 1 0% 100% 0.00 7471 dn->dn_struct_rwlock dnode_new_blkid+0x4f8 1 0% 100% 0.00 1135 dn->dn_mtx dnode_sync+0xd08 1 0% 100% 0.00 251 dn->dn_mtx dbuf_dirty+0xd6c 1 0% 100% 0.00 22439 dn->dn_mtx dmu_objset_userquota_get_ids+0x5f0 1 0% 100% 0.00 1150 dn->dn_mtx 0xc0000000025b74ec 1 0% 100% 0.00 44161 dn->dn_mtx dnode_sync+0x6e0 1 0% 100% 0.00 132998 buf_hash_table.ht_locks[i].ht_lock arc_buf_destroy+0x330 1 0% 100% 0.00 85527 db->db_mtx dbuf_new_size+0x2dc 1 0% 100% 0.00 413690 db->db_mtx dbuf_dirty+0xdd8 ------------------------------------------------------------------------------- SX shared spin: 517376 events in 462.353 seconds (1119 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 398633 77% 77% 0.00 71164 vm map (user) _sx_slock_int+0xa0 76944 15% 92% 0.00 168381 vm map (user) (usermode) 30302 6% 98% 0.00 671 zf->zf_rwlock _sx_slock_int+0xa0 8351 2% 99% 0.00 678 zf->zf_rwlock (usermode) 2367 0% 100% 0.00 217411 dn->dn_struct_rwlock _sx_slock_int+0xa0 367 0% 100% 0.00 611124 dn->dn_struct_rwlock (usermode) 206 0% 100% 0.00 4556 zap->zap_rwlock (usermode) 206 0% 100% 0.00 3510 zap->zap_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- root@fledgling:/usr/src #