-------------------------------------------------------------- >>> Kernel build for GENERIC64 completed on Thu May 30 12:50:44 CDT 2019 -------------------------------------------------------------- >>> Kernel(s) GENERIC64 built in 176 seconds, ncpu: 72, make -j72 -------------------------------------------------------------- Adaptive mutex spin: 6449153 events in 177.342 seconds (36366 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1241343 19% 19% 0.00 11095 page pv moea64_remove_pages+0x214 1191560 18% 38% 0.00 9198 page pv moea64_enter+0x904 943630 15% 52% 0.00 4480 256 Bucket (usermode) 402212 6% 59% 0.00 15321 page pv (usermode) 295083 5% 63% 0.00 15910 vm active pagequeue 0xc0000000031072a8 204530 3% 66% 0.00 2118 UPVO entry (usermode) 189209 3% 69% 0.00 11574 pmap moea64_enter+0x918 151872 2% 72% 0.00 4155 256 Bucket uma_zalloc_arg+0x98c 114378 2% 73% 0.00 7260 struct mount mtx (usermode) 98211 2% 75% 0.00 5178 256 Bucket 0xc0000000030dedd8 93958 1% 76% 0.00 1733 UPVO entry 0xc0000000030df7ac 86829 1% 78% 0.00 7418 vnode interlock vget+0x2f8 75220 1% 79% 0.00 7777 struct mount mtx vfs_unbusy+0x15c 67715 1% 80% 0.00 422119 process lock ast+0x7e0 58939 1% 81% 0.00 34008 vm active pagequeue (usermode) 57200 1% 82% 0.00 3838 struct mount mtx vfs_rel+0x128 57038 1% 83% 0.00 8109 page pv moea64_enter+0x9a8 56851 1% 84% 0.00 15251 pmap (usermode) 54395 1% 84% 0.00 7521 vnode interlock (usermode) 52998 1% 85% 0.00 24788 pmap moea64_is_prefaultable+0x13c 52067 1% 86% 0.00 9471 umtxql (usermode) 46363 1% 87% 0.00 54440 kernelpmap moea64_kenter_attr+0x550 44646 1% 87% 0.00 39742 kernelpmap moea64_enter+0x918 43455 1% 88% 0.00 3142 UPVO entry uma_zfree_arg+0x5c8 41167 1% 89% 0.00 54105 kernelpmap moea64_kextract+0x174 41104 1% 89% 0.00 11982 struct mount mtx vfs_busy+0x434 40389 1% 90% 0.00 50847 kernelpmap moea64_remove+0x420 37526 1% 91% 0.00 55748 kernelpmap (usermode) 33958 1% 91% 0.00 91215 process lock kern_thr_exit+0x12c 33373 1% 92% 0.00 11487 page pv moea64_remove+0x22c 30137 0% 92% 0.00 7433 struct mount mtx vfs_ref+0x114 29761 0% 93% 0.00 21341 vm page vm_page_change_lock+0x160 28220 0% 93% 0.00 5421 256 Bucket uma_zalloc_arg+0x9e4 26675 0% 93% 0.00 73158 zio_write_issue 0xc000000002d74f5c 25809 0% 94% 0.00 21554 struct mount mtx insmntque1+0x3f8 25303 0% 94% 0.00 6963 umtxql 0xc000000002d1afb4 25124 0% 95% 0.00 2668 vnode interlock 0xc000000002e1ab8c 22082 0% 95% 0.00 80984 process lock kern_thr_exit+0x24c 21920 0% 95% 0.00 10411 umtxql 0xc000000002d1ccbc 17986 0% 96% 0.00 4269 struct mount vlist mtx _vdrop+0x6f4 16662 0% 96% 0.00 51242 zio_write_issue 0xc000000002cbb898 15430 0% 96% 0.00 89289 zio_write_issue taskqueue_enqueue+0x158 14707 0% 96% 0.00 5988 vnode interlock 0xc000000002e1ab70 14482 0% 96% 0.00 9988 dp_sync_taskq (usermode) 13879 0% 97% 0.00 111907 zio_write_issue (usermode) 12985 0% 97% 0.00 11316 zio_write_intr (usermode) 10964 0% 97% 0.00 442300 process lock (usermode) 10844 0% 97% 0.00 14006 vm active pagequeue vm_page_dequeue+0x1b4 10336 0% 97% 0.00 11365 umtxql 0xc000000002d1dae0 10080 0% 98% 0.00 3084 struct mount vlist mtx insmntque1+0x420 9943 0% 98% 0.00 59457 vm page vm_fault_hold+0x2724 9094 0% 98% 0.00 5477 vnode interlock 0xc0000000030c6708 8375 0% 98% 0.00 9427 dp_sync_taskq taskqueue_enqueue+0x158 8358 0% 98% 0.00 3617 umtxql 0xc000000002d1b260 7417 0% 98% 0.00 8200 dp_sync_taskq 0xc000000002cbb898 7126 0% 98% 0.00 8591 zio_write_intr 0xc000000002cbb898 6460 0% 98% 0.00 102843 vm page (usermode) 6443 0% 99% 0.00 11245 vm page free queue vm_page_alloc_domain_after+0x250 6365 0% 99% 0.00 14521 UMA Slabs (usermode) 6289 0% 99% 0.00 24483 dp_sync_taskq 0xc000000002d74f5c 5013 0% 99% 0.00 4496 umtxql 0xc000000002d1db10 4566 0% 99% 0.00 11249 zio_write_intr taskqueue_enqueue+0x158 4207 0% 99% 0.00 14238 UPVO entry uma_zalloc_arg+0x98c 3917 0% 99% 0.00 15441 zio_write_intr 0xc000000002d74f5c 3711 0% 99% 0.00 1480 vnode_free_list getnewvnode_reserve+0x480 3573 0% 99% 0.00 4623 struct mount vlist mtx _vhold+0x474 3415 0% 99% 0.00 7040 umtxql 0xc000000002d1db54 3412 0% 99% 0.00 7417 vnode interlock vref+0x1e8 2848 0% 99% 0.00 18146 vm page free queue 0xc000000003109cc8 2717 0% 99% 0.00 12658 vm page free queue (usermode) 2524 0% 99% 0.00 6985 select mtxpool selrecord+0x1dc 2230 0% 99% 0.00 3854 struct mount vlist mtx (usermode) 2006 0% 99% 0.00 20558 UMA Slabs 0xc0000000030dbaa4 1800 0% 99% 0.00 12106 umtxql 0xc000000002cbb898 1668 0% 99% 0.00 7774 abd_chunk (usermode) 1662 0% 99% 0.00 21022 UMA Slabs 0xc0000000030dedd8 1414 0% 100% 0.00 4017 ttymtx (usermode) 1389 0% 100% 0.00 1550 ttymtx 0xc000000002db0ba0 1348 0% 100% 0.00 4857 abd_chunk 0xc0000000030dbab8 1207 0% 100% 0.00 4127 vnode interlock vinactive+0x25c 1079 0% 100% 0.00 1102 vnode interlock 0xc0000000030c7358 922 0% 100% 0.00 1554 vnode_free_list (usermode) 869 0% 100% 0.00 3128 struct mount mtx vn_finished_write+0x174 869 0% 100% 0.00 5453 select mtxpool 0xc000000002d8b728 855 0% 100% 0.00 4073 pipe mutex 0xc000000002d8cc5c 725 0% 100% 0.00 11977 umtxql 0xc000000002d1b514 683 0% 100% 0.00 1383 pipe mutex 0xc000000002d8d764 636 0% 100% 0.00 4204 UPVO entry 0xc0000000030dbab8 611 0% 100% 0.00 14930 vm page free queue 0xc000000003109a08 599 0% 100% 0.00 8881 vm page 0xc0000000030e442c 593 0% 100% 0.00 1872 pipe mutex 0xc000000002d8bf18 553 0% 100% 0.00 6094 ttymtx ttyoutq_read_uio+0x34c 548 0% 100% 0.00 2469 struct mount mtx vn_finished_secondary_write+0x174 544 0% 100% 0.00 2692 pipe mutex (usermode) 496 0% 100% 0.00 68826 UMA Slabs uma_large_malloc_domain+0x2ac 467 0% 100% 0.00 365 pipe mutex 0xc000000002d8e004 453 0% 100% 0.00 1183 pipe mutex 0xc000000002d8c150 444 0% 100% 0.00 3866 struct mount mtx vn_start_write+0x1a0 434 0% 100% 0.00 3121 ttymtx 0xc000000002dafc0c 426 0% 100% 0.00 11253 256 Bucket 0xc0000000030df7ac 423 0% 100% 0.00 4366 ttymtx ttydisc_write+0x388 423 0% 100% 0.00 5937 select mtxpool (usermode) 387 0% 100% 0.00 6600 UPVO entry 0xc0000000030dedd8 349 0% 100% 0.00 6343 page pv moea64_page_wired_mappings+0x1b8 349 0% 100% 0.00 1524 kernel arena vmem_xalloc+0x8c0 341 0% 100% 0.00 12026 256 Bucket uma_zfree_arg+0x5c8 320 0% 100% 0.00 108966 nvme qpair lock 0xc000000002a0bae4 313 0% 100% 0.00 7423 CAM device lock 0xc000000002523fac 298 0% 100% 0.00 1799 umtxql 0xc000000002d1c97c 278 0% 100% 0.00 7644 CAM device lock (usermode) 277 0% 100% 0.00 1979 FFS (usermode) 274 0% 100% 0.00 1493 vnode interlock 0xc0000000030c612c 273 0% 100% 0.00 6619 struct mount mtx vfs_notify_upper+0x460 270 0% 100% 0.00 9193 CAM device lock 0xc000000002524924 269 0% 100% 0.00 677 256 Bucket 0xc0000000030dbab8 266 0% 100% 0.00 2127 FFS 0xc0000000030bff8c 255 0% 100% 0.00 1730 vnode_free_list _vhold+0x4c0 253 0% 100% 0.00 4086 sellck 0xc000000002d8af60 238 0% 100% 0.00 2158 vnode interlock 0xc000000002e02370 231 0% 100% 0.00 42749 vm active pagequeue 0xc00000000310d2e8 222 0% 100% 0.00 12888 ncvn (usermode) 216 0% 100% 0.00 1828 dnode_t 0xc0000000030dbab8 186 0% 100% 0.00 1242 TID lock (usermode) 185 0% 100% 0.00 1022 TID lock alloc_unr+0x13c 170 0% 100% 0.00 16095 UPVO entry uma_zalloc_arg+0x9e4 167 0% 100% 0.00 3926 sellck (usermode) 164 0% 100% 0.00 2156 vnode interlock rangelock_unlock+0xd0 148 0% 100% 0.00 1891 struct mount mtx 0xc000000002e1c2c8 144 0% 100% 0.00 12190 page pv 0xc00000000313beb4 142 0% 100% 0.00 4003 256 0xc0000000030dbab8 140 0% 100% 0.00 912 vnode interlock 0xc000000002cd9248 137 0% 100% 0.00 9318 UMA Slabs 0xc0000000030dbab8 135 0% 100% 0.00 1424 kernel arena domain vmem_xalloc+0x8c0 135 0% 100% 0.00 6502 dp_sync_taskq taskqueue_quiesce+0x2e8 134 0% 100% 0.00 150153 vm page 0xc0000000030e6b40 133 0% 100% 0.00 494 vnode interlock vop_stdset_text+0x130 132 0% 100% 0.00 2301 512 0xc0000000030dbab8 131 0% 100% 0.00 3299 struct mount mtx vn_start_secondary_write+0x1c4 131 0% 100% 0.00 1970 SLB table moea64_release_vsid+0x144 127 0% 100% 0.00 852 vm inactive pagequeue 0xc0000000031072a8 118 0% 100% 0.00 305 SLB table moea64_get_unique_vsid+0x2a8 113 0% 100% 0.00 170094 vm page dmu_read_pages+0x360 111 0% 100% 0.00 1513 vnode interlock 0xc0000000030b28cc 109 0% 100% 0.00 1439 ttymtx 0xc000000002da9fc4 109 0% 100% 0.00 3428 sellck 0xc000000002cbb898 109 0% 100% 0.00 786 VNODE 0xc0000000030dbab8 104 0% 100% 0.00 55051 vm page dmu_read_pages+0x7b8 97 0% 100% 0.00 1557 vm page vm_imgact_unmap_page+0x184 91 0% 100% 0.00 1593 kernel arena (usermode) 87 0% 100% 0.00 9846 UMA Slabs 0xc0000000030df7ac 83 0% 100% 0.00 579 vnode interlock _vhold+0x4a4 82 0% 100% 0.00 7109 vm page exec_unmap_first_page+0x188 82 0% 100% 0.00 3022 umtxql kern_umtx_wake+0x324 81 0% 100% 0.00 1588 ncvn 0xc000000002dfd074 80 0% 100% 0.00 122340 vm page vm_pageout_flush+0x280 75 0% 100% 0.00 10145 reseed mutex (usermode) 74 0% 100% 0.00 400 pipe mutex 0xc000000002d8f10c 72 0% 100% 0.00 2255 thread_ctor 0xc000000002d08504 72 0% 100% 0.00 6289 bge0 (usermode) 71 0% 100% 0.00 4683 select mtxpool 0xc000000002d8b158 70 0% 100% 0.00 3359 process_exec exec_new_vmspace+0x138 70 0% 100% 0.00 2367 FFS softdep_freefile+0x918 66 0% 100% 0.00 1562 Global Softdep Lock 0xc00000000308da18 65 0% 100% 0.00 4615 process_ctor 0xc000000002cc2108 63 0% 100% 0.00 6994 reseed mutex 0xc000000002a3c914 62 0% 100% 0.00 3185 vm page vm_imgact_map_page+0x43c 61 0% 100% 0.00 947 vm pgcache uma_zfree_arg+0x5c8 60 0% 100% 0.00 1051 pipe mutex 0xc000000002cbb898 60 0% 100% 0.00 2175 FFS 0xc0000000030973d0 54 0% 100% 0.00 11208 vm page vm_page_rename+0x224 54 0% 100% 0.00 10433 process lock 0xc000000002c86d88 54 0% 100% 0.00 40682 abd_chunk uma_zalloc_arg+0x98c 53 0% 100% 0.00 506 thread_ctor 0xc000000002d08698 53 0% 100% 0.00 3378 abd_chunk 0xc0000000030dedd8 52 0% 100% 0.00 1084 zfs_znode_cache 0xc0000000030dbab8 52 0% 100% 0.00 30188 kernel arena domain vmem_xfree+0x39c 50 0% 100% 0.00 45498 vm page exec_map_first_page+0x7b8 48 0% 100% 0.00 28180 vm page vm_fault_hold+0x27ec 47 0% 100% 0.00 2654 FFS 0xc000000003073d0c 46 0% 100% 0.00 21775 vm page 0xc000000002df23cc 46 0% 100% 0.00 3834 struct mount vlist mtx vfs_msync+0x1d4 46 0% 100% 0.00 1616 dmu_buf_impl_t 0xc0000000030dbab8 43 0% 100% 0.00 4905 metaslab_group_taskq taskqueue_enqueue+0x158 43 0% 100% 0.00 4662 dnode_t 0xc0000000030dedd8 41 0% 100% 0.00 441 CAM queue lock 0xc000000002511be8 40 0% 100% 0.00 159522 vm active pagequeue 0xc00000000310d62c 40 0% 100% 0.00 15597 pipe mutex 0xc000000002d8f144 39 0% 100% 0.00 6404 ttymtx 0xc000000002cc4e6c 39 0% 100% 0.00 2677 FFS ffs_freefile+0x358 38 0% 100% 0.00 5346 umtxql 0xc000000002d1c7f0 38 0% 100% 0.00 1228 kernel arena domain (usermode) 38 0% 100% 0.00 1248 FFS 0xc0000000030706a4 37 0% 100% 0.00 278 process_ctor 0xc000000002cc2494 36 0% 100% 0.00 1235 nvme qpair lock nvme_qpair_submit_request+0x120 36 0% 100% 0.00 27763 ncvn vn_vptocnp+0x4a4 34 0% 100% 0.00 4074 metaslab_group_taskq (usermode) 32 0% 100% 0.00 10751 vm page free queue vm_page_free_toq+0x208 32 0% 100% 0.00 2777 process_exit (usermode) 32 0% 100% 0.00 886 ncneglh cache_lookup+0x1608 31 0% 100% 0.00 791 zio_free_issue_6 taskqueue_enqueue+0x158 31 0% 100% 0.00 2394 dnode_t (usermode) 30 0% 100% 0.00 1232 zio_free_issue_0 taskqueue_enqueue+0x158 28 0% 100% 0.00 4288 thread_dtor 0xc000000002cc264c 28 0% 100% 0.00 24143 process lock ast+0x6e0 28 0% 100% 0.00 2105 FFS 0xc000000003075128 27 0% 100% 0.00 1721 zio_free_issue_3 taskqueue_enqueue+0x158 27 0% 100% 0.00 569 vm inactive pagequeue vm_page_dequeue+0x1b4 27 0% 100% 0.00 882 vm inactive pagequeue (usermode) 27 0% 100% 0.00 925 dnode_t uma_zalloc_arg+0x98c 26 0% 100% 0.00 376 zio_free_issue_4 taskqueue_enqueue+0x158 26 0% 100% 0.00 4165 vnode interlock zfs_zget+0x37c 26 0% 100% 0.00 1429 Global Softdep Lock (usermode) 26 0% 100% 0.00 4633 FFS ffs_alloc+0x474 26 0% 100% 0.00 2297 4096 0xc0000000030dbab8 25 0% 100% 0.00 332 process_exec exec_new_vmspace+0x5ec 25 0% 100% 0.00 766 FFS softdep_setup_freeblocks+0xc20 25 0% 100% 0.00 524 CAM queue lock (usermode) 24 0% 100% 0.00 526 zio_free_issue_7 taskqueue_enqueue+0x158 24 0% 100% 0.00 5314 vm active pagequeue 0xc00000000310d640 24 0% 100% 0.00 8741143 pmap 0xc00000000313bc8c 24 0% 100% 0.00 37377 abd_chunk uma_zfree_arg+0x5c8 23 0% 100% 0.00 484 so_snd sopoll_generic+0x470 23 0% 100% 0.00 1387 ncneglh cache_enter_time+0x1a34 23 0% 100% 0.00 2577 VNODE 0xc0000000030dedd8 22 0% 100% 0.00 726 zio_free_issue_7 (usermode) 22 0% 100% 0.00 232 vm map (system) vm_map_lookup+0x15c 22 0% 100% 0.00 1295 CAM queue lock 0xc00000000250c3e8 22 0% 100% 0.00 4335 256 (usermode) 21 0% 100% 0.00 914 vm pgcache uma_zalloc_arg+0x9e4 21 0% 100% 0.00 97359 vm page 0xc0000000030fd5a0 21 0% 100% 0.00 146955 vm page vm_object_madvise+0x54c 21 0% 100% 0.00 2925 thread_ctor 0xc000000002cc2318 21 0% 100% 0.00 6459 abd_chunk uma_zalloc_arg+0x9e4 21 0% 100% 0.00 4033 VNODE uma_zalloc_arg+0x9e4 21 0% 100% 0.00 2873 TID lock clean_unrhdrl+0x94 20 0% 100% 0.00 1883 512 (usermode) 19 0% 100% 0.00 932 zio_free_issue_2 taskqueue_enqueue+0x158 19 0% 100% 0.00 5341 umtxql 0xc000000002d1982c 19 0% 100% 0.00 1466 so_snd (usermode) 19 0% 100% 0.00 1495 sa_cache 0xc0000000030dbab8 19 0% 100% 0.00 4827 bge0 0xc000000002824dac 19 0% 100% 0.00 1046 VNODE uma_zalloc_arg+0x98c 19 0% 100% 0.00 1018 SLB table (usermode) 19 0% 100% 0.00 1064 FFS 0xc0000000030a6938 18 0% 100% 0.00 609 zio_free_issue_5 taskqueue_enqueue+0x158 18 0% 100% 0.00 668 zio_free_issue_1 taskqueue_enqueue+0x158 18 0% 100% 0.00 1549 zfs_znode_cache (usermode) 18 0% 100% 0.00 3772 umtxql 0xc000000002d28064 18 0% 100% 0.00 3781 umtxql 0xc000000002d1bd18 18 0% 100% 0.00 1247 so_snd tcp_do_segment+0x3ca0 18 0% 100% 0.00 3467 256 0xc0000000030dedd8 17 0% 100% 0.00 666 zio_free_issue_6 0xc000000002cbb898 17 0% 100% 0.00 1498 zio_free_issue_3 0xc000000002cbb898 17 0% 100% 0.00 2835 zfs_znode_cache 0xc0000000030dedd8 17 0% 100% 0.00 770 ncneglh cache_lookup+0x1638 17 0% 100% 0.00 2613 dnode_t uma_zalloc_arg+0x9e4 17 0% 100% 0.00 3393 512 0xc0000000030dedd8 16 0% 100% 0.00 710 zio_free_issue_2 0xc000000002cbb898 16 0% 100% 0.00 77831 nvme qpair lock (usermode) 16 0% 100% 0.00 616 ncvn 0xc000000002dfd060 16 0% 100% 0.00 22948 bufq dirty lock (usermode) 15 0% 100% 0.00 442 zio_free_issue_6 0xc000000002d74f5c 15 0% 100% 0.00 673 zio_free_issue_5 0xc000000002d74f5c 15 0% 100% 0.00 214 vm pgcache uma_zalloc_arg+0x98c 15 0% 100% 0.00 13411 reseed mutex 0xc000000002a3cbd4 15 0% 100% 0.00 2249 process lock kern_sigprocmask+0x378 15 0% 100% 0.00 6490 process lock exit1+0x1424 15 0% 100% 0.00 639 ncneglh (usermode) 15 0% 100% 0.00 1089 dmu_buf_impl_t uma_zalloc_arg+0x98c 15 0% 100% 0.00 787 256 uma_zalloc_arg+0x98c 14 0% 100% 0.00 477 zio_free_issue_7 0xc000000002cbb898 14 0% 100% 0.00 661 zio_free_issue_2 (usermode) 14 0% 100% 0.00 2013 umtxql 0xc000000002d24718 14 0% 100% 0.00 3834 process_exit exit1+0x564 14 0% 100% 0.00 1592 TID lock 0xc000000002d08b4c 13 0% 100% 0.00 606 zio_free_issue_5 (usermode) 13 0% 100% 0.00 963 zio_free_issue_4 0xc000000002d74f5c 13 0% 100% 0.00 860 zio_free_issue_1 0xc000000002cbb898 13 0% 100% 0.00 459 zio_free_issue_1 (usermode) 13 0% 100% 0.00 760 so_snd sosend_generic+0x3bc 13 0% 100% 0.00 1243 procid proc_id_clear+0x164 13 0% 100% 0.00 356 ncnegl cache_enter_time+0x1a00 12 0% 100% 0.00 947 zio_free_issue_7 0xc000000002d74f5c 12 0% 100% 0.00 406 zio_free_issue_0 0xc000000002cbb898 12 0% 100% 0.00 22025 vm page vm_page_sleep_if_busy+0x1c8 12 0% 100% 0.00 1483 sa_cache uma_zalloc_arg+0x98c 12 0% 100% 0.00 308 FFS ffs_balloc_ufs2+0x1b98 11 0% 100% 0.00 542 zio_null_intr taskqueue_enqueue+0x158 11 0% 100% 0.00 4439 zio_free_issue_3 0xc000000002d74f5c 11 0% 100% 0.00 1689 zio_free_issue_3 (usermode) 11 0% 100% 0.00 551 thread_ctor 0xc000000002cc24c0 11 0% 100% 0.00 1494 sleep mtxpool (usermode) 11 0% 100% 0.00 3631 process_exec (usermode) 11 0% 100% 0.00 3382 dmu_buf_impl_t 0xc0000000030dedd8 11 0% 100% 0.00 3726 512 uma_zalloc_arg+0x9e4 10 0% 100% 0.00 2446 FFS 0xc000000003076528 9 0% 100% 0.00 331 zio_free_issue_4 0xc000000002cbb898 9 0% 100% 0.00 651 zio_free_issue_1 0xc000000002d74f5c 9 0% 100% 0.00 465 zio_free_issue_0 0xc000000002d74f5c 9 0% 100% 0.00 4768 umtxql 0xc000000002d1bd7c 9 0% 100% 0.00 1463 so_snd tcp_output+0x1de8 9 0% 100% 0.00 180 procid fork1+0x279c 9 0% 100% 0.00 456 process_exit exit1+0x1b10 9 0% 100% 0.00 509 ncnegl cache_lookup+0x15b0 9 0% 100% 0.00 19151 kernelpmap 0xc00000000313bc8c 9 0% 100% 0.00 296 g_disk_start (usermode) 9 0% 100% 0.00 309 CAM queue lock xpt_action_default+0x1254 9 0% 100% 0.00 491 512 uma_zalloc_arg+0x98c 8 0% 100% 0.00 312 zio_free_issue_5 0xc000000002cbb898 8 0% 100% 0.00 1279 zio_free_issue_0 (usermode) 8 0% 100% 0.00 606 vnode_free_list getnewvnode+0x3cc 8 0% 100% 0.00 228297 vm page vm_fault_hold+0x27a8 8 0% 100% 0.00 3466 vm object_list (usermode) 8 0% 100% 0.00 357 thread_dtor 0xc000000002cc29f8 8 0% 100% 0.00 1829 thread_dtor (usermode) 8 0% 100% 0.00 189 sleep mtxpool foffset_lock+0x1c0 8 0% 100% 0.00 2395 process lock kern_sigaction+0x67c 8 0% 100% 0.00 1559 kernel arena vmem_xfree+0x39c 7 0% 100% 0.00 805 zio_free_issue_4 (usermode) 7 0% 100% 0.00 852 zfs_znode_cache uma_zalloc_arg+0x98c 7 0% 100% 0.00 6457 umtxql 0xc000000002d280dc 7 0% 100% 0.00 2773 umtxql 0xc000000002d23f30 7 0% 100% 0.00 2283 thread_ctor (usermode) 7 0% 100% 0.00 445 sleep mtxpool foffset_unlock+0x1b0 7 0% 100% 0.00 254 procid (usermode) 7 0% 100% 0.00 627 bufq dirty lock 0xc000000002dea19c 7 0% 100% 0.00 4122 TID lock free_unr+0x45c 6 0% 100% 0.00 909 zio_null_issue (usermode) 6 0% 100% 0.00 6512 sa_cache 0xc0000000030dedd8 6 0% 100% 0.00 7770 reseed mutex 0xc000000002a3ce4c 6 0% 100% 0.00 1659 process_dtor (usermode) 6 0% 100% 0.00 1175 ncnegl cache_enter_time+0x1b24 6 0% 100% 0.00 231 jail mutex prison_proc_free+0x1a8 6 0% 100% 0.00 925 S VFS Cache 0xc0000000030dbab8 6 0% 100% 0.00 203 Global Softdep Lock 0xc00000000308f498 6 0% 100% 0.00 2029 FFS 0xc00000000307067c 6 0% 100% 0.00 1443 4096 (usermode) 5 0% 100% 0.00 1435 zio_free_intr taskqueue_enqueue+0x158 5 0% 100% 0.00 1951 vnode_free_list 0xc000000002e1adb8 5 0% 100% 0.00 1722 umtxql 0xc000000002d25c2c 5 0% 100% 0.00 572 umtxql 0xc000000002d246c4 5 0% 100% 0.00 383 thread_dtor 0xc000000002d08b88 5 0% 100% 0.00 253 sleep mtxpool g_io_request+0x414 5 0% 100% 0.00 1996 process_dtor 0xc000000002cc2850 5 0% 100% 0.00 17866 metaslab_group_taskq 0xc000000002d74f5c 5 0% 100% 0.00 2124 VNODE (usermode) 5 0% 100% 0.00 769 Syncer mtx brelvp+0x300 5 0% 100% 0.00 4404 4096 0xc0000000030dedd8 5 0% 100% 0.00 5768 256 uma_zalloc_arg+0x9e4 4 0% 100% 0.00 196 zio_free_issue_2 0xc000000002d74f5c 4 0% 100% 0.00 1267 vm page vm_page_pa_tryrelock+0x298 4 0% 100% 0.00 1861 vm page 0xc0000000026a4258 4 0% 100% 0.00 1621 thread_dtor 0xc000000002d0887c 4 0% 100% 0.00 232 struct mount vlist mtx 0xc000000002e1c324 4 0% 100% 0.00 405 ncvn cache_lookup+0xa9c 4 0% 100% 0.00 498 ncneglh 0xc000000002df921c 4 0% 100% 0.00 1673 ncnegl 0xc000000002df9200 4 0% 100% 0.00 3217 metaslab_group_taskq 0xc000000002cbb898 4 0% 100% 0.00 239 jail mutex 0xc000000002cb90ec 4 0% 100% 0.00 10060 64 (usermode) 3 0% 100% 0.00 328 zio_free_issue_6 (usermode) 3 0% 100% 0.00 1694 zio_free_intr 0xc000000002d74f5c 3 0% 100% 0.00 3893 zfs_znode_cache uma_zalloc_arg+0x9e4 3 0% 100% 0.00 230 vm pgcache (usermode) 3 0% 100% 0.00 10947 vm page 0xc0000000030e3d54 3 0% 100% 0.00 578 vm page 0xc0000000026a4214 3 0% 100% 0.00 3212 umtxql 0xc000000002d23eec 3 0% 100% 0.00 2114 umtxql 0xc000000002d1cf40 3 0% 100% 0.00 544 so_snd sbdrop+0x130 3 0% 100% 0.00 6492 sa_cache uma_zalloc_arg+0x9e4 3 0% 100% 0.00 241 process_dtor 0xc000000002cc29cc 3 0% 100% 0.00 2204 process lock thread_create+0x6f0 3 0% 100% 0.00 221 process lock kthread_suspend_check+0x184 3 0% 100% 0.00 14561 process lock fork_rfppwait+0x44c 3 0% 100% 0.00 1342 pmap trap+0x145c 3 0% 100% 0.00 3162 kernel arena domain 0xc000000002d8370c 3 0% 100% 0.00 241 ncnegl (usermode) 3 0% 100% 0.00 4332 dmu_buf_impl_t (usermode) 3 0% 100% 0.00 31271 64 0xc0000000030dbab8 3 0% 100% 0.00 2923 4096 uma_zalloc_arg+0x9e4 3 0% 100% 0.00 3774 16384 0xc0000000030dedd8 3 0% 100% 0.00 224 16384 0xc0000000030dbab8 2 0% 100% 0.00 545328 zio_read_intr_7 (usermode) 2 0% 100% 0.00 496 zio_read_intr_3 0xc000000002cbb898 2 0% 100% 0.00 236 zio_read_intr_1 taskqueue_enqueue+0x158 2 0% 100% 0.00 520 zio_read_intr_0 taskqueue_enqueue+0x158 2 0% 100% 0.00 221 zio_null_issue taskqueue_enqueue+0x158 2 0% 100% 0.00 1303 vm object_list 0xc0000000030f8a40 2 0% 100% 0.00 20501 umtxql 0xc000000002d24704 2 0% 100% 0.00 5349 ttymtx 0xc000000002cbb898 2 0% 100% 0.00 43139 sleep mtxpool bdone+0x144 2 0% 100% 0.00 4830 sa_cache (usermode) 2 0% 100% 0.00 1533 process_ctor (usermode) 2 0% 100% 0.00 977 process lock proc_reap+0xda8 2 0% 100% 0.00 1627 pbuf uma_zalloc_arg+0x9e4 2 0% 100% 0.00 357 exec args kva exec_alloc_args+0x1e4 2 0% 100% 0.00 216 bufq dirty lock 0xc000000002de93e8 2 0% 100% 0.00 5440 bge0 0xc000000002822ab0 2 0% 100% 0.00 4826 S VFS Cache 0xc0000000030dedd8 2 0% 100% 0.00 1360 S VFS Cache (usermode) 2 0% 100% 0.00 427 RADIX NODE uma_zfree_arg+0x5c8 2 0% 100% 0.00 164 Global Softdep Lock 0xc0000000030a21a8 2 0% 100% 0.00 23714 64 0xc0000000030dedd8 1 0% 100% 0.00 293 zio_read_intr_6 taskqueue_enqueue+0x158 1 0% 100% 0.00 794 zio_read_intr_6 0xc000000002d74f5c 1 0% 100% 0.00 5849 zio_read_intr_3 taskqueue_enqueue+0x158 1 0% 100% 0.00 343 zio_read_intr_2 taskqueue_enqueue+0x158 1 0% 100% 0.00 308484 zio_read_intr_4 taskqueue_enqueue+0x158 1 0% 100% 0.00 5667 zio_read_intr_3 0xc000000002d74f5c 1 0% 100% 0.00 179 zio_read_intr_4 0xc000000002d74f5c 1 0% 100% 0.00 721 zio_read_intr_6 0xc000000002cbb898 1 0% 100% 0.00 242 zio_null_intr 0xc000000002d74f5c 1 0% 100% 0.00 155 zio_null_intr 0xc000000002cbb898 1 0% 100% 0.00 195 zio_null_intr (usermode) 1 0% 100% 0.00 369 zio_read_intr_0 0xc000000002cbb898 1 0% 100% 0.00 211 zio_read_intr_1 0xc000000002cbb898 1 0% 100% 0.00 215 zio_read_intr_1 0xc000000002d74f5c 1 0% 100% 0.00 293 vnode interlock 0xc0000000030a55d8 1 0% 100% 0.00 258576 xhci0 xhci_interrupt+0x428 1 0% 100% 0.00 137 zio_cache uma_zfree_arg+0x5c8 1 0% 100% 0.00 141 vm page vm_fault_hold+0x10f0 1 0% 100% 0.00 396 vm map (system) (usermode) 1 0% 100% 0.00 131268 vm page vm_page_readahead_finish+0x25c 1 0% 100% 0.00 182 vnode interlock 0xc000000002c2c028 1 0% 100% 0.00 2062 struct mount mtx 0xc000000002e2071c 1 0% 100% 0.00 232 so_rcv tcp_do_segment+0x3c24 1 0% 100% 0.00 1704 so_rcv sopoll_generic+0x484 1 0% 100% 0.00 232 procid proc_id_set+0x164 1 0% 100% 0.00 5690 umtxql 0xc000000002d19868 1 0% 100% 0.00 3017 umtxql 0xc000000002d2474c 1 0% 100% 0.00 347 umtxql 0xc000000002d280c8 1 0% 100% 0.00 1756 umtxql 0xc000000002d28104 1 0% 100% 0.00 3094 dmu_buf_impl_t uma_zalloc_arg+0x9e4 1 0% 100% 0.00 240 XPT topology lock (usermode) 1 0% 100% 0.00 3736 XPT topology lock xpt_lock_buses+0xac 1 0% 100% 0.00 201 THREAD 0xc0000000030dbab8 1 0% 100% 0.00 1996 RADIX NODE (usermode) 1 0% 100% 0.00 2221 RADIX NODE uma_zalloc_arg+0x98c 1 0% 100% 0.00 2121 THREAD 0xc0000000030dedd8 1 0% 100% 0.00 191 4096 uma_zalloc_arg+0x98c 1 0% 100% 0.00 334 16384 uma_zalloc_arg+0x98c 1 0% 100% 0.00 449 CAM device lock 0xc00000000252337c 1 0% 100% 0.00 330 FFS ffs_balloc_ufs2+0xb8c 1 0% 100% 0.00 143 dqhlock quotaref+0x198 1 0% 100% 0.00 184 jail mutex prison_proc_hold+0x114 1 0% 100% 0.00 141 jail mutex (usermode) 1 0% 100% 0.00 217 g_disk_start 0xc000000002c1fe10 1 0% 100% 0.00 1242 exec args kva (usermode) 1 0% 100% 0.00 1154 g_bio (usermode) 1 0% 100% 0.00 310 pipe mutex 0xc000000002d8dc64 1 0% 100% 0.00 3273 pbuf 0xc0000000030d7c60 1 0% 100% 0.00 275 pbuf 0xc0000000030df7ac 1 0% 100% 0.00 224 process lock fork_rfppwait+0x104 1 0% 100% 0.00 6546 process lock fork_rfppwait+0x1f0 1 0% 100% 0.00 1805 process lock fork1+0x2870 1 0% 100% 0.00 332 process lock fork1+0x29bc 1 0% 100% 0.00 2785 process lock exit1+0x1a54 1 0% 100% 0.00 5293 process lock proc_reap+0xc38 1 0% 100% 0.00 11170 process lock 0xc000000002cbb898 1 0% 100% 0.00 580 process lock killjobc+0x5c8 1 0% 100% 0.00 6215 process lock 0xc000000002d6ec78 1 0% 100% 0.00 1250 process lock tty_wait_background+0x2b4 ------------------------------------------------------------------------------- Adaptive mutex block: 1534 events in 177.342 seconds (9 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 264 17% 17% 0.00 467651 page pv moea64_enter+0x904 159 10% 28% 0.00 458990 page pv moea64_remove_pages+0x214 90 6% 33% 0.00 401092 process lock ast+0x7e0 82 5% 39% 0.00 131232 kernelpmap moea64_kextract+0x174 78 5% 44% 0.00 832106 page pv moea64_remove+0x22c 74 5% 49% 0.00 67336 kernelpmap moea64_enter+0x918 63 4% 53% 0.00 590007 page pv (usermode) 50 3% 56% 0.00 61327 struct mount mtx insmntque1+0x3f8 49 3% 59% 0.00 30890 zio_write_intr taskqueue_enqueue+0x158 48 3% 62% 0.00 168571 pmap moea64_enter+0x918 44 3% 65% 0.00 159881 nvme qpair lock 0xc000000002a0bae4 35 2% 68% 0.00 116487 kernelpmap (usermode) 34 2% 70% 0.00 65912 zio_write_issue 0xc000000002d74f5c 30 2% 72% 0.00 111566 vnode interlock vget+0x2f8 25 2% 73% 0.00 35782 zio_write_intr 0xc000000002cbb898 25 2% 75% 0.00 1695319 vm page vm_page_change_lock+0x160 22 1% 76% 0.00 55645 zio_write_intr (usermode) 19 1% 78% 0.00 31236 umtxql 0xc000000002d1ccbc 19 1% 79% 0.00 28735 sellck 0xc000000002cbb898 18 1% 80% 0.00 76421 vm active pagequeue 0xc0000000031072a8 17 1% 81% 0.00 38540 zio_write_intr 0xc000000002d74f5c 15 1% 82% 0.00 56209 page pv moea64_enter+0x9a8 15 1% 83% 0.00 239518 kernelpmap moea64_remove+0x420 14 1% 84% 0.00 34437 zio_write_issue 0xc000000002cbb898 14 1% 85% 0.00 143959 pmap moea64_is_prefaultable+0x13c 13 1% 86% 0.00 967489 vnode interlock (usermode) 13 1% 87% 0.00 676850 vm page vm_fault_hold+0x2724 13 1% 87% 0.00 190084 process lock kern_thr_exit+0x12c 12 1% 88% 0.00 38071 zio_write_issue taskqueue_enqueue+0x158 11 1% 89% 0.00 49040 struct mount vlist mtx _vdrop+0x6f4 10 1% 90% 0.00 341754 struct mount mtx (usermode) 10 1% 90% 0.00 28519 CAM device lock 0xc000000002524924 8 1% 91% 0.00 64039 pmap (usermode) 7 0% 91% 0.00 118317 umtxql (usermode) 7 0% 92% 0.00 179826 process lock kern_thr_exit+0x24c 6 0% 92% 0.00 50251 struct mount vlist mtx insmntque1+0x420 6 0% 93% 0.00 57951 struct mount mtx vfs_busy+0x434 6 0% 93% 0.00 65257 kernelpmap moea64_kenter_attr+0x550 5 0% 93% 0.00 1373751 vm page (usermode) 5 0% 94% 0.00 621265 abd_chunk uma_zalloc_arg+0x98c 4 0% 94% 0.00 47209 zio_write_issue (usermode) 4 0% 94% 0.00 55804 vm page free queue vm_page_alloc_domain_after+0x250 4 0% 94% 0.00 38533 struct mount vlist mtx _vhold+0x474 4 0% 95% 0.00 31821 sellck (usermode) 4 0% 95% 0.00 31372 dp_sync_taskq 0xc000000002cbb898 4 0% 95% 0.00 458988 abd_chunk (usermode) 3 0% 95% 0.00 29449 xhci0 0xc000000002cbb898 3 0% 96% 0.00 50733 xhci0 (usermode) 3 0% 96% 0.00 184176 vm active pagequeue (usermode) 3 0% 96% 0.00 35273 umtxql 0xc000000002d1afb4 3 0% 96% 0.00 96248 process lock (usermode) 3 0% 96% 0.00 70759 dp_sync_taskq (usermode) 3 0% 96% 0.00 99660 UMA Slabs (usermode) 2 0% 97% 0.00 30398 vnode interlock 0xc000000002e1ab70 2 0% 97% 0.00 177690 vm page free queue 0xc000000003109a08 2 0% 97% 0.00 31568 vm page free queue (usermode) 2 0% 97% 0.00 32883 umtxql 0xc000000002d1dae0 2 0% 97% 0.00 28228 umass0 0xc000000002b366c8 2 0% 97% 0.00 37233 struct mount mtx vfs_unbusy+0x15c 2 0% 97% 0.00 48294 select mtxpool 0xc000000002d8b728 2 0% 98% 0.00 29690 pipe mutex 0xc000000002cbb898 2 0% 98% 0.00 39333 dp_sync_taskq 0xc000000002d74f5c 2 0% 98% 0.00 59870 dp_sync_taskq taskqueue_enqueue+0x158 2 0% 98% 0.00 30564 bufq dirty lock (usermode) 2 0% 98% 0.00 19212 bge0 (usermode) 2 0% 98% 0.00 27285 CAM queue lock 0xc000000002511be8 1 0% 98% 0.00 26225 zfs_znode_cache 0xc0000000030dbab8 1 0% 98% 0.00 24824 vnode interlock vref+0x1e8 1 0% 98% 0.00 34790 vnode interlock 0xc0000000030c6708 1 0% 98% 0.00 25271 vnode_free_list getnewvnode_reserve+0x480 1 0% 99% 0.00 30776 vnode_free_list 0xc000000002e1adb8 1 0% 99% 0.00 1238987 vnode interlock 0xc000000002cd9248 1 0% 99% 0.00 139147 vm page free queue 0xc000000003109cc8 1 0% 99% 0.00 46098 vm pgcache uma_zalloc_arg+0x98c 1 0% 99% 0.00 28602 vm active pagequeue vm_page_dequeue+0x1b4 1 0% 99% 0.00 119084 vm page vm_object_madvise+0x54c 1 0% 99% 0.00 778698 vm page vm_page_sleep_if_busy+0x1c8 1 0% 99% 0.00 30756 struct mount mtx vfs_notify_upper+0x460 1 0% 99% 0.00 29173 pipe mutex (usermode) 1 0% 99% 0.00 37666 process lock ast+0x6e0 1 0% 99% 0.00 37125 process lock 0xc000000002c86d88 1 0% 99% 0.00 23062 process lock 0xc000000002cbb898 1 0% 99% 0.00 148034 select mtxpool selrecord+0x1dc 1 0% 99% 0.00 153033 select mtxpool (usermode) 1 0% 99% 0.00 13614 sleep mtxpool 0xc000000002cbb898 1 0% 99% 0.00 365409 UPVO entry uma_zfree_arg+0x5c8 1 0% 100% 0.00 216041 UPVO entry (usermode) 1 0% 100% 0.00 720478 UMA Slabs uma_large_malloc_domain+0x2ac 1 0% 100% 0.00 24784 FFS softdep_setup_freeblocks+0xc20 1 0% 100% 0.00 69387 256 Bucket uma_zalloc_arg+0x98c 1 0% 100% 0.00 2479661 256 Bucket 0xc0000000030dedd8 1 0% 100% 0.00 25996 dnode_t 0xc0000000030dbab8 1 0% 100% 0.00 51115 abd_chunk 0xc0000000030dbab8 1 0% 100% 0.00 115494 UPVO entry 0xc0000000030df7ac ------------------------------------------------------------------------------- Spin lock spin: 1440344 events in 177.342 seconds (8122 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 718866 50% 50% 0.00 255358 turnstile chain turnstile_trywait+0x208 398322 28% 78% 0.00 305019 turnstile chain (usermode) 156527 11% 88% 0.00 17774 sleepq chain sleepq_lock+0xe4 77487 5% 94% 0.00 24921 sleepq chain (usermode) 26404 2% 96% 0.00 50468 turnstile chain turnstile_chain_lock+0xdc 14770 1% 97% 0.00 692 turnstile lock turnstile_trywait+0x150 6763 0% 97% 0.00 443 turnstile lock (usermode) 1980 0% 97% 0.00 10085 sched lock 0 0xc000000002d353b4 929 0% 97% 0.00 12060 sched lock 10 0xc000000002d353b4 784 0% 97% 0.00 1027 turnstile lock turnstile_lookup+0xd4 748 0% 97% 0.00 12694 sched lock 5 0xc000000002d353b4 673 0% 97% 0.00 19086 sched lock 15 0xc000000002d353b4 542 0% 98% 0.00 29603 sched lock 20 0xc000000002d353b4 499 0% 98% 0.00 5317 sched lock 0 (usermode) 400 0% 98% 0.00 7579 sched lock 25 0xc000000002d353b4 398 0% 98% 0.00 8326 sched lock 30 0xc000000002d353b4 388 0% 98% 0.00 7280 sched lock 50 0xc000000002d353cc 339 0% 98% 0.00 7250 sched lock 65 0xc000000002d353cc 337 0% 98% 0.00 11785 sched lock 35 0xc000000002d353b4 335 0% 98% 0.00 518 callout 0xc000000002d1384c 325 0% 98% 0.00 5647 sched lock 30 0xc000000002d353cc 319 0% 98% 0.00 5129 sched lock 60 0xc000000002d353cc 319 0% 98% 0.00 9848 sched lock 35 0xc000000002d353cc 314 0% 98% 0.00 12699 sched lock 45 0xc000000002d353cc 308 0% 98% 0.00 7476 sched lock 55 0xc000000002d353cc 294 0% 98% 0.00 7542 sched lock 10 (usermode) 280 0% 98% 0.00 4056 sched lock 0 sched_add+0x48c 274 0% 98% 0.00 6213 sched lock 40 0xc000000002d353cc 262 0% 98% 0.00 4758 sched lock 40 0xc000000002d353b4 251 0% 98% 0.00 10511 sched lock 15 (usermode) 237 0% 98% 0.00 2390 sched lock 65 sched_add+0x48c 236 0% 98% 0.00 16489 sched lock 15 0xc000000002d353cc 217 0% 98% 0.00 17157 sched lock 5 (usermode) 216 0% 98% 0.00 363 td_contested turnstile_wait+0x298 214 0% 98% 0.00 2126 sched lock 50 sched_add+0x48c 206 0% 98% 0.00 7975 sched lock 50 0xc000000002d353b4 204 0% 98% 0.00 8340 sched lock 45 0xc000000002d353b4 202 0% 98% 0.00 6427 sched lock 30 (usermode) 201 0% 98% 0.00 6251 sched lock 25 0xc000000002d353cc 192 0% 98% 0.00 2154 sched lock 60 sched_add+0x48c 191 0% 98% 0.00 5634 sched lock 25 (usermode) 189 0% 98% 0.00 3055 sched lock 10 sched_add+0x48c 188 0% 98% 0.00 3294 sched lock 70 0xc000000002d353cc 188 0% 98% 0.00 17960 sched lock 35 (usermode) 187 0% 98% 0.00 1842 sched lock 55 sched_add+0x48c 184 0% 98% 0.00 1533 sched lock 67 sched_add+0x48c 181 0% 98% 0.00 22386 sched lock 20 0xc000000002d353cc 179 0% 98% 0.00 6265 sched lock 50 (usermode) 177 0% 98% 0.00 8148 sched lock 45 (usermode) 175 0% 98% 0.00 10560 sched lock 4 0xc000000002d353b4 171 0% 98% 0.00 7059 sched lock 55 (usermode) 169 0% 98% 0.00 6010 sched lock 9 0xc000000002d353b4 169 0% 98% 0.00 2799 sched lock 3 sched_add+0x48c 167 0% 98% 0.00 1836 sched lock 45 sched_add+0x48c 167 0% 98% 0.00 20893 sched lock 20 (usermode) 166 0% 98% 0.00 1140 sched lock 64 sched_add+0x48c 166 0% 98% 0.00 2639 sched lock 40 sched_add+0x48c 165 0% 98% 0.00 5958 sched lock 8 0xc000000002d353b4 165 0% 98% 0.00 1557 sched lock 54 sched_add+0x48c 164 0% 98% 0.00 8714 sched lock 14 0xc000000002d353b4 162 0% 98% 0.00 2304 sched lock 35 sched_add+0x48c 161 0% 98% 0.00 6061 sched lock 7 0xc000000002d353b4 161 0% 98% 0.00 5447 sched lock 3 0xc000000002d353b4 161 0% 98% 0.00 5998 sched lock 2 0xc000000002d353b4 161 0% 98% 0.00 6081 sched lock 15 sched_add+0x48c 157 0% 98% 0.00 1584 sched lock 56 sched_add+0x48c 156 0% 98% 0.00 1373 sched lock 53 sched_add+0x48c 155 0% 98% 0.00 5462 sched lock 65 (usermode) 154 0% 98% 0.00 1446 sched lock 63 sched_add+0x48c 154 0% 98% 0.00 8718 sched lock 17 0xc000000002d353b4 152 0% 98% 0.00 5748 sched lock 60 (usermode) 152 0% 98% 0.00 28019 sched lock 12 0xc000000002d353b4 150 0% 98% 0.00 1928 sched lock 44 sched_add+0x48c 150 0% 98% 0.00 10703 sched lock 40 (usermode) 148 0% 98% 0.00 1874 sched lock 70 sched_add+0x48c 147 0% 98% 0.00 5201 sched lock 60 0xc000000002d353b4 147 0% 98% 0.00 2467 sched lock 57 sched_add+0x48c 147 0% 99% 0.00 10515 sched lock 10 0xc000000002d353cc 146 0% 99% 0.00 1277 sched lock 66 sched_add+0x48c 146 0% 99% 0.00 6163 sched lock 6 0xc000000002d353b4 146 0% 99% 0.00 3921 sched lock 49 sched_add+0x48c 146 0% 99% 0.00 3574 sched lock 30 sched_add+0x48c 145 0% 99% 0.00 7300 sched lock 20 sched_add+0x48c 144 0% 99% 0.00 1666 sched lock 62 sched_add+0x48c 142 0% 99% 0.00 5741 sched lock 1 0xc000000002d353b4 141 0% 99% 0.00 1599 sched lock 69 sched_add+0x48c 141 0% 99% 0.00 1583 sched lock 46 sched_add+0x48c 141 0% 99% 0.00 2537 sched lock 25 sched_add+0x48c 141 0% 99% 0.00 5653 sched lock 13 0xc000000002d353b4 140 0% 99% 0.00 5048 sched lock 63 0xc000000002d353cc 140 0% 99% 0.00 1283 sched lock 52 sched_add+0x48c 139 0% 99% 0.00 986 sched lock 48 sched_add+0x48c 138 0% 99% 0.00 6626 sched lock 28 0xc000000002d353b4 138 0% 99% 0.00 5004 sched lock 16 0xc000000002d353b4 136 0% 99% 0.00 1461 sched lock 71 sched_add+0x48c 135 0% 99% 0.00 2777 sched lock 58 sched_add+0x48c 134 0% 99% 0.00 5485 sched lock 4 (usermode) 134 0% 99% 0.00 2823 sched lock 1 sched_add+0x48c 133 0% 99% 0.00 1496 sched lock 59 sched_add+0x48c 133 0% 99% 0.00 1634 sched lock 43 sched_add+0x48c 133 0% 99% 0.00 6917 sched lock 18 0xc000000002d353b4 132 0% 99% 0.00 6585 sched lock 19 0xc000000002d353b4 131 0% 99% 0.00 2556 sched lock 47 sched_add+0x48c 131 0% 99% 0.00 3984 sched lock 29 sched_add+0x48c 131 0% 99% 0.00 3212 sched lock 2 sched_add+0x48c 130 0% 99% 0.00 1492 sched lock 68 sched_add+0x48c 130 0% 99% 0.00 4194 sched lock 67 0xc000000002d353cc 130 0% 99% 0.00 6866 sched lock 19 (usermode) 129 0% 99% 0.00 6379 sched lock 59 0xc000000002d353cc 128 0% 99% 0.00 2855 sched lock 5 sched_add+0x48c 128 0% 99% 0.00 1170 sched lock 42 sched_add+0x48c 127 0% 99% 0.00 1216 sched lock 51 sched_add+0x48c 126 0% 99% 0.00 3002 sched lock 69 0xc000000002d353cc 123 0% 99% 0.00 1407 sched lock 28 sched_add+0x48c 122 0% 99% 0.00 1299 sched lock 61 sched_add+0x48c 122 0% 99% 0.00 3577 sched lock 43 0xc000000002d353cc 122 0% 99% 0.00 1724 sched lock 32 sched_add+0x48c 121 0% 99% 0.00 5990 sched lock 14 (usermode) 120 0% 99% 0.00 9833 sched lock 2 (usermode) 119 0% 99% 0.00 5169 sched lock 24 0xc000000002d353b4 119 0% 99% 0.00 5169 sched lock 11 0xc000000002d353b4 118 0% 99% 0.00 11421 sched lock 4 sched_add+0x48c 118 0% 99% 0.00 3517 sched lock 22 sched_add+0x48c 118 0% 99% 0.00 37481 sched lock 12 (usermode) 117 0% 99% 0.00 6569 sched lock 41 sched_add+0x48c 115 0% 99% 0.00 1760 zombie lock thread_stash+0x170 115 0% 99% 0.00 7285 sched lock 7 (usermode) 115 0% 99% 0.00 3978 sched lock 3 (usermode) 114 0% 99% 0.00 2728 sched lock 8 sched_add+0x48c 114 0% 99% 0.00 2444 sched lock 6 sched_add+0x48c 113 0% 99% 0.00 6042 sched lock 44 (usermode) 113 0% 99% 0.00 4819 sched lock 39 0xc000000002d353cc 113 0% 99% 0.00 5708 sched lock 32 (usermode) 113 0% 99% 0.00 5945 sched lock 22 0xc000000002d353b4 112 0% 99% 0.00 6489 sched lock 55 0xc000000002d353b4 112 0% 99% 0.00 9937 sched lock 32 0xc000000002d353b4 112 0% 99% 0.00 2263 sched lock 31 sched_add+0x48c 112 0% 99% 0.00 9772 sched lock 23 sched_add+0x48c 111 0% 99% 0.00 7600 sched lock 71 0xc000000002d353cc 111 0% 99% 0.00 3264 sched lock 53 0xc000000002d353cc 111 0% 99% 0.00 5471 sched lock 14 sched_add+0x48c 110 0% 99% 0.00 2654 sched lock 62 0xc000000002d353cc 110 0% 99% 0.00 4906 sched lock 34 (usermode) 110 0% 99% 0.00 2784 sched lock 19 sched_add+0x48c 110 0% 99% 0.00 4974 sched lock 11 (usermode) 108 0% 99% 0.00 2627 sched lock 64 0xc000000002d353cc 107 0% 99% 0.00 6309 sched lock 8 (usermode) 107 0% 99% 0.00 2160 sched lock 39 sched_add+0x48c 106 0% 99% 0.00 21025 sched lock 58 0xc000000002d353cc 106 0% 99% 0.00 3898 sched lock 54 0xc000000002d353cc 106 0% 99% 0.00 4808 sched lock 23 0xc000000002d353b4 106 0% 99% 0.00 3172 sched lock 11 sched_add+0x48c 105 0% 99% 0.00 2867 sched lock 68 0xc000000002d353cc 105 0% 99% 0.00 2447 sched lock 46 0xc000000002d353cc 105 0% 99% 0.00 1501 sched lock 34 sched_add+0x48c 104 0% 99% 0.00 2268 sched lock 7 sched_add+0x48c 104 0% 99% 0.00 9179 sched lock 21 0xc000000002d353b4 104 0% 99% 0.00 12446 sched lock 21 sched_add+0x48c 103 0% 99% 0.00 2642 sched lock 47 0xc000000002d353cc 103 0% 99% 0.00 1543 sched lock 38 sched_add+0x48c 103 0% 99% 0.00 1972 sched lock 27 sched_add+0x48c 101 0% 99% 0.00 4002 sched lock 48 (usermode) 101 0% 99% 0.00 6032 sched lock 27 0xc000000002d353b4 100 0% 99% 0.00 3296 sched lock 61 0xc000000002d353cc 100 0% 99% 0.00 3286 sched lock 52 0xc000000002d353cc 100 0% 99% 0.00 5227 sched lock 29 0xc000000002d353b4 100 0% 99% 0.00 6788 sched lock 18 (usermode) 99 0% 99% 0.00 5765 sched lock 6 (usermode) 99 0% 99% 0.00 3329 sched lock 33 sched_add+0x48c 99 0% 99% 0.00 4997 sched lock 27 (usermode) 99 0% 99% 0.00 5643 sched lock 1 (usermode) 98 0% 99% 0.00 3068 sched lock 51 0xc000000002d353cc 98 0% 99% 0.00 2593 sched lock 24 sched_add+0x48c 98 0% 99% 0.00 5142 sched lock 23 (usermode) 98 0% 99% 0.00 5485 sched lock 21 (usermode) 98 0% 99% 0.00 2475 sched lock 17 sched_add+0x48c 98 0% 99% 0.00 7294 sched lock 17 (usermode) 96 0% 99% 0.00 2351 sched lock 66 0xc000000002d353cc 96 0% 99% 0.00 4489 sched lock 44 0xc000000002d353b4 96 0% 99% 0.00 1612 sched lock 37 sched_add+0x48c 96 0% 99% 0.00 2665 sched lock 13 sched_add+0x48c 95 0% 99% 0.00 5165 sched lock 44 0xc000000002d353cc 95 0% 99% 0.00 7468 sched lock 29 0xc000000002d353cc 94 0% 99% 0.00 6663 sched lock 9 (usermode) 94 0% 99% 0.00 3317 sched lock 70 (usermode) 93 0% 99% 0.00 11851 sched lock 9 sched_add+0x48c 93 0% 99% 0.00 2981 sched lock 56 0xc000000002d353cc 93 0% 99% 0.00 1058 sched lock 36 sched_add+0x48c 93 0% 99% 0.00 8126 sched lock 12 sched_add+0x48c 92 0% 99% 0.00 3427 sched lock 46 (usermode) 92 0% 99% 0.00 6659 sched lock 22 (usermode) 91 0% 99% 0.00 8771 sched lock 34 0xc000000002d353b4 90 0% 99% 0.00 4951 sched lock 28 (usermode) 90 0% 99% 0.00 3839 sched lock 16 sched_add+0x48c 89 0% 99% 0.00 2490 sched lock 52 (usermode) 89 0% 99% 0.00 3848 sched lock 37 (usermode) 89 0% 99% 0.00 4206 sched lock 34 0xc000000002d353cc 89 0% 99% 0.00 4558 sched lock 31 0xc000000002d353b4 89 0% 99% 0.00 1395 sched lock 26 sched_add+0x48c 89 0% 99% 0.00 9535 sched lock 16 (usermode) 89 0% 99% 0.00 555 callout (usermode) 88 0% 99% 0.00 4548 sched lock 26 (usermode) 86 0% 99% 0.00 6270 sched lock 31 0xc000000002d353cc 86 0% 99% 0.00 6754 sched lock 29 (usermode) 85 0% 99% 0.00 3543 sched lock 59 (usermode) 85 0% 100% 0.00 4905 sched lock 43 (usermode) 85 0% 100% 0.00 2944 sched lock 42 0xc000000002d353cc 85 0% 100% 0.00 3615 sched lock 42 (usermode) 84 0% 100% 0.00 2844 sched lock 57 0xc000000002d353cc 84 0% 100% 0.00 7091 sched lock 32 0xc000000002d353cc 83 0% 100% 0.00 6225 sched lock 31 (usermode) 82 0% 100% 0.00 98132 sched lock 49 0xc000000002d353cc 82 0% 100% 0.00 3280 sched lock 47 (usermode) 82 0% 100% 0.00 4571 sched lock 41 0xc000000002d353cc 82 0% 100% 0.00 5334 sched lock 18 sched_add+0x48c 80 0% 100% 0.00 5802 sched lock 53 (usermode) 80 0% 100% 0.00 43149 sched lock 49 (usermode) 80 0% 100% 0.00 3078 sched lock 48 0xc000000002d353cc 80 0% 100% 0.00 4172 sched lock 41 (usermode) 80 0% 100% 0.00 4305 sched lock 39 0xc000000002d353b4 79 0% 100% 0.00 4433 sched lock 63 (usermode) 79 0% 100% 0.00 2439 sched lock 57 (usermode) 79 0% 100% 0.00 9633 sched lock 36 0xc000000002d353b4 79 0% 100% 0.00 4268 sched lock 24 (usermode) 77 0% 100% 0.00 2325 sched lock 39 (usermode) 77 0% 100% 0.00 6543 sched lock 38 0xc000000002d353cc 77 0% 100% 0.00 4793 sched lock 19 0xc000000002d353cc 76 0% 100% 0.00 6250 sched lock 26 0xc000000002d353b4 75 0% 100% 0.00 2583 sched lock 62 (usermode) 75 0% 100% 0.00 3274 sched lock 54 (usermode) 74 0% 100% 0.00 3226 sched lock 69 (usermode) 74 0% 100% 0.00 3503 sched lock 66 (usermode) 73 0% 100% 0.00 4564 sched lock 56 (usermode) 72 0% 100% 0.00 8481 sched lock 71 (usermode) 72 0% 100% 0.00 5131 sched lock 23 0xc000000002d353cc 71 0% 100% 0.00 2429 td_contested turnstile_unpend+0x428 71 0% 100% 0.00 2619 sched lock 61 (usermode) 71 0% 100% 0.00 5560 sched lock 27 0xc000000002d353cc 71 0% 100% 0.00 7223 sched lock 22 0xc000000002d353cc 70 0% 100% 0.00 2779 sched lock 64 (usermode) 69 0% 100% 0.00 5337 sched lock 26 0xc000000002d353cc 69 0% 100% 0.00 2160 sched lock 0 sched_switch+0x81c 68 0% 100% 0.00 4045 sched lock 33 0xc000000002d353b4 67 0% 100% 0.00 2289 sched lock 68 (usermode) 67 0% 100% 0.00 5551 sched lock 51 (usermode) 67 0% 100% 0.00 2972 sched lock 41 0xc000000002d353b4 67 0% 100% 0.00 11314 sched lock 37 0xc000000002d353cc 67 0% 100% 0.00 9886 sched lock 36 (usermode) 67 0% 100% 0.00 4637 sched lock 13 (usermode) 66 0% 100% 0.00 4274 sched lock 21 0xc000000002d353cc 65 0% 100% 0.00 7897 sched lock 65 0xc000000002d353b4 65 0% 100% 0.00 7923 sched lock 14 0xc000000002d353cc 64 0% 100% 0.00 4692 sched lock 65 sched_switch+0x81c 64 0% 100% 0.00 17244 sched lock 5 0xc000000002d353cc 64 0% 100% 0.00 13114 sched lock 37 0xc000000002d353b4 64 0% 100% 0.00 2844 sched lock 33 0xc000000002d353cc 63 0% 100% 0.00 3295 sched lock 48 0xc000000002d353b4 63 0% 100% 0.00 3160 sched lock 46 0xc000000002d353b4 63 0% 100% 0.00 4842 sched lock 20 sched_switch+0x81c 62 0% 100% 0.00 1385 zombie lock (usermode) 62 0% 100% 0.00 2425 sched lock 67 (usermode) 62 0% 100% 0.00 5439 sched lock 38 (usermode) 62 0% 100% 0.00 4961 sched lock 33 (usermode) 61 0% 100% 0.00 4854 sched lock 28 0xc000000002d353cc 59 0% 100% 0.00 2674 sched lock 58 (usermode) 59 0% 100% 0.00 3541 sched lock 42 0xc000000002d353b4 57 0% 100% 0.00 545 td_contested turnstile_broadcast+0x1bc 56 0% 100% 0.00 7271 sched lock 53 0xc000000002d353b4 55 0% 100% 0.00 6761 sched lock 17 0xc000000002d353cc 54 0% 100% 0.00 6288 sched lock 24 0xc000000002d353cc 53 0% 100% 0.00 51974 sched lock 49 0xc000000002d353b4 53 0% 100% 0.00 9484 sched lock 36 0xc000000002d353cc 52 0% 100% 0.00 5106 sched lock 0 sched_switch+0xca0 49 0% 100% 0.00 4943 sched lock 38 0xc000000002d353b4 47 0% 100% 0.00 7216 sched lock 16 0xc000000002d353cc 47 0% 100% 0.00 5771 sched lock 15 sched_switch+0x81c 46 0% 100% 0.00 4535 sched lock 40 sched_switch+0x81c 46 0% 100% 0.00 7288 sched lock 18 0xc000000002d353cc 46 0% 100% 0.00 267 callout 0xc000000002d14554 45 0% 100% 0.00 1718 td_contested (usermode) 45 0% 100% 0.00 5480 sched lock 43 0xc000000002d353b4 44 0% 100% 0.00 4890 sched lock 50 sched_switch+0x81c 44 0% 100% 0.00 6055 sched lock 11 0xc000000002d353cc 43 0% 100% 0.00 5298 sched lock 70 sched_switch+0x81c 43 0% 100% 0.00 4390 sched lock 30 sched_switch+0x81c 43 0% 100% 0.00 4312 sched lock 10 sched_switch+0x81c 42 0% 100% 0.00 8313 sched lock 7 0xc000000002d353cc 42 0% 100% 0.00 2877 sched lock 47 0xc000000002d353b4 40 0% 100% 0.00 3485 sched lock 5 sched_switch+0x81c 39 0% 100% 0.00 4971 sched lock 45 sched_switch+0x81c 39 0% 100% 0.00 4625 sched lock 25 sched_switch+0x81c 38 0% 100% 0.00 4505 sched lock 55 sched_switch+0x81c 38 0% 100% 0.00 2843 sched lock 52 0xc000000002d353b4 37 0% 100% 0.00 5182 sched lock 60 sched_switch+0x81c 37 0% 100% 0.00 4199 sched lock 13 0xc000000002d353cc 36 0% 100% 0.00 3925 sched lock 51 0xc000000002d353b4 35 0% 100% 0.00 5259 sched lock 9 0xc000000002d353cc 35 0% 100% 0.00 4140 sched lock 56 0xc000000002d353b4 34 0% 100% 0.00 6787 sched lock 54 0xc000000002d353b4 34 0% 100% 0.00 4652 sched lock 3 0xc000000002d353cc 32 0% 100% 0.00 9644 sched lock 45 sched_switch+0xca0 29 0% 100% 0.00 3793 sched lock 64 0xc000000002d353b4 29 0% 100% 0.00 7588 sched lock 59 0xc000000002d353b4 29 0% 100% 0.00 5808 sched lock 57 0xc000000002d353b4 29 0% 100% 0.00 3715 callout 0xc000000002d12b60 28 0% 100% 0.00 234 td_contested turnstile_wait+0x584 27 0% 100% 0.00 4880 sched lock 35 sched_switch+0x81c 26 0% 100% 0.00 12006 sched lock 60 sched_switch+0xca0 26 0% 100% 0.00 2334 sched lock 58 0xc000000002d353b4 25 0% 100% 0.00 11146 sched lock 30 sched_switch+0xca0 23 0% 100% 0.00 9549 sched lock 65 sched_switch+0xca0 23 0% 100% 0.00 6451 sched lock 6 0xc000000002d353cc 22 0% 100% 0.00 10153 sched lock 8 0xc000000002d353cc 22 0% 100% 0.00 10046 sched lock 50 sched_switch+0xca0 21 0% 100% 0.00 32315 sched lock 12 0xc000000002d353cc 20 0% 100% 0.00 10352 process slock kern_thr_exit+0x304 19 0% 100% 0.00 3489 sched lock 61 0xc000000002d353b4 19 0% 100% 0.00 6459 sched lock 10 sched_switch+0xca0 18 0% 100% 0.00 2788 sched lock 62 0xc000000002d353b4 18 0% 100% 0.00 5924 sched lock 55 sched_switch+0xca0 18 0% 100% 0.00 4728 sched lock 2 0xc000000002d353cc 17 0% 100% 0.00 11891 sched lock 35 sched_switch+0xca0 16 0% 100% 0.00 3030 sched lock 70 0xc000000002d353b4 16 0% 100% 0.00 5203 sched lock 15 sched_switch+0xca0 15 0% 100% 0.00 2270 sched lock 68 0xc000000002d353b4 15 0% 100% 0.00 8355 sched lock 5 sched_switch+0xca0 14 0% 100% 0.00 7179 sched lock 40 sched_switch+0xca0 14 0% 100% 0.00 3441 sched lock 1 0xc000000002d353cc 13 0% 100% 0.00 1814 sched lock 69 0xc000000002d353b4 13 0% 100% 0.00 1863 sched lock 63 0xc000000002d353b4 11 0% 100% 0.00 5225 sched lock 4 0xc000000002d353cc 11 0% 100% 0.00 236 et_hw_mtx cpu_new_callout+0x370 10 0% 100% 0.00 2449 sched lock 66 0xc000000002d353b4 10 0% 100% 0.00 4030 sched lock 40 0xc000000002d3539c 10 0% 100% 0.00 994 sched lock 3 0xc000000002d3539c 9 0% 100% 0.00 1331 sched lock 8 sched_switch+0x81c 9 0% 100% 0.00 394 sched lock 6 sched_switch+0x81c 9 0% 100% 0.00 6183 sched lock 54 sched_switch+0xca0 9 0% 100% 0.00 10826 sched lock 20 sched_switch+0xca0 9 0% 100% 0.00 1393 sched lock 1 0xc000000002d3539c 9 0% 100% 0.00 3175 sched lock 0 0xc000000002d3539c 8 0% 100% 0.00 533 zombie lock thread_reap+0x218 8 0% 100% 0.00 1263 sched lock 9 sched_switch+0x81c 8 0% 100% 0.00 5391 sched lock 66 sched_switch+0xca0 8 0% 100% 0.00 3179 sched lock 48 sched_switch+0xca0 8 0% 100% 0.00 5120 sched lock 45 0xc000000002d3539c 8 0% 100% 0.00 835 sched lock 31 sched_switch+0x81c 8 0% 100% 0.00 2508 sched lock 25 0xc000000002d3539c 8 0% 100% 0.00 65215 sched lock 25 sched_switch+0xca0 8 0% 100% 0.00 4018 sched lock 10 0xc000000002d3539c 7 0% 100% 0.00 7718 sched lock 70 sched_switch+0xca0 7 0% 100% 0.00 1696 sched lock 61 sched_switch+0x81c 7 0% 100% 0.00 867 sched lock 4 sched_switch+0x81c 7 0% 100% 0.00 4253 sched lock 32 sched_switch+0xca0 7 0% 100% 0.00 2138 sched lock 26 sched_switch+0x81c 7 0% 100% 0.00 1333 sched lock 12 0xc000000002d3539c 7 0% 100% 0.00 2534 sched lock 1 sched_switch+0x81c 7 0% 100% 0.00 696 callout softclock+0x154 6 0% 100% 0.00 895 sched lock 8 0xc000000002d3539c 6 0% 100% 0.00 781 sched lock 67 0xc000000002d353b4 6 0% 100% 0.00 4588 sched lock 61 sched_switch+0xca0 6 0% 100% 0.00 8747 sched lock 59 sched_switch+0xca0 6 0% 100% 0.00 4053 sched lock 57 sched_switch+0xca0 6 0% 100% 0.00 1637 sched lock 5 0xc000000002d3539c 6 0% 100% 0.00 687 sched lock 47 sched_switch+0x81c 6 0% 100% 0.00 4069 sched lock 42 sched_switch+0xca0 6 0% 100% 0.00 3503 sched lock 4 0xc000000002d3539c 6 0% 100% 0.00 4525 sched lock 35 0xc000000002d3539c 6 0% 100% 0.00 573 sched lock 27 sched_switch+0x81c 6 0% 100% 0.00 1712 sched lock 20 0xc000000002d3539c 6 0% 100% 0.00 528 sched lock 19 sched_switch+0x81c 6 0% 100% 0.00 391 sched lock 13 0xc000000002d3539c 5 0% 100% 0.00 44140 td_contested turnstile_claim+0x290 5 0% 100% 0.00 3037 sched lock 63 sched_switch+0xca0 5 0% 100% 0.00 301 sched lock 51 sched_switch+0x81c 5 0% 100% 0.00 1379 sched lock 48 sched_switch+0x81c 5 0% 100% 0.00 348 sched lock 47 0xc000000002d3539c 5 0% 100% 0.00 4853 sched lock 44 sched_switch+0xca0 5 0% 100% 0.00 547 sched lock 44 sched_switch+0x81c 5 0% 100% 0.00 2706 sched lock 42 sched_switch+0x81c 5 0% 100% 0.00 1093 sched lock 38 sched_switch+0x81c 5 0% 100% 0.00 1252 sched lock 32 sched_switch+0x81c 5 0% 100% 0.00 983 sched lock 3 sched_switch+0x81c 5 0% 100% 0.00 1595 sched lock 23 sched_switch+0x81c 5 0% 100% 0.00 765 sched lock 22 sched_switch+0x81c 5 0% 100% 0.00 432 sched lock 21 0xc000000002d3539c 5 0% 100% 0.00 1928 sched lock 21 sched_switch+0x81c 5 0% 100% 0.00 4137 sched lock 18 sched_switch+0xca0 5 0% 100% 0.00 757 sched lock 18 sched_switch+0x81c 5 0% 100% 0.00 287 sched lock 17 sched_switch+0x81c 5 0% 100% 0.00 1232 sched lock 13 sched_switch+0x81c 5 0% 100% 0.00 692 et_hw_mtx (usermode) 4 0% 100% 0.00 1064 sched lock 9 0xc000000002d3539c 4 0% 100% 0.00 5784 sched lock 69 sched_switch+0xca0 4 0% 100% 0.00 8807 sched lock 67 sched_switch+0xca0 4 0% 100% 0.00 4298 sched lock 6 sched_switch+0xca0 4 0% 100% 0.00 622 sched lock 59 sched_switch+0x81c 4 0% 100% 0.00 1900 sched lock 58 sched_switch+0x81c 4 0% 100% 0.00 1039 sched lock 53 sched_switch+0x81c 4 0% 100% 0.00 3477 sched lock 53 sched_switch+0xca0 4 0% 100% 0.00 1917 sched lock 52 0xc000000002d3539c 4 0% 100% 0.00 3554 sched lock 49 sched_switch+0xca0 4 0% 100% 0.00 5524 sched lock 47 sched_switch+0xca0 4 0% 100% 0.00 343 sched lock 42 0xc000000002d3539c 4 0% 100% 0.00 2655 sched lock 41 sched_switch+0x81c 4 0% 100% 0.00 1886 sched lock 32 0xc000000002d3539c 4 0% 100% 0.00 548 sched lock 30 0xc000000002d3539c 4 0% 100% 0.00 432 sched lock 29 sched_switch+0x81c 4 0% 100% 0.00 3983 sched lock 24 sched_switch+0xca0 4 0% 100% 0.00 771 sched lock 2 0xc000000002d3539c 4 0% 100% 0.00 5782 sched lock 19 sched_switch+0xca0 4 0% 100% 0.00 1754 sched lock 17 0xc000000002d3539c 4 0% 100% 0.00 1469 sched lock 16 sched_switch+0x81c 4 0% 100% 0.00 1225 sched lock 14 sched_switch+0x81c 4 0% 100% 0.00 5155 sched lock 13 sched_switch+0xca0 4 0% 100% 0.00 2276 sched lock 12 sched_switch+0x81c 4 0% 100% 0.00 309 sched lock 11 0xc000000002d3539c 4 0% 100% 0.00 1027 sched lock 11 sched_switch+0x81c 3 0% 100% 0.00 2249 sched lock 7 sched_switch+0x81c 3 0% 100% 0.00 9584 sched lock 68 sched_switch+0xca0 3 0% 100% 0.00 11090 sched lock 64 sched_switch+0xca0 3 0% 100% 0.00 4003 sched lock 62 sched_switch+0xca0 3 0% 100% 0.00 5069 sched lock 56 sched_switch+0xca0 3 0% 100% 0.00 384 sched lock 56 sched_switch+0x81c 3 0% 100% 0.00 9984 sched lock 50 0xc000000002d3539c 3 0% 100% 0.00 276 sched lock 49 sched_switch+0x81c 3 0% 100% 0.00 3489 sched lock 48 0xc000000002d3539c 3 0% 100% 0.00 2233 sched lock 46 sched_switch+0xca0 3 0% 100% 0.00 5639 sched lock 43 sched_switch+0xca0 3 0% 100% 0.00 1966 sched lock 43 sched_switch+0x81c 3 0% 100% 0.00 1289 sched lock 41 0xc000000002d3539c 3 0% 100% 0.00 4898 sched lock 4 sched_switch+0xca0 3 0% 100% 0.00 3303 sched lock 39 sched_switch+0xca0 3 0% 100% 0.00 4692 sched lock 38 sched_switch+0xca0 3 0% 100% 0.00 397 sched lock 34 sched_switch+0x81c 3 0% 100% 0.00 442 sched lock 31 0xc000000002d3539c 3 0% 100% 0.00 5066 sched lock 3 sched_switch+0xca0 3 0% 100% 0.00 2801 sched lock 27 sched_switch+0xca0 3 0% 100% 0.00 2620 sched lock 24 0xc000000002d3539c 3 0% 100% 0.00 5842 sched lock 23 sched_switch+0xca0 3 0% 100% 0.00 475 sched lock 22 0xc000000002d3539c 3 0% 100% 0.00 1539 sched lock 2 sched_switch+0x81c 3 0% 100% 0.00 2617 sched lock 18 0xc000000002d3539c 3 0% 100% 0.00 439 sched lock 14 0xc000000002d3539c 2 0% 100% 0.00 9240 sched lock 71 sched_switch+0xca0 2 0% 100% 0.00 3317 sched lock 7 sched_switch+0xca0 2 0% 100% 0.00 376 sched lock 7 0xc000000002d3539c 2 0% 100% 0.00 810 sched lock 64 sched_switch+0x81c 2 0% 100% 0.00 782 sched lock 63 sched_switch+0x81c 2 0% 100% 0.00 3510 sched lock 58 sched_switch+0xca0 2 0% 100% 0.00 1154 sched lock 57 sched_switch+0x81c 2 0% 100% 0.00 411 sched lock 54 sched_switch+0x81c 2 0% 100% 0.00 4123 sched lock 51 sched_switch+0xca0 2 0% 100% 0.00 243 sched lock 49 0xc000000002d3539c 2 0% 100% 0.00 382 sched lock 46 0xc000000002d3539c 2 0% 100% 0.00 1914 sched lock 44 0xc000000002d3539c 2 0% 100% 0.00 223 sched lock 38 0xc000000002d3539c 2 0% 100% 0.00 5394 sched lock 37 sched_switch+0xca0 2 0% 100% 0.00 679 sched lock 37 sched_switch+0x81c 2 0% 100% 0.00 1812 sched lock 36 sched_switch+0xca0 2 0% 100% 0.00 809 sched lock 36 sched_switch+0x81c 2 0% 100% 0.00 958 sched lock 34 sched_switch+0xca0 2 0% 100% 0.00 227 sched lock 33 sched_switch+0x81c 2 0% 100% 0.00 444 sched lock 28 0xc000000002d3539c 2 0% 100% 0.00 481 sched lock 26 0xc000000002d3539c 2 0% 100% 0.00 7277 sched lock 17 sched_switch+0xca0 2 0% 100% 0.00 12675 sched lock 15 0xc000000002d3539c 2 0% 100% 0.00 4046 sched lock 14 sched_switch+0xca0 2 0% 100% 0.00 285 sched lock 11 sched_switch+0xca0 2 0% 100% 0.00 200 pstatl (usermode) 2 0% 100% 0.00 1130 pstatl thread_exit+0x3dc 2 0% 100% 0.00 4182 sched lock 1 sched_switch+0xca0 1 0% 100% 0.00 330546 smp rendezvous smp_rendezvous_cpus+0x33c 1 0% 100% 0.00 183 sched lock 71 sched_switch+0x81c 1 0% 100% 0.00 1166 sched lock 8 sched_switch+0xca0 1 0% 100% 0.00 227 sched lock 62 0xc000000002d3539c 1 0% 100% 0.00 3859 sched lock 65 0xc000000002d3539c 1 0% 100% 0.00 1761 sched lock 66 sched_switch+0x81c 1 0% 100% 0.00 179 sched lock 62 sched_switch+0x81c 1 0% 100% 0.00 273 sched lock 60 0xc000000002d3539c 1 0% 100% 0.00 291 sched lock 6 0xc000000002d3539c 1 0% 100% 0.00 271 sched lock 56 0xc000000002d3539c 1 0% 100% 0.00 6480 sched lock 55 0xc000000002d3539c 1 0% 100% 0.00 410 sched lock 58 0xc000000002d3539c 1 0% 100% 0.00 142 sched lock 52 sched_switch+0xca0 1 0% 100% 0.00 1597 sched lock 51 0xc000000002d3539c 1 0% 100% 0.00 399 sched lock 54 0xc000000002d3539c 1 0% 100% 0.00 824 sched lock 46 sched_switch+0x81c 1 0% 100% 0.00 6874 sched lock 41 sched_switch+0xca0 1 0% 100% 0.00 541 sched lock 36 0xc000000002d3539c 1 0% 100% 0.00 3717 sched lock 34 0xc000000002d3539c 1 0% 100% 0.00 208 sched lock 39 sched_switch+0x81c 1 0% 100% 0.00 364 sched lock 37 0xc000000002d3539c 1 0% 100% 0.00 303 sched lock 39 0xc000000002d3539c 1 0% 100% 0.00 506 sched lock 23 0xc000000002d3539c 1 0% 100% 0.00 260 sched lock 24 sched_switch+0x81c 1 0% 100% 0.00 6032 sched lock 2 sched_switch+0xca0 1 0% 100% 0.00 514 sched lock 21 sched_switch+0xca0 1 0% 100% 0.00 1138 sched lock 29 sched_switch+0xca0 1 0% 100% 0.00 428 sched lock 27 0xc000000002d3539c 1 0% 100% 0.00 1706 sched lock 28 sched_switch+0x81c 1 0% 100% 0.00 11364 sched lock 31 sched_switch+0xca0 1 0% 100% 0.00 6411 sched lock 33 sched_switch+0xca0 1 0% 100% 0.00 429 sched lock 33 0xc000000002d3539c 1 0% 100% 0.00 357 et_hw_mtx cpu_idleclock+0x37c 1 0% 100% 0.00 15051 process slock (usermode) 1 0% 100% 0.00 1648 sched lock 16 sched_switch+0xca0 1 0% 100% 0.00 272 sched lock 19 0xc000000002d3539c ------------------------------------------------------------------------------- Thread lock spin: 162028 events in 177.342 seconds (914 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 92271 57% 57% 0.00 4833 sleepq chain _thread_lock+0xf4 34065 21% 78% 0.00 4896 sleepq chain (usermode) 3969 2% 80% 0.00 4931 turnstile lock _thread_lock+0xf4 937 1% 81% 0.00 4944 sched lock 0 _thread_lock+0xf4 775 0% 81% 0.00 52536 sched lock 0 (usermode) 607 0% 82% 0.00 3651 sched lock 65 _thread_lock+0xf4 549 0% 82% 0.00 6100 sched lock 15 _thread_lock+0xf4 546 0% 83% 0.00 6130 sched lock 10 _thread_lock+0xf4 432 0% 83% 0.00 4594 sched lock 5 _thread_lock+0xf4 430 0% 83% 0.00 4512 sched lock 35 _thread_lock+0xf4 428 0% 83% 0.00 5254 sched lock 20 _thread_lock+0xf4 414 0% 84% 0.00 4201 sched lock 50 _thread_lock+0xf4 405 0% 84% 0.00 10778 sched lock 25 _thread_lock+0xf4 403 0% 84% 0.00 6470 sched lock 30 _thread_lock+0xf4 401 0% 84% 0.00 6792 sched lock 60 _thread_lock+0xf4 393 0% 85% 0.00 4784 sched lock 45 _thread_lock+0xf4 388 0% 85% 0.00 5480 sched lock 40 _thread_lock+0xf4 384 0% 85% 0.00 5807 sched lock 55 _thread_lock+0xf4 307 0% 85% 0.00 3339 sched lock 11 _thread_lock+0xf4 292 0% 85% 0.00 4651 turnstile lock (usermode) 291 0% 86% 0.00 3539 sched lock 70 _thread_lock+0xf4 269 0% 86% 0.00 4949 sched lock 3 _thread_lock+0xf4 268 0% 86% 0.00 2742 sched lock 64 _thread_lock+0xf4 268 0% 86% 0.00 4059 sched lock 56 _thread_lock+0xf4 262 0% 86% 0.00 4772 sched lock 43 _thread_lock+0xf4 261 0% 86% 0.00 8038 sched lock 5 (usermode) 261 0% 87% 0.00 13604 sched lock 10 (usermode) 259 0% 87% 0.00 6005 sched lock 25 (usermode) 254 0% 87% 0.00 5020 sched lock 30 (usermode) 253 0% 87% 0.00 4383 sched lock 20 (usermode) 251 0% 87% 0.00 4120 sched lock 60 (usermode) 251 0% 87% 0.00 3652 sched lock 34 _thread_lock+0xf4 249 0% 88% 0.00 3857 sched lock 2 _thread_lock+0xf4 248 0% 88% 0.00 4733 sched lock 50 (usermode) 246 0% 88% 0.00 3365 sched lock 7 _thread_lock+0xf4 245 0% 88% 0.00 4291 sched lock 1 _thread_lock+0xf4 242 0% 88% 0.00 8058 sched lock 14 _thread_lock+0xf4 241 0% 88% 0.00 4327 sched lock 32 _thread_lock+0xf4 239 0% 88% 0.00 4953 sched lock 67 _thread_lock+0xf4 234 0% 89% 0.00 2723 sched lock 54 _thread_lock+0xf4 232 0% 89% 0.00 3079 sched lock 53 _thread_lock+0xf4 228 0% 89% 0.00 5319 sched lock 35 (usermode) 226 0% 89% 0.00 3354 sched lock 55 (usermode) 225 0% 89% 0.00 3720 sched lock 45 (usermode) 222 0% 89% 0.00 5674 sched lock 40 (usermode) 220 0% 89% 0.00 3134 sched lock 71 _thread_lock+0xf4 219 0% 90% 0.00 3994 sched lock 6 _thread_lock+0xf4 219 0% 90% 0.00 3304 sched lock 47 _thread_lock+0xf4 219 0% 90% 0.00 3938 sched lock 44 _thread_lock+0xf4 219 0% 90% 0.00 3474 sched lock 21 _thread_lock+0xf4 218 0% 90% 0.00 4503 sched lock 9 _thread_lock+0xf4 216 0% 90% 0.00 2685 sched lock 62 _thread_lock+0xf4 216 0% 90% 0.00 4294 sched lock 51 _thread_lock+0xf4 213 0% 90% 0.00 3577 sched lock 63 _thread_lock+0xf4 212 0% 91% 0.00 3187 sched lock 38 _thread_lock+0xf4 211 0% 91% 0.00 4067 sched lock 13 _thread_lock+0xf4 210 0% 91% 0.00 2969 sched lock 69 _thread_lock+0xf4 210 0% 91% 0.00 4596 sched lock 46 _thread_lock+0xf4 210 0% 91% 0.00 3752 sched lock 27 _thread_lock+0xf4 209 0% 91% 0.00 6847 sched lock 4 _thread_lock+0xf4 207 0% 91% 0.00 2799 sched lock 59 _thread_lock+0xf4 205 0% 91% 0.00 3800 sched lock 8 _thread_lock+0xf4 204 0% 92% 0.00 3219 sched lock 39 _thread_lock+0xf4 203 0% 92% 0.00 5553 sched lock 58 _thread_lock+0xf4 203 0% 92% 0.00 6129 sched lock 52 _thread_lock+0xf4 201 0% 92% 0.00 6173 sched lock 23 _thread_lock+0xf4 201 0% 92% 0.00 12316 sched lock 19 _thread_lock+0xf4 200 0% 92% 0.00 3914 sched lock 29 _thread_lock+0xf4 199 0% 92% 0.00 3036 sched lock 49 _thread_lock+0xf4 199 0% 92% 0.00 4963 sched lock 28 _thread_lock+0xf4 197 0% 93% 0.00 2904 sched lock 70 (usermode) 195 0% 93% 0.00 10055 sched lock 67 (usermode) 194 0% 93% 0.00 2800 sched lock 48 _thread_lock+0xf4 193 0% 93% 0.00 3830 sched lock 41 _thread_lock+0xf4 192 0% 93% 0.00 2480 sched lock 66 _thread_lock+0xf4 192 0% 93% 0.00 5004 sched lock 42 _thread_lock+0xf4 191 0% 93% 0.00 2707 sched lock 48 (usermode) 190 0% 93% 0.00 125401 sched lock 57 _thread_lock+0xf4 189 0% 94% 0.00 3554 sched lock 22 _thread_lock+0xf4 187 0% 94% 0.00 5476 sched lock 37 _thread_lock+0xf4 186 0% 94% 0.00 3576 sched lock 53 (usermode) 186 0% 94% 0.00 7600 sched lock 12 _thread_lock+0xf4 185 0% 94% 0.00 4878 sched lock 16 _thread_lock+0xf4 184 0% 94% 0.00 2505 sched lock 61 _thread_lock+0xf4 184 0% 94% 0.00 3138 sched lock 26 _thread_lock+0xf4 182 0% 94% 0.00 5314 sched lock 71 (usermode) 181 0% 94% 0.00 4038 sched lock 17 _thread_lock+0xf4 178 0% 95% 0.00 2909 sched lock 68 _thread_lock+0xf4 177 0% 95% 0.00 3415 sched lock 66 (usermode) 176 0% 95% 0.00 3615 sched lock 33 _thread_lock+0xf4 176 0% 95% 0.00 4257 sched lock 31 _thread_lock+0xf4 173 0% 95% 0.00 3859 sched lock 28 (usermode) 172 0% 95% 0.00 3348 sched lock 54 (usermode) 172 0% 95% 0.00 4061 sched lock 32 (usermode) 167 0% 95% 0.00 4452 sched lock 19 (usermode) 166 0% 95% 0.00 2813 sched lock 41 (usermode) 166 0% 96% 0.00 2835 sched lock 36 _thread_lock+0xf4 166 0% 96% 0.00 3806 sched lock 23 (usermode) 165 0% 96% 0.00 2524 sched lock 49 (usermode) 164 0% 96% 0.00 3864 sched lock 63 (usermode) 164 0% 96% 0.00 3132 sched lock 46 (usermode) 163 0% 96% 0.00 3198 sched lock 52 (usermode) 163 0% 96% 0.00 3154 sched lock 42 (usermode) 163 0% 96% 0.00 6749 sched lock 15 (usermode) 162 0% 96% 0.00 4544 sched lock 62 (usermode) 162 0% 96% 0.00 2866 sched lock 44 (usermode) 162 0% 97% 0.00 5583 sched lock 29 (usermode) 160 0% 97% 0.00 2710 sched lock 69 (usermode) 159 0% 97% 0.00 3379 sched lock 61 (usermode) 157 0% 97% 0.00 5090 sched lock 4 (usermode) 157 0% 97% 0.00 4165 sched lock 24 _thread_lock+0xf4 156 0% 97% 0.00 3541 sched lock 34 (usermode) 153 0% 97% 0.00 12475 sched lock 68 (usermode) 153 0% 97% 0.00 3345 sched lock 65 (usermode) 153 0% 97% 0.00 9581 sched lock 14 (usermode) 152 0% 97% 0.00 7466 sched lock 7 (usermode) 151 0% 98% 0.00 4378 sched lock 18 _thread_lock+0xf4 150 0% 98% 0.00 4495 sched lock 9 (usermode) 149 0% 98% 0.00 2991 sched lock 31 (usermode) 146 0% 98% 0.00 2794 sched lock 51 (usermode) 146 0% 98% 0.00 3019 sched lock 13 (usermode) 145 0% 98% 0.00 5076 sched lock 8 (usermode) 144 0% 98% 0.00 4270 sched lock 59 (usermode) 144 0% 98% 0.00 3001 sched lock 36 (usermode) 144 0% 98% 0.00 2228 sched lock 33 (usermode) 140 0% 98% 0.00 3137 sched lock 47 (usermode) 140 0% 98% 0.00 8321 sched lock 27 (usermode) 140 0% 98% 0.00 3593 sched lock 21 (usermode) 140 0% 99% 0.00 5201 sched lock 2 (usermode) 138 0% 99% 0.00 2643 sched lock 43 (usermode) 138 0% 99% 0.00 518430 sched lock 17 (usermode) 137 0% 99% 0.00 3708 sched lock 38 (usermode) 136 0% 99% 0.00 32694 sched lock 6 (usermode) 136 0% 99% 0.00 21938 sched lock 22 (usermode) 133 0% 99% 0.00 48916 sched lock 18 (usermode) 131 0% 99% 0.00 3284 sched lock 39 (usermode) 131 0% 99% 0.00 3171 sched lock 37 (usermode) 131 0% 99% 0.00 4030 sched lock 26 (usermode) 131 0% 99% 0.00 10086 sched lock 16 (usermode) 122 0% 99% 0.00 2574 sched lock 58 (usermode) 122 0% 100% 0.00 3520 sched lock 24 (usermode) 122 0% 100% 0.00 3994 sched lock 1 (usermode) 119 0% 100% 0.00 2928 sched lock 57 (usermode) 118 0% 100% 0.00 7106 sched lock 12 (usermode) 110 0% 100% 0.00 9853 sched lock 3 (usermode) 97 0% 100% 0.00 2823 sched lock 64 (usermode) 97 0% 100% 0.00 3564 sched lock 56 (usermode) 71 0% 100% 0.00 7126 sched lock 11 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 76382 events in 177.342 seconds (431 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 24935 33% 33% 0.00 16587356 vm object vm_fault_hold+0x1b28 10634 14% 47% 0.00 29618992 vm object vm_map_insert+0x758 8009 10% 57% 0.00 31687491 vm object vm_object_deallocate+0x800 5919 8% 65% 0.00 10540497 vm object (usermode) 5073 7% 71% 0.00 2176951 vm object 0xc000000003117de8 4462 6% 77% 0.00 24348844 vm object 0xc0000000030f0be8 4455 6% 83% 0.00 56647843 vm object vm_object_reference+0x134 2912 4% 87% 0.00 41359313 vm object vm_object_shadow+0x378 2613 3% 90% 0.00 2003464 vm object vm_object_collapse+0x710 2289 3% 93% 0.00 38253786 vm object vm_object_shadow+0x390 876 1% 94% 0.00 6663552 vm object vm_object_deallocate+0x814 679 1% 95% 0.00 50020048 vm object vm_imgact_map_page+0x3f0 659 1% 96% 0.00 1511370 vm object zfs_vmobject_wlock+0x9c 518 1% 97% 0.00 58688870 vm object exec_map_first_page+0x780 511 1% 98% 0.00 741686 kernel vm object kmem_back_domain+0x340 400 1% 98% 0.00 1781626 vm object 0xc000000003118428 257 0% 98% 0.00 644382 kernel vm object (usermode) 243 0% 99% 0.00 2559924 vm object vm_page_sleep_if_busy+0x1f8 232 0% 99% 0.00 2816866 vm object vm_fault_hold+0x2700 228 0% 99% 0.00 13223252 vm object vnode_create_vobject+0x18c 222 0% 100% 0.00 847205 kernel vm object 0xc0000000030e8b68 58 0% 100% 0.00 1108035 vm object vmspace_fork+0x2d4 58 0% 100% 0.00 27516 Per-Filesystem Softdep Lock 0xc000000003087ebc 38 0% 100% 0.00 2225429 vm object vm_object_split+0xbd0 21 0% 100% 0.00 6128671 vm object bufdone+0x45c 12 0% 100% 0.00 20299754 vm object allocbuf+0xa84 11 0% 100% 0.00 64451 Per-Filesystem Softdep Lock (usermode) 6 0% 100% 0.00 11281728 vm object vfs_bio_getpages+0x648 6 0% 100% 0.00 160056 Per-Filesystem Softdep Lock 0xc00000000308da54 5 0% 100% 0.00 20368232 vm object vfs_busy_pages+0x2a8 5 0% 100% 0.00 207704 Per-Filesystem Softdep Lock 0xc000000002ce4a60 4 0% 100% 0.00 23171144 vm object vfs_bio_getpages+0x728 4 0% 100% 0.00 26201 bufobj interlock bufobj_wdrop+0x13c 4 0% 100% 0.00 78927 bufobj interlock (usermode) 3 0% 100% 0.00 176690 vm object vm_fault_hold+0x1298 3 0% 100% 0.00 111642 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 2 0% 100% 0.00 63925 vm object bdwrite+0x2f4 2 0% 100% 0.00 62568 bufobj interlock bqrelse+0x2a0 2 0% 100% 0.00 111802 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 2 0% 100% 0.00 214637 Per-Filesystem Softdep Lock softdep_freefile+0xbb4 2 0% 100% 0.00 115578 Per-Filesystem Softdep Lock softdep_freefile+0x58c 2 0% 100% 0.00 214982 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 1 0% 100% 0.00 38162 Per-Filesystem Softdep Lock 0xc0000000030a5ca8 1 0% 100% 0.00 61632 Per-Filesystem Softdep Lock softdep_setup_unlink+0x1b8 1 0% 100% 0.00 22761 Per-Filesystem Softdep Lock 0xc000000003086e70 1 0% 100% 0.00 47461 Per-Filesystem Softdep Lock 0xc000000003094844 1 0% 100% 0.00 268669 Per-Filesystem Softdep Lock 0xc0000000030939d4 1 0% 100% 0.00 167262 Per-Filesystem Softdep Lock softdep_load_inodeblock+0x1d8 ------------------------------------------------------------------------------- R/W writer blocked by readers: 149513 events in 177.342 seconds (843 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 51686 35% 35% 0.00 27947071 vm object vm_fault_hold+0x1b28 20954 14% 49% 0.00 8652742 vm object (usermode) 15499 10% 59% 0.00 18163046 vm object 0xc0000000030f0be8 10657 7% 66% 0.00 35892506 vm object vm_object_deallocate+0x800 10453 7% 73% 0.00 30580397 vm object vnode_create_vobject+0x18c 9197 6% 79% 0.00 27157072 vm object vm_object_deallocate+0x814 6708 4% 84% 0.00 1397424 vm object 0xc000000003117de8 6437 4% 88% 0.00 39382886 vm object vm_object_reference+0x134 3612 2% 90% 0.00 22226572 vm object vm_object_shadow+0x378 2782 2% 92% 0.00 41515530 vm object vm_imgact_map_page+0x3f0 2249 2% 94% 0.00 20203944 vm object vm_map_insert+0x758 2136 1% 95% 0.00 1799366 vm object vm_fault_hold+0x1298 1676 1% 96% 0.00 1460897 vm object zfs_vmobject_wlock+0x9c 1644 1% 97% 0.00 2531180 vm object vm_fault_hold+0x2700 816 1% 98% 0.00 1549932 vm object 0xc000000003118428 815 1% 99% 0.00 1376295 vm object vm_object_collapse+0x710 516 0% 99% 0.00 737485 vm object vm_page_sleep_if_busy+0x1f8 472 0% 99% 0.00 22937733 vm object vm_object_shadow+0x390 445 0% 99% 0.00 673135 vm object vmspace_fork+0x2d4 317 0% 100% 0.00 87533 bufobj interlock (usermode) 190 0% 100% 0.00 120797 bufobj interlock bqrelse+0x2a0 108 0% 100% 0.00 39829095 vm object exec_map_first_page+0x780 28 0% 100% 0.00 22736343 vm object bufdone+0x45c 28 0% 100% 0.00 87476 bufobj interlock 0xc0000000030b1b10 17 0% 100% 0.00 19794611 vm object allocbuf+0xa84 16 0% 100% 0.00 773727 vm object vm_object_split+0xbd0 15 0% 100% 0.00 121937 bufobj interlock reassignbuf+0x350 14 0% 100% 0.00 14313598 vm object vfs_bio_getpages+0x648 9 0% 100% 0.00 40444162 vm object vfs_bio_getpages+0x728 8 0% 100% 0.00 19605782 vm object vfs_busy_pages+0x2a8 8 0% 100% 0.00 46542 bufobj interlock bufobj_wdrop+0x13c 1 0% 100% 0.00 92389 bufobj interlock brelvp+0x2c4 ------------------------------------------------------------------------------- R/W reader blocked by writer: 11380 events in 177.342 seconds (64 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 8970 79% 79% 0.00 14509515 vm object __rw_rlock_int+0x9c 2409 21% 100% 0.00 15143639 vm object (usermode) 1 0% 100% 0.00 30647 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 20844 events in 177.342 seconds (118 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 18058 87% 87% 0.00 5348050 vm object __rw_rlock_int+0x9c 1858 9% 96% 0.00 4751080 vm object (usermode) 851 4% 100% 0.00 139692 bufobj interlock __rw_rlock_int+0x9c 77 0% 100% 0.00 161019 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 497520 events in 177.342 seconds (2805 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 95104 19% 19% 0.00 1029488 vm object vm_fault_hold+0x1b28 70586 14% 33% 0.00 14854 vm object vm_fault_hold+0x2700 68352 14% 47% 0.00 228602 vm object (usermode) 45522 9% 56% 0.00 914775 vm object vm_map_insert+0x758 36265 7% 63% 0.00 431731 vm object 0xc000000003117de8 32194 6% 70% 0.00 897280 vm object vm_object_deallocate+0x800 21206 4% 74% 0.00 579125 vm object vm_object_shadow+0x378 18575 4% 78% 0.00 1076978 vm object vm_object_reference+0x134 15152 3% 81% 0.00 1092493 vm object 0xc0000000030f0be8 12064 2% 83% 0.00 636443 vm object vm_object_collapse+0x710 10190 2% 85% 0.00 1224755 kernel vm object kmem_back_domain+0x340 8960 2% 87% 0.00 772213 kernel vm object 0xc0000000030e8b68 7806 2% 89% 0.00 349445 vm object vm_imgact_map_page+0x3f0 6781 1% 90% 0.00 1317549 vm object vm_object_shadow+0x390 6183 1% 91% 0.00 11761 Per-Filesystem Softdep Lock (usermode) 6015 1% 93% 0.00 1114295 kernel vm object (usermode) 4347 1% 94% 0.00 521230 vm object zfs_vmobject_wlock+0x9c 3249 1% 94% 0.00 1020405 vm object vm_object_deallocate+0x814 2673 1% 95% 0.00 12173 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 2344 0% 95% 0.00 376787 vm object vm_page_sleep_if_busy+0x1f8 2278 0% 96% 0.00 21717 vm object vm_fault_hold+0x1298 2252 0% 96% 0.00 549032 vm object 0xc000000003118428 2085 0% 97% 0.00 10301 Per-Filesystem Softdep Lock 0xc00000000308da54 1258 0% 97% 0.00 47329 Per-Filesystem Softdep Lock 0xc000000002ce4a60 1218 0% 97% 0.00 1917664 vm object exec_map_first_page+0x780 1164 0% 97% 0.00 24099 vm object bufdone+0x45c 975 0% 97% 0.00 3117 vm object bdwrite+0x2f4 969 0% 98% 0.00 4183 bufobj interlock (usermode) 866 0% 98% 0.00 4901 bufobj interlock bqrelse+0x2a0 778 0% 98% 0.00 1514480 vm object vnode_create_vobject+0x18c 651 0% 98% 0.00 25319 tidhash (usermode) 598 0% 98% 0.00 8157 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 590 0% 98% 0.00 12318 Per-Filesystem Softdep Lock 0xc000000003087ebc 578 0% 98% 0.00 26950 tidhash tidhash_remove+0x13c 508 0% 99% 0.00 157885 vm object vmspace_fork+0x2d4 503 0% 99% 0.00 7841 Per-Filesystem Softdep Lock 0xc000000003097404 491 0% 99% 0.00 18582 Per-Filesystem Softdep Lock 0xc000000003086e70 417 0% 99% 0.00 8169 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c 415 0% 99% 0.00 255449 vm object vm_object_split+0xbd0 413 0% 99% 0.00 12787 Per-Filesystem Softdep Lock softdep_prealloc+0x460 411 0% 99% 0.00 10170 Per-Filesystem Softdep Lock softdep_freefile+0xbb4 403 0% 99% 0.00 17103 Per-Filesystem Softdep Lock softdep_slowdown+0x374 376 0% 99% 0.00 9182 Per-Filesystem Softdep Lock 0xc0000000030a5ca8 296 0% 99% 0.00 9132 Per-Filesystem Softdep Lock softdep_setup_create+0x188 267 0% 99% 0.00 5709 Per-Filesystem Softdep Lock softdep_load_inodeblock+0x1d8 262 0% 99% 0.00 13430 Per-Filesystem Softdep Lock softdep_setup_unlink+0x1b8 236 0% 99% 0.00 6337 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c 219 0% 100% 0.00 7589 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c 205 0% 100% 0.00 12863 Per-Filesystem Softdep Lock 0xc000000003094844 198 0% 100% 0.00 8662 Per-Filesystem Softdep Lock softdep_setup_allocdirect+0x170 197 0% 100% 0.00 7230 Per-Filesystem Softdep Lock softdep_freefile+0x58c 170 0% 100% 0.00 22138 Per-Filesystem Softdep Lock 0xc00000000308705c 160 0% 100% 0.00 6393 Per-Filesystem Softdep Lock softdep_freefile+0x944 132 0% 100% 0.00 6788 bufobj interlock bufobj_wdrop+0x13c 127 0% 100% 0.00 15083 Per-Filesystem Softdep Lock 0xc000000003085470 124 0% 100% 0.00 269587 vm object allocbuf+0xa84 117 0% 100% 0.00 5815 tcpinp (usermode) 108 0% 100% 0.00 16848 Per-Filesystem Softdep Lock 0xc00000000309f9b8 83 0% 100% 0.00 8355 Per-Filesystem Softdep Lock 0xc000000003090108 73 0% 100% 0.00 8114 Per-Filesystem Softdep Lock softdep_change_linkcnt+0x154 61 0% 100% 0.00 3866 bufobj interlock getblkx+0x6a4 58 0% 100% 0.00 6982 tcpinp 0xc000000002eb6908 55 0% 100% 0.00 8300 bufobj interlock bufobj_wref+0x114 52 0% 100% 0.00 15004 Per-Filesystem Softdep Lock 0xc00000000309a1ac 48 0% 100% 0.00 3465 bufobj interlock reassignbuf+0x350 42 0% 100% 0.00 6425 Per-Filesystem Softdep Lock 0xc000000003092458 40 0% 100% 0.00 272309 vm object vfs_busy_pages+0x2a8 38 0% 100% 0.00 33141 Per-Filesystem Softdep Lock 0xc0000000030a1778 37 0% 100% 0.00 300403 vm object vfs_bio_getpages+0x648 34 0% 100% 0.00 12075 Per-Filesystem Softdep Lock softdep_setup_directory_add+0xc2c 33 0% 100% 0.00 963 tidhash tidhash_add+0x16c 32 0% 100% 0.00 5709 vm object cluster_wbuild+0x52c 31 0% 100% 0.00 8087 Per-Filesystem Softdep Lock 0xc000000003094878 30 0% 100% 0.00 4214 Per-Filesystem Softdep Lock softdep_setup_blkmapdep+0x294 27 0% 100% 0.00 4127 bufobj interlock 0xc0000000030b1b10 26 0% 100% 0.00 38353 Per-Filesystem Softdep Lock 0xc0000000030a17d0 19 0% 100% 0.00 8191 Per-Filesystem Softdep Lock 0xc00000000308e6dc 13 0% 100% 0.00 800741 vm object vfs_bio_getpages+0x728 11 0% 100% 0.00 2341 vm object vnode_pager_setsize+0x2bc 11 0% 100% 0.00 10430 Per-Filesystem Softdep Lock softdep_setup_inomapdep+0x31c 10 0% 100% 0.00 26491 Per-Filesystem Softdep Lock 0xc000000003090544 10 0% 100% 0.00 3923 Per-Filesystem Softdep Lock 0xc00000000308dda4 10 0% 100% 0.00 25584 Per-Filesystem Softdep Lock 0xc000000003085394 7 0% 100% 0.00 2161 bufobj interlock cluster_wbuild+0x234 7 0% 100% 0.00 15940 Per-Filesystem Softdep Lock 0xc000000003099988 7 0% 100% 0.00 8987 Per-Filesystem Softdep Lock softdep_freefile+0xbec 7 0% 100% 0.00 7326 Per-Filesystem Softdep Lock 0xc000000003089de0 4 0% 100% 0.00 785 vfs hash vfs_hash_remove+0x13c 3 0% 100% 0.00 2587 bufobj interlock vn_fsync_buf+0x244 2 0% 100% 0.00 1718 vfs hash (usermode) 2 0% 100% 0.00 7204 bufobj interlock cluster_wbuild+0x510 2 0% 100% 0.00 10451 Per-Filesystem Softdep Lock 0xc0000000030a4c9c 2 0% 100% 0.00 892 Per-Filesystem Softdep Lock softdep_freefile+0x5cc 2 0% 100% 0.00 6191 Per-Filesystem Softdep Lock 0xc000000003090514 2 0% 100% 0.00 14043 Per-Filesystem Softdep Lock softdep_move_dependencies+0x264 1 0% 100% 0.00 1779 vm object vm_page_grab+0xf8 1 0% 100% 0.00 11046 vm object vm_object_coalesce+0x3c0 1 0% 100% 0.00 3277 tcpinp 0xc000000002f87f90 1 0% 100% 0.00 2978 Per-Filesystem Softdep Lock 0xc0000000030a6764 1 0% 100% 0.00 40668 Per-Filesystem Softdep Lock softdep_sync_buf+0x133c 1 0% 100% 0.00 105486 Per-Filesystem Softdep Lock 0xc000000003099970 ------------------------------------------------------------------------------- R/W writer spin on readers: 309738 events in 177.342 seconds (1747 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 74279 24% 24% 0.00 1622905 vm object vm_fault_hold+0x1b28 55335 18% 42% 0.00 350503 vm object (usermode) 26899 9% 51% 0.00 168203 vm object vm_fault_hold+0x2700 25382 8% 59% 0.00 1069162 vm object 0xc0000000030f0be8 21236 7% 66% 0.00 279190 vm object vm_fault_hold+0x1298 18048 6% 71% 0.00 634589 vm object 0xc000000003117de8 16683 5% 77% 0.00 1747353 vm object vm_object_deallocate+0x800 15212 5% 82% 0.00 1836544 vm object vnode_create_vobject+0x18c 13939 5% 86% 0.00 1428965 vm object vm_object_deallocate+0x814 8123 3% 89% 0.00 2336447 vm object vm_object_reference+0x134 6041 2% 91% 0.00 930949 vm object vm_map_insert+0x758 5754 2% 93% 0.00 1705795 vm object vm_object_shadow+0x378 5153 2% 94% 0.00 884167 vm object zfs_vmobject_wlock+0x9c 3342 1% 95% 0.00 2520208 vm object vm_imgact_map_page+0x3f0 2897 1% 96% 0.00 423220 vm object vm_object_collapse+0x710 2312 1% 97% 0.00 978749 vm object 0xc000000003118428 2107 1% 98% 0.00 41570 bufobj interlock (usermode) 1755 1% 98% 0.00 35264 bufobj interlock bqrelse+0x2a0 1631 1% 99% 0.00 152925 vm object vmspace_fork+0x2d4 1248 0% 99% 0.00 790776 vm object vm_page_sleep_if_busy+0x1f8 948 0% 100% 0.00 1185792 vm object vm_object_shadow+0x390 215 0% 100% 0.00 36006 bufobj interlock 0xc0000000030b1b10 200 0% 100% 0.00 101001 vm object vfs_bio_getpages+0x648 183 0% 100% 0.00 2299524 vm object exec_map_first_page+0x780 171 0% 100% 0.00 182512 vm object allocbuf+0xa84 114 0% 100% 0.00 8853 bufobj interlock bufobj_wref+0x114 103 0% 100% 0.00 842177 vm object bufdone+0x45c 96 0% 100% 0.00 501451 vm object vfs_bio_getpages+0x728 91 0% 100% 0.00 38480 bufobj interlock reassignbuf+0x350 76 0% 100% 0.00 39841 bufobj interlock bufobj_wdrop+0x13c 70 0% 100% 0.00 237755 vm object vfs_busy_pages+0x2a8 55 0% 100% 0.00 257018 vm object vm_object_split+0xbd0 28 0% 100% 0.00 1252 ncbuc 0xc000000002dfd7c8 5 0% 100% 0.00 215 vfs hash vfs_hash_remove+0x13c 5 0% 100% 0.00 620 ncbuc (usermode) 1 0% 100% 0.00 64335 bufobj interlock brelvp+0x2c4 1 0% 100% 0.00 3603 ncbuc 0xc000000002dfd7dc ------------------------------------------------------------------------------- R/W reader spin on writer: 126918 events in 177.342 seconds (716 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 91726 72% 72% 0.00 155486 vm object __rw_rlock_int+0x9c 33957 27% 99% 0.00 90561 vm object (usermode) 964 1% 100% 0.00 5968 bufobj interlock __rw_rlock_int+0x9c 218 0% 100% 0.00 3333 bufobj interlock (usermode) 33 0% 100% 0.00 1673 ncbuc __rw_rlock_int+0x9c 10 0% 100% 0.00 546 vfs hash __rw_rlock_int+0x9c 8 0% 100% 0.00 1106 ncbuc (usermode) 2 0% 100% 0.00 230 vfs hash (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 66140 events in 177.342 seconds (373 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 55346 84% 84% 0.00 348941 vm object __rw_rlock_int+0x9c 6055 9% 93% 0.00 346569 vm object (usermode) 4303 7% 99% 0.00 83009 bufobj interlock __rw_rlock_int+0x9c 435 1% 100% 0.00 82711 bufobj interlock (usermode) 1 0% 100% 0.00 10672 ncbuc (usermode) ------------------------------------------------------------------------------- SX exclusive block: 8782 events in 177.342 seconds (50 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 4555 52% 52% 0.00 266703 vm map (user) (usermode) 1366 16% 67% 0.00 345150 vm map (user) _vm_map_lock+0x138 827 9% 77% 0.00 1992304 zfsvfs->z_hold_mtx[i] zfs_zget+0x344 535 6% 83% 0.00 4238690 vq->vq_lock (usermode) 347 4% 87% 0.00 4040734 vq->vq_lock vdev_queue_io+0x268 188 2% 89% 0.00 494777 vm map (user) _vm_map_lock_upgrade+0x180 181 2% 91% 0.00 8313400 zfsvfs->z_hold_mtx[i] (usermode) 172 2% 93% 0.00 576176 dn->dn_struct_rwlock 0xc0000000025a6044 150 2% 95% 0.00 442656 dn->dn_struct_rwlock (usermode) 39 0% 95% 0.00 2497615 h->hash_mutexes[i] dbuf_find+0x114 37 0% 96% 0.00 3109526 h->hash_mutexes[i] (usermode) 35 0% 96% 0.00 437431 zp->z_acl_lock 0xc000000002678cac 27 0% 96% 0.00 344333 spa->spa_alloc_locks[i] 0xc0000000026bcf70 21 0% 97% 0.00 646377 dd->dd_lock 0xc0000000025e75ec 20 0% 97% 0.00 273724 zilog->zl_itxg[i].itxg_lock zil_itx_assign+0x5cc 19 0% 97% 0.00 3446857 dn->dn_dbufs_mtx (usermode) 19 0% 97% 0.00 130022 dd->dd_lock (usermode) 18 0% 97% 0.00 282397 spa->spa_alloc_locks[i] (usermode) 15 0% 98% 0.00 24022 zio->io_lock 0xc000000002cf97d8 14 0% 98% 0.00 2368949 db->db_mtx dbuf_find+0x12c 14 0% 98% 0.00 1181485 db->db_mtx (usermode) 12 0% 98% 0.00 3435164 dn->dn_dbufs_mtx 0xc0000000025a2970 9 0% 98% 0.00 138666 zp->z_acl_lock (usermode) 9 0% 98% 0.00 32569 rrl->rr_lock 0xc00000000260a56c 8 0% 98% 0.00 184559 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 8 0% 98% 0.00 28336 sa->sa_lock 0xc00000000260e244 8 0% 99% 0.00 29411 ms->ms_lock metaslab_alloc_dva+0x1138 8 0% 99% 0.00 2833655 dn->dn_dbufs_mtx dbuf_destroy+0x724 6 0% 99% 0.00 22866 rrl->rr_lock (usermode) 6 0% 99% 0.00 80155 os->os_lock 0xc0000000025ca334 6 0% 99% 0.00 821341 dp->dp_lock dsl_pool_undirty_space+0x18c 5 0% 99% 0.00 32649 rrl->rr_lock rrw_exit+0x310 5 0% 99% 0.00 30233 ms->ms_lock (usermode) 5 0% 99% 0.00 21226 mc->mc_lock metaslab_class_throttle_unreserve+0x154 5 0% 99% 0.00 126096 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 5 0% 99% 0.00 576568 h->hash_mutexes[i] dbuf_destroy+0x6dc 5 0% 99% 0.00 1080124 dp->dp_lock (usermode) 5 0% 99% 0.00 36119 dd->dd_lock dsl_dir_tempreserve_clear+0x88 5 0% 99% 0.00 86804 buf_hash_table.ht_locks[i].ht_lock arc_buf_access+0x488 4 0% 99% 0.00 2898183 vq->vq_lock 0xc000000002655fb4 4 0% 99% 0.00 295248 proctree kern_wait6+0x3e8 4 0% 99% 0.00 35921 mg->mg_lock (usermode) 3 0% 99% 0.00 389837 zs->zs_lock (usermode) 3 0% 99% 0.00 397845 zilog->zl_itxg[i].itxg_lock (usermode) 3 0% 100% 0.00 33832 os->os_lock (usermode) 3 0% 100% 0.00 30749 mg->mg_lock metaslab_alloc_dva+0x7d4 3 0% 100% 0.00 23442 mg->mg_lock metaslab_alloc_dva+0x3a8 3 0% 100% 0.00 36509 dp->dp_lock dsl_pool_need_dirty_delay+0x194 2 0% 100% 0.00 250067 zs->zs_lock dmu_zfetch+0xf0 2 0% 100% 0.00 835276 tx->tx_cpu[c].tc_open_lock txg_hold_open+0x1b8 2 0% 100% 0.00 26011 scl->scl_lock spa_config_enter+0x140 2 0% 100% 0.00 32485 sa->sa_lock (usermode) 2 0% 100% 0.00 25269 ds->ds_lock dsl_dataset_check_quota+0x2f0 2 0% 100% 0.00 3744286 dn->dn_mtx dnode_add_ref+0x1b8 2 0% 100% 0.00 1782793 db->db_mtx 0xc0000000025a6010 2 0% 100% 0.00 32839 db->db_mtx dbuf_read+0xa44 1 0% 100% 0.00 55730 zp->z_range_lock zfs_range_lock+0x854 1 0% 100% 0.00 27392 zio->io_lock (usermode) 1 0% 100% 0.00 44756 vq->vq_lock vdev_queue_io_done+0x278 1 0% 100% 0.00 161653 tl->tl_lock txg_list_add+0x16c 1 0% 100% 0.00 24355 scl->scl_lock spa_config_exit+0x84 1 0% 100% 0.00 39354 proctree (usermode) 1 0% 100% 0.00 298817 proctree fork1+0x28bc 1 0% 100% 0.00 443203 proctree exit1+0x1aec 1 0% 100% 0.00 23534 proctree sys_setpgid+0x5c8 1 0% 100% 0.00 46788 dd->dd_lock dsl_dir_willuse_space+0x204 1 0% 100% 0.00 41548 dd->dd_lock dsl_dir_diduse_space+0x270 1 0% 100% 0.00 273699 db->db_mtx 0xc0000000025a29a0 1 0% 100% 0.00 2305944 db->db_mtx dbuf_rele+0xbc 1 0% 100% 0.00 34170 mc->mc_lock (usermode) 1 0% 100% 0.00 24364 as->as_lock aggsum_add+0x360 1 0% 100% 0.00 498853 as->as_lock (usermode) 1 0% 100% 0.00 3023322 buf_hash_table.ht_locks[i].ht_lock (usermode) ------------------------------------------------------------------------------- SX shared block: 4439 events in 177.342 seconds (25 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2760 62% 62% 0.00 724695 vm map (user) _sx_slock_int+0xa0 1651 37% 99% 0.00 1524572 vm map (user) (usermode) 11 0% 100% 0.00 59199 zf->zf_rwlock _sx_slock_int+0xa0 9 0% 100% 0.00 5758045 dn->dn_struct_rwlock _sx_slock_int+0xa0 7 0% 100% 0.00 41174 zf->zf_rwlock (usermode) 1 0% 100% 0.00 28753 dn->dn_struct_rwlock (usermode) ------------------------------------------------------------------------------- SX exclusive spin: 2326924 events in 177.342 seconds (13121 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 308171 13% 13% 0.00 2935 db->db_mtx dbuf_read+0xa44 269853 12% 25% 0.00 6347 dd->dd_lock 0xc0000000025e75ec 265068 11% 36% 0.00 5660 dd->dd_lock (usermode) 192173 8% 44% 0.00 1730 db->db_mtx dbuf_rele+0xbc 173229 7% 52% 0.00 11921 vm map (user) _vm_map_lock+0x138 172301 7% 59% 0.00 4912 db->db_mtx dbuf_find+0x12c 147447 6% 66% 0.00 21001 vm map (user) (usermode) 123754 5% 71% 0.00 1968 zilog->zl_itxg[i].itxg_lock zil_itx_assign+0x5cc 108611 5% 76% 0.00 8270 db->db_mtx (usermode) 80364 3% 79% 0.00 15174 h->hash_mutexes[i] dbuf_find+0x114 74160 3% 82% 0.00 3826 dd->dd_lock dsl_dir_tempreserve_clear+0x88 67271 3% 85% 0.00 2880 rrl->rr_lock rrw_exit+0x310 65259 3% 88% 0.00 1897 rrl->rr_lock 0xc00000000260a56c 35573 2% 90% 0.00 2198 rrl->rr_lock (usermode) 35453 2% 91% 0.00 2052 zilog->zl_itxg[i].itxg_lock (usermode) 32223 1% 92% 0.00 1281 dp->dp_lock dsl_pool_need_dirty_delay+0x194 21222 1% 93% 0.00 2512 zp->z_acl_lock 0xc000000002678cac 18987 1% 94% 0.00 9878 vm map (user) _vm_map_lock_upgrade+0x180 12178 1% 95% 0.00 282875 vq->vq_lock vdev_queue_io+0x268 11178 0% 95% 0.00 1513 dp->dp_lock (usermode) 9674 0% 96% 0.00 290613 vq->vq_lock (usermode) 8684 0% 96% 0.00 3033 zp->z_acl_lock (usermode) 6448 0% 96% 0.00 186080 h->hash_mutexes[i] (usermode) 6028 0% 96% 0.00 12166 proctree kern_wait6+0x3e8 5475 0% 97% 0.00 1900 sa->sa_lock 0xc00000000260e244 5069 0% 97% 0.00 6745 dd->dd_lock dsl_dir_willuse_space+0x204 4801 0% 97% 0.00 49916 zfsvfs->z_hold_mtx[i] zfs_zget+0x344 3872 0% 97% 0.00 4485 tm->tm_lock trim_map_free+0x19c 3353 0% 97% 0.00 6791 mg->mg_lock metaslab_alloc_dva+0x3a8 2885 0% 98% 0.00 11506 mg->mg_lock metaslab_alloc_dva+0x7d4 2686 0% 98% 0.00 413476 os->os_userused_lock (usermode) 2587 0% 98% 0.00 1734 os->os_lock 0xc0000000025ca334 2400 0% 98% 0.00 22867 ms->ms_lock metaslab_alloc_dva+0x1138 2220 0% 98% 0.00 39039 zio->io_lock (usermode) 2039 0% 98% 0.00 109366 spa->spa_alloc_locks[i] 0xc0000000026bcf70 2017 0% 98% 0.00 23618 dd->dd_lock dsl_dir_diduse_space+0x270 2014 0% 98% 0.00 11458 mg->mg_lock (usermode) 1804 0% 98% 0.00 473574 os->os_userused_lock 0xc0000000025b7700 1778 0% 98% 0.00 13227 zio->io_lock 0xc0000000026c1474 1593 0% 98% 0.00 36430 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 1331 0% 99% 0.00 106312 zio->io_lock 0xc0000000026bf498 1305 0% 99% 0.00 5319 scl->scl_lock spa_config_enter+0x140 1277 0% 99% 0.00 190413 spa->spa_alloc_locks[i] (usermode) 1248 0% 99% 0.00 3004 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 1207 0% 99% 0.00 1634 dn->dn_mtx dnode_block_freed+0x320 1192 0% 99% 0.00 15192 vq->vq_lock vdev_queue_io_done+0x278 1184 0% 99% 0.00 75355 scl->scl_lock spa_config_exit+0x84 1150 0% 99% 0.00 1780 os->os_lock (usermode) 1132 0% 99% 0.00 11870 zio->io_lock zio_add_child+0x274 1111 0% 99% 0.00 70586 zfsvfs->z_hold_mtx[i] (usermode) 1106 0% 99% 0.00 33572 dn->dn_struct_rwlock 0xc0000000025a6044 1064 0% 99% 0.00 373 zs->zs_lock dmu_zfetch+0xf0 1032 0% 99% 0.00 183731 h->hash_mutexes[i] dbuf_destroy+0x6dc 807 0% 99% 0.00 386721 os->os_userused_lock 0xc0000000025b7898 805 0% 99% 0.00 175995 vq->vq_lock 0xc000000002655fb4 798 0% 99% 0.00 18587 ms->ms_lock (usermode) 797 0% 99% 0.00 2468 mc->mc_lock metaslab_class_throttle_unreserve+0x154 759 0% 99% 0.00 12432 vq->vq_lock vdev_queue_io_done+0x12c 718 0% 99% 0.00 1973 dp->dp_lock dsl_pool_dirty_space+0x17c 704 0% 99% 0.00 17310 proctree fork1+0x28bc 695 0% 99% 0.00 2872 db->db_mtx dmu_buf_rele+0xbc 670 0% 99% 0.00 2483 zfsvfs->z_znodes_lock 0xc0000000026ab588 640 0% 99% 0.00 1748 dd->dd_lock dsl_dir_space_available+0x284 588 0% 99% 0.00 59602 buf_hash_table.ht_locks[i].ht_lock arc_buf_access+0x488 561 0% 100% 0.00 16962 proctree (usermode) 503 0% 100% 0.00 2125 dp->dp_lock dsl_pool_undirty_space+0x18c 489 0% 100% 0.00 3107 db->db_mtx dmu_buf_will_dirty+0x2e8 476 0% 100% 0.00 3484 db->db_mtx dmu_buf_hold_array_by_dnode+0x6d0 455 0% 100% 0.00 2257 sa->sa_lock (usermode) 438 0% 100% 0.00 80186 as->as_lock aggsum_add+0x360 420 0% 100% 0.00 4786 ds->ds_lock dsl_dataset_block_kill+0x614 408 0% 100% 0.00 6071 db->db_mtx dmu_buf_replace_user+0x150 406 0% 100% 0.00 6035 ds->ds_lock dsl_dataset_block_born+0x3d4 393 0% 100% 0.00 8075 ds->ds_lock dsl_dataset_block_kill+0x5e4 360 0% 100% 0.00 9113 proctree exit1+0x1aec 355 0% 100% 0.00 172045 dn->dn_struct_rwlock (usermode) 333 0% 100% 0.00 3217 mc->mc_lock (usermode) 328 0% 100% 0.00 2234 db->db_mtx 0xc0000000025a2414 325 0% 100% 0.00 2557 zfsvfs->z_znodes_lock (usermode) 325 0% 100% 0.00 2450 hdl->sa_lock sa_bulk_lookup+0x154 311 0% 100% 0.00 2775 dn->dn_mtx (usermode) 308 0% 100% 0.00 2013 db->db_mtx dbuf_dirty+0xcd0 260 0% 100% 0.00 943524 dn->dn_dbufs_mtx (usermode) 254 0% 100% 0.00 4994 ds->ds_lock (usermode) 210 0% 100% 0.00 916787 dn->dn_dbufs_mtx 0xc0000000025a2970 204 0% 100% 0.00 678657 dn->dn_dbufs_mtx dbuf_destroy+0x724 201 0% 100% 0.00 9101 db->db_mtx 0xc000000002cf97d8 198 0% 100% 0.00 60229 scl->scl_lock (usermode) 198 0% 100% 0.00 54113 h->hash_mutexes[i] 0xc0000000025a2988 169 0% 100% 0.00 4701 db->db_mtx dbuf_prefetch+0x5ec 159 0% 100% 0.00 35408 os->os_obj_lock (usermode) 144 0% 100% 0.00 11164 proctree killjobc+0x600 133 0% 100% 0.00 1606 ms->ms_lock metaslab_free_concrete+0x2e8 129 0% 100% 0.00 874 hdl->sa_lock (usermode) 127 0% 100% 0.00 601 mls->mls_lock multilist_insert+0x1d4 108 0% 100% 0.00 8366 proctree sys_setpgid+0x5c8 108 0% 100% 0.00 365863 db->db_mtx 0xc0000000025a29a0 105 0% 100% 0.00 1953 dn->dn_mtx dnode_rele+0xbc 99 0% 100% 0.00 1378 ds->ds_lock dsl_dataset_check_quota+0x2f0 97 0% 100% 0.00 224935 buf_hash_table.ht_locks[i].ht_lock (usermode) 88 0% 100% 0.00 1658 zilog->zl_itxg[i].itxg_lock zil_async_to_sync+0xd0 85 0% 100% 0.00 2353 tl->tl_lock txg_list_add+0x16c 82 0% 100% 0.00 1272 db->db_mtx 0xc0000000025a492c 76 0% 100% 0.00 1586 allproc fork1+0x26bc 73 0% 100% 0.00 829 dn->dn_mtx dnode_hold_impl+0x97c 72 0% 100% 0.00 639 hdl->sa_lock sa_lookup_uio+0x198 67 0% 100% 0.00 1072 zp->z_range_lock zfs_range_unlock+0x3b4 65 0% 100% 0.00 29844 os->os_obj_lock 0xc0000000025b3054 64 0% 100% 0.00 3846 dn->dn_mtx 0xc0000000025a605c 63 0% 100% 0.00 376 mls->mls_lock multilist_remove+0x1d4 62 0% 100% 0.00 1131 bpl->bpl_lock bplist_append+0x158 58 0% 100% 0.00 23046 as->as_lock (usermode) 53 0% 100% 0.00 572 dn->dn_mtx dnode_diduse_space+0x184 51 0% 100% 0.00 756 zp->z_range_lock zfs_range_lock+0x854 43 0% 100% 0.00 435 zs->zs_lock (usermode) 39 0% 100% 0.00 837 mls->mls_lock (usermode) 31 0% 100% 0.00 3500 dp->dp_lock dsl_pool_mos_diduse_space+0x160 31 0% 100% 0.00 2591 dp->dp_lock dmu_free_long_range+0x54c 30 0% 100% 0.00 56888 buf_hash_table.ht_locks[i].ht_lock 0xc00000000258b984 29 0% 100% 0.00 691 bpl->bpl_lock (usermode) 26 0% 100% 0.00 1138 allproc exit1+0x1a84 25 0% 100% 0.00 775 zp->z_range_lock (usermode) 22 0% 100% 0.00 20133 db->db_mtx dbuf_read+0xdf4 20 0% 100% 0.00 9944 tx->tx_sync_lock txg_kick+0x174 20 0% 100% 0.00 2981 tl->tl_lock (usermode) 19 0% 100% 0.00 912 dn->dn_mtx dmu_tx_assign+0x1b0 18 0% 100% 0.00 3398 sa->sa_lock 0xc00000000260d738 17 0% 100% 0.00 1428 dn->dn_mtx dbuf_dirty+0xce8 13 0% 100% 0.00 1011 tm->tm_lock trim_map_write_start+0x3f8 13 0% 100% 0.00 302 hdl->sa_lock sa_lookup+0x158 13 0% 100% 0.00 1093 dn->dn_mtx dmu_tx_commit+0x80 12 0% 100% 0.00 652 sa->sa_lock 0xc000000002610a34 12 0% 100% 0.00 2817 db->db_mtx dnode_rele_and_unlock+0x18c 12 0% 100% 0.00 1089 allproc (usermode) 11 0% 100% 0.00 2271 dn->dn_mtx dmu_object_info_from_dnode+0x1d0 10 0% 100% 0.00 681 zio->io_lock 0xc0000000026bda18 10 0% 100% 0.00 5333 tm->tm_lock trim_map_write_done+0x188 10 0% 100% 0.00 1180 dp->dp_lock dmu_free_long_range+0x384 10 0% 100% 0.00 687232 buf_hash_table.ht_locks[i].ht_lock 0xc00000000258bc0c 9 0% 100% 0.00 2664 zio->io_lock 0xc0000000026be2dc 8 0% 100% 0.00 5031 tx->tx_sync_lock (usermode) 7 0% 100% 0.00 4381 zio->io_lock 0xc000000002cf97d8 6 0% 100% 0.00 476 zombproc (usermode) 6 0% 100% 0.00 326 vd->vdev_stat_lock vdev_stat_update+0x348 6 0% 100% 0.00 302 vd->vdev_dtl_lock vdev_dtl_contains+0x194 6 0% 100% 0.00 1075 db->db_mtx 0xc0000000025a2470 5 0% 100% 0.00 484 zombproc exit1+0x1aa0 5 0% 100% 0.00 2036 zilog->zl_itxg[i].itxg_lock zil_itx_assign+0xfc 5 0% 100% 0.00 296 vc->vc_lock vdev_cache_read+0x5b8 5 0% 100% 0.00 3009 tx->tx_sync_lock 0xc000000002cf97d8 5 0% 100% 0.00 9414 tx->tx_cpu[c].tc_open_lock txg_hold_open+0x1b8 5 0% 100% 0.00 972 dn->dn_mtx dbuf_destroy+0x6c0 5 0% 100% 0.00 45138 db->db_mtx 0xc0000000025a6010 5 0% 100% 0.00 1196 db->db_mtx 0xc0000000025a3cc4 4 0% 100% 0.00 2121 dn->dn_mtx dbuf_dirty+0xd38 4 0% 100% 0.00 55439 db->db_mtx 0xc0000000025a62f0 4 0% 100% 0.00 3583 db->db_mtx dbuf_dirty+0xdc0 3 0% 100% 0.00 2132 zio->io_lock 0xc0000000026bf808 3 0% 100% 0.00 876 zap->zap_rwlock 0xc000000002668334 3 0% 100% 0.00 4870 tx->tx_cpu[c].tc_open_lock (usermode) 3 0% 100% 0.00 128568 buf_hash_table.ht_locks[i].ht_lock arc_release+0x928 3 0% 100% 0.00 403352 buf_hash_table.ht_locks[i].ht_lock arc_buf_destroy+0x330 3 0% 100% 0.00 730 as->as_buckets[i].asc_lock aggsum_add+0x330 2 0% 100% 0.00 304 zio->io_lock 0xc0000000026bfc60 2 0% 100% 0.00 295 zio->io_lock 0xc0000000026bda48 2 0% 100% 0.00 1344 tx->tx_cpu[c].tc_open_lock 0xc000000002637e00 2 0% 100% 0.00 3243 tm->tm_lock (usermode) 2 0% 100% 0.00 297 db->db_mtx 0xc0000000025a42b4 2 0% 100% 0.00 3123 as->as_buckets[i].asc_lock (usermode) 1 0% 100% 0.00 248 zombproc proc_reap+0xca8 1 0% 100% 0.00 5669 zio->io_lock 0xc0000000026be83c 1 0% 100% 0.00 285 zio->io_lock 0xc0000000026bb8e0 1 0% 100% 0.00 10825 zfsvfs->z_hold_mtx[i] zfs_mknode+0xa48 1 0% 100% 0.00 221 vd->vdev_stat_lock (usermode) 1 0% 100% 0.00 307 vc->vc_lock (usermode) 1 0% 100% 0.00 1558 tx->tx_cpu[c].tc_lock txg_rele_to_sync+0x170 1 0% 100% 0.00 306 tx->tx_cpu[c].tc_lock 0xc000000002637fa4 1 0% 100% 0.00 550 tl->tl_lock txg_list_remove+0x164 1 0% 100% 0.00 273 mls->mls_lock multilist_sublist_lock_obj+0xd0 1 0% 100% 0.00 236 os->os_user_ptr_lock zfs_get_vfs_flag_unmounted+0x150 1 0% 100% 0.00 62222 dn->dn_struct_rwlock dnode_sync+0xd38 1 0% 100% 0.00 529 dn->dn_mtx dnode_sync+0x6e0 1 0% 100% 0.00 248 dp->dp_lock dsl_pool_sync+0x59c 1 0% 100% 0.00 264 dd->dd_lock dsl_dir_sync+0x148 1 0% 100% 0.00 435 dn->dn_mtx dbuf_dirty+0xd84 1 0% 100% 0.00 359 dn->dn_mtx dbuf_dirty+0xd6c 1 0% 100% 0.00 5960 dn->dn_mtx dmu_objset_userquota_get_ids+0x5f0 1 0% 100% 0.00 439 as->as_lock aggsum_value+0x3a8 1 0% 100% 0.00 1724 db->db_mtx 0xc0000000025a4e8c ------------------------------------------------------------------------------- SX shared spin: 435141 events in 177.342 seconds (2454 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 237120 54% 54% 0.00 17743 vm map (user) _sx_slock_int+0xa0 120178 28% 82% 0.00 373 zf->zf_rwlock _sx_slock_int+0xa0 61642 14% 96% 0.00 30682 vm map (user) (usermode) 14572 3% 100% 0.00 397 zf->zf_rwlock (usermode) 996 0% 100% 0.00 55771 dn->dn_struct_rwlock _sx_slock_int+0xa0 360 0% 100% 0.00 1541 zap->zap_rwlock _sx_slock_int+0xa0 198 0% 100% 0.00 25996 dn->dn_struct_rwlock (usermode) 75 0% 100% 0.00 1103 zap->zap_rwlock (usermode) ------------------------------------------------------------------------------- root@fledgling:/usr/src #