-------------------------------------------------------------- >>> Kernel build for GENERIC64 completed on Tue May 28 21:58:06 CDT 2019 -------------------------------------------------------------- >>> Kernel(s) GENERIC64 built in 137 seconds, ncpu: 72 -------------------------------------------------------------- Adaptive mutex spin: 53952 events in 139.370 seconds (387 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 13389 25% 25% 0.00 46913 dp_sync_taskq (usermode) 8969 17% 41% 0.00 55887 dp_sync_taskq 0xc000000002d74ef8 3726 7% 48% 0.00 39022 dp_sync_taskq taskqueue_enqueue+0x158 3265 6% 54% 0.00 122362 kernelpmap moea64_kextract+0x174 2843 5% 60% 0.00 54249 dp_sync_taskq 0xc000000002cbb944 2799 5% 65% 0.00 81389 zio_write_issue 0xc000000002d74ef8 2739 5% 70% 0.00 38981 zio_write_intr (usermode) 2294 4% 74% 0.00 67844 zio_write_issue 0xc000000002cbb944 2114 4% 78% 0.00 150323 kernelpmap (usermode) 2074 4% 82% 0.00 76045 kernelpmap moea64_enter+0x918 1839 3% 85% 0.00 41144 zio_write_intr 0xc000000002cbb944 1648 3% 88% 0.00 40138 zio_write_intr 0xc000000002d74ef8 1364 3% 91% 0.00 66618 zio_write_issue taskqueue_enqueue+0x158 1254 2% 93% 0.00 106535 zio_write_issue (usermode) 812 2% 95% 0.00 33061 zio_write_intr taskqueue_enqueue+0x158 745 1% 96% 0.00 24552 page pv moea64_enter+0x904 307 1% 97% 0.00 47290 page pv (usermode) 248 0% 97% 0.00 31084 page pv moea64_remove_pages+0x214 211 0% 98% 0.00 124191 page pv moea64_remove+0x22c 186 0% 98% 0.00 17030 struct mount vlist mtx vfs_msync+0x1d4 116 0% 98% 0.00 31663 struct mount vlist mtx (usermode) 105 0% 98% 0.00 14910 struct mount vlist mtx _vdrop+0x6f4 94 0% 98% 0.00 1461 struct mount vlist mtx _vhold+0x474 85 0% 99% 0.00 27865 process lock kthread_suspend_check+0x184 72 0% 99% 0.00 129484 nvme qpair lock 0xc000000002a0bb90 58 0% 99% 0.00 118792 kernelpmap moea64_remove+0x420 46 0% 99% 0.00 27878 dp_sync_taskq taskqueue_quiesce+0x2e8 41 0% 99% 0.00 20434 process lock (usermode) 33 0% 99% 0.00 1621 ttymtx 0xc000000002db0b3c 33 0% 99% 0.00 8054 page pv moea64_enter+0x9a8 33 0% 99% 0.00 20633 metaslab_group_taskq (usermode) 23 0% 99% 0.00 22723 metaslab_group_taskq taskqueue_enqueue+0x158 21 0% 99% 0.00 10868 CAM device lock (usermode) 19 0% 99% 0.00 3668 CAM device lock 0xc000000002523fac 17 0% 99% 0.00 5976 CAM device lock 0xc000000002524924 12 0% 99% 0.00 13194 vm active pagequeue vm_page_dequeue+0x1b4 12 0% 99% 0.00 8374 ttymtx 0xc000000002dafba8 11 0% 99% 0.00 12631 ttymtx (usermode) 10 0% 99% 0.00 13721 vm active pagequeue 0xc000000003109348 10 0% 99% 0.00 4606 ttymtx 0xc000000002da9f60 10 0% 100% 0.00 35768 nvme qpair lock (usermode) 9 0% 100% 0.00 7707 nvme qpair lock nvme_qpair_submit_request+0x120 8 0% 100% 0.00 3472 zio_free_issue_4 taskqueue_enqueue+0x158 8 0% 100% 0.00 7049 zio_free_issue_0 taskqueue_enqueue+0x158 8 0% 100% 0.00 27618 metaslab_group_taskq 0xc000000002d74ef8 7 0% 100% 0.00 8504 zio_free_issue_4 0xc000000002d74ef8 7 0% 100% 0.00 4545 zio_free_issue_4 0xc000000002cbb944 7 0% 100% 0.00 6427 zio_free_issue_3 taskqueue_enqueue+0x158 7 0% 100% 0.00 1766 zio_free_issue_3 0xc000000002cbb944 7 0% 100% 0.00 2083 zio_free_issue_0 0xc000000002cbb944 7 0% 100% 0.00 5597 pipe mutex (usermode) 7 0% 100% 0.00 6922 kernel arena domain vmem_xalloc+0x8c0 7 0% 100% 0.00 3421 CAM queue lock 0xc000000002511be8 6 0% 100% 0.00 704 zio_free_issue_7 taskqueue_enqueue+0x158 6 0% 100% 0.00 2535 zio_free_issue_7 0xc000000002cbb944 6 0% 100% 0.00 3324 zio_free_issue_6 0xc000000002d74ef8 6 0% 100% 0.00 2222 zio_free_issue_6 taskqueue_enqueue+0x158 6 0% 100% 0.00 1108 zio_free_issue_5 0xc000000002d74ef8 6 0% 100% 0.00 1825 zio_free_issue_5 taskqueue_enqueue+0x158 6 0% 100% 0.00 9781 zio_free_issue_0 0xc000000002d74ef8 5 0% 100% 0.00 11199 zio_free_issue_7 0xc000000002d74ef8 5 0% 100% 0.00 594 zio_free_issue_1 taskqueue_enqueue+0x158 5 0% 100% 0.00 23877 zio_free_issue_0 (usermode) 5 0% 100% 0.00 8881 ttymtx ttyoutq_read_uio+0x34c 5 0% 100% 0.00 47205 metaslab_group_taskq 0xc000000002cbb944 4 0% 100% 0.00 14992 zio_free_issue_6 0xc000000002cbb944 4 0% 100% 0.00 6105 zio_free_issue_3 0xc000000002d74ef8 4 0% 100% 0.00 724 zio_free_issue_2 taskqueue_enqueue+0x158 4 0% 100% 0.00 5330 zio_free_issue_1 0xc000000002cbb944 4 0% 100% 0.00 5758 vm page vm_page_change_lock+0x160 4 0% 100% 0.00 1181 vm page vm_fault_hold+0x2724 4 0% 100% 0.00 930 vm active pagequeue (usermode) 4 0% 100% 0.00 33578 ttymtx ttyinq_read_uio+0x1e0 4 0% 100% 0.00 3505 kernel arena domain vmem_xfree+0x39c 3 0% 100% 0.00 3775 zio_free_issue_7 (usermode) 3 0% 100% 0.00 1704 zio_free_issue_6 (usermode) 3 0% 100% 0.00 19893 zio_free_issue_2 0xc000000002d74ef8 3 0% 100% 0.00 1985 zio_free_issue_1 0xc000000002d74ef8 3 0% 100% 0.00 7361 so_snd sosend_generic+0x3bc 3 0% 100% 0.00 394 pipe mutex 0xc000000002d8f0a8 3 0% 100% 0.00 2466 UMA Slabs uma_large_malloc_domain+0x2ac 3 0% 100% 0.00 7699 UMA Slabs (usermode) 3 0% 100% 0.00 2437 CAM queue lock 0xc00000000250c3e8 2 0% 100% 0.00 1237 zio_null_issue taskqueue_enqueue+0x158 2 0% 100% 0.00 2425 zio_null_intr 0xc000000002d74ef8 2 0% 100% 0.00 863 zio_null_issue (usermode) 2 0% 100% 0.00 4672 zio_free_issue_5 (usermode) 2 0% 100% 0.00 11512 vm page 0xc0000000030ff640 2 0% 100% 0.00 1188 vm page 0xc0000000030e64cc 2 0% 100% 0.00 273 ttymtx 0xc000000002daa620 2 0% 100% 0.00 16977 ttymtx 0xc000000002da9c80 2 0% 100% 0.00 34291 struct mount vlist mtx 0xc000000002e20280 2 0% 100% 0.00 25313 so_snd tcp_do_segment+0x3ca0 2 0% 100% 0.00 398 so_snd sopoll_generic+0x470 2 0% 100% 0.00 6832 bge0 0xc000000002824dac 1 0% 100% 0.00 556 zio_null_intr (usermode) 1 0% 100% 0.00 333 zio_null_intr 0xc000000002cbb944 1 0% 100% 0.00 17326 zio_free_issue_4 (usermode) 1 0% 100% 0.00 2656 zio_free_issue_5 0xc000000002cbb944 1 0% 100% 0.00 2460 zio_free_issue_3 (usermode) 1 0% 100% 0.00 4634 zio_free_issue_2 0xc000000002cbb944 1 0% 100% 0.00 258 zio_free_issue_2 (usermode) 1 0% 100% 0.00 861 zio_free_issue_1 (usermode) 1 0% 100% 0.00 207 ttymtx ttydisc_write+0x388 1 0% 100% 0.00 307 vm page vm_object_unwire+0x2d0 1 0% 100% 0.00 797 vnode interlock 0xc000000002e1ab28 1 0% 100% 0.00 1498 xhci0 0xc000000002cbb944 1 0% 100% 0.00 16617 ttymtx 0xc000000002cbb944 1 0% 100% 0.00 277 so_snd tcp_output+0x1de8 1 0% 100% 0.00 446 pipe mutex 0xc000000002d8cbf8 1 0% 100% 0.00 1092 pipe mutex 0xc000000002d8beb4 1 0% 100% 0.00 432 pipe mutex 0xc000000002d8c0ec 1 0% 100% 0.00 18278 so_snd sbdrop+0x130 1 0% 100% 0.00 2467 bge0 (usermode) 1 0% 100% 0.00 924 UMA Slabs 0xc0000000030e184c 1 0% 100% 0.00 578 USB device mutex 0xc000000002b36774 1 0% 100% 0.00 160 CAM queue lock xpt_action_default+0x1254 ------------------------------------------------------------------------------- Adaptive mutex block: 41 events in 139.370 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 5 12% 12% 0.00 105294 nvme qpair lock 0xc000000002a0bb90 4 10% 22% 0.00 48228 zio_write_intr taskqueue_enqueue+0x158 4 10% 32% 0.00 75146 kernelpmap moea64_kextract+0x174 4 10% 41% 0.00 29423 kernelpmap moea64_enter+0x918 4 10% 51% 0.00 66112 dp_sync_taskq 0xc000000002cbb944 3 7% 59% 0.00 59189 zio_write_intr 0xc000000002cbb944 3 7% 66% 0.00 63198 zio_write_intr (usermode) 3 7% 73% 0.00 196347 dp_sync_taskq taskqueue_enqueue+0x158 2 5% 78% 0.00 43484 zio_write_issue (usermode) 2 5% 83% 0.00 72708 dp_sync_taskq (usermode) 1 2% 85% 0.00 43825 zio_write_issue 0xc000000002d74ef8 1 2% 88% 0.00 41196 zio_write_issue 0xc000000002cbb944 1 2% 90% 0.00 129694 page pv moea64_remove+0x22c 1 2% 93% 0.00 42076 ttymtx 0xc000000002dafba8 1 2% 95% 0.00 78305 kernelpmap (usermode) 1 2% 98% 0.00 66205 dp_sync_taskq 0xc000000002d74ef8 1 2% 100% 0.00 78356 CAM device lock 0xc000000002524924 ------------------------------------------------------------------------------- Spin lock spin: 6329 events in 139.370 seconds (45 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2842 45% 45% 0.00 11642 sleepq chain (usermode) 2798 44% 89% 0.00 14278 sleepq chain sleepq_lock+0xe4 55 1% 90% 0.00 31434 sched lock 10 0xc000000002d35384 34 1% 91% 0.00 27049 sched lock 5 0xc000000002d35384 30 0% 91% 0.00 38232 sched lock 30 0xc000000002d35384 28 0% 91% 0.00 22488 sched lock 15 0xc000000002d35384 22 0% 92% 0.00 12169 sched lock 30 0xc000000002d3539c 21 0% 92% 0.00 29628 sched lock 50 0xc000000002d3539c 19 0% 92% 0.00 55449 sched lock 35 0xc000000002d35384 19 0% 93% 0.00 58407 sched lock 25 0xc000000002d35384 18 0% 93% 0.00 27903 sched lock 45 0xc000000002d35384 17 0% 93% 0.00 96635 turnstile chain turnstile_trywait+0x208 17 0% 94% 0.00 15166 sched lock 30 (usermode) 15 0% 94% 0.00 45563 sched lock 45 0xc000000002d3539c 15 0% 94% 0.00 45743 sched lock 25 0xc000000002d3539c 15 0% 94% 0.00 15222 sched lock 0 0xc000000002d35384 13 0% 94% 0.00 26118 sched lock 55 0xc000000002d3539c 12 0% 95% 0.00 82576 sched lock 25 (usermode) 11 0% 95% 0.00 56223 turnstile chain turnstile_chain_lock+0xdc 11 0% 95% 0.00 11347 sched lock 40 0xc000000002d3539c 11 0% 95% 0.00 10283 sched lock 35 0xc000000002d3539c 10 0% 95% 0.00 10525 sched lock 70 0xc000000002d3539c 10 0% 95% 0.00 20867 sched lock 60 0xc000000002d3539c 10 0% 96% 0.00 13968 sched lock 15 0xc000000002d3539c 9 0% 96% 0.00 95086 turnstile chain (usermode) 9 0% 96% 0.00 24050 sched lock 20 0xc000000002d35384 8 0% 96% 0.00 49722 sched lock 40 0xc000000002d35384 8 0% 96% 0.00 37846 sched lock 35 (usermode) 8 0% 96% 0.00 56262 sched lock 10 (usermode) 7 0% 96% 0.00 15266 turnstile lock turnstile_trywait+0x150 7 0% 97% 0.00 15500 sched lock 65 0xc000000002d3539c 7 0% 97% 0.00 9542 sched lock 50 (usermode) 6 0% 97% 0.00 6991 sched lock 45 sched_switch+0x81c 6 0% 97% 0.00 14165 sched lock 45 (usermode) 6 0% 97% 0.00 13692 sched lock 20 (usermode) 5 0% 97% 0.00 30761 turnstile lock (usermode) 5 0% 97% 0.00 12516 sched lock 65 (usermode) 5 0% 97% 0.00 30411 sched lock 60 (usermode) 5 0% 97% 0.00 33716 sched lock 45 sched_switch+0xca0 5 0% 97% 0.00 9188 sched lock 20 0xc000000002d3539c 5 0% 97% 0.00 9150 sched lock 15 (usermode) 4 0% 97% 0.00 7722 sched lock 55 sched_switch+0xca0 4 0% 98% 0.00 7319 sched lock 55 sched_switch+0x81c 4 0% 98% 0.00 13663 sched lock 40 sched_switch+0x81c 4 0% 98% 0.00 46348 sched lock 30 sched_switch+0xca0 4 0% 98% 0.00 5664 sched lock 15 sched_switch+0x81c 4 0% 98% 0.00 4097 sched lock 10 sched_switch+0x81c 4 0% 98% 0.00 11174 sched lock 0 (usermode) 3 0% 98% 0.00 3959 sched lock 70 sched_switch+0x81c 3 0% 98% 0.00 17449 sched lock 66 0xc000000002d3539c 3 0% 98% 0.00 459 sched lock 66 sched_add+0x48c 3 0% 98% 0.00 14684 sched lock 55 (usermode) 3 0% 98% 0.00 88492 sched lock 50 0xc000000002d35384 3 0% 98% 0.00 9101 sched lock 5 sched_switch+0x81c 3 0% 98% 0.00 49739 sched lock 5 (usermode) 3 0% 98% 0.00 13632 sched lock 35 sched_switch+0x81c 3 0% 98% 0.00 7120 sched lock 30 sched_switch+0x81c 3 0% 98% 0.00 6224 sched lock 20 sched_switch+0x81c 3 0% 98% 0.00 12252 sched lock 18 sched_add+0x48c 3 0% 98% 0.00 18519 sched lock 15 sched_switch+0xca0 3 0% 98% 0.00 22551 sched lock 10 0xc000000002d3539c 2 0% 98% 0.00 1504 sched lock 70 sched_add+0x48c 2 0% 99% 0.00 61936 sched lock 70 (usermode) 2 0% 99% 0.00 33751 sched lock 65 sched_switch+0xca0 2 0% 99% 0.00 308 sched lock 62 sched_switch+0x81c 2 0% 99% 0.00 34002 sched lock 60 0xc000000002d35384 2 0% 99% 0.00 33768 sched lock 60 sched_switch+0xca0 2 0% 99% 0.00 9134 sched lock 60 sched_switch+0x81c 2 0% 99% 0.00 665 sched lock 57 sched_add+0x48c 2 0% 99% 0.00 425 sched lock 52 sched_add+0x48c 2 0% 99% 0.00 21231 sched lock 52 (usermode) 2 0% 99% 0.00 13004 sched lock 50 sched_switch+0x81c 2 0% 99% 0.00 426 sched lock 49 sched_add+0x48c 2 0% 99% 0.00 1457 sched lock 43 (usermode) 2 0% 99% 0.00 5632 sched lock 40 (usermode) 2 0% 99% 0.00 9015 sched lock 39 0xc000000002d3539c 2 0% 99% 0.00 9463 sched lock 38 0xc000000002d35384 2 0% 99% 0.00 723 sched lock 37 sched_add+0x48c 2 0% 99% 0.00 167273 sched lock 31 0xc000000002d35384 2 0% 99% 0.00 12954 sched lock 25 sched_switch+0x81c 2 0% 99% 0.00 839 sched lock 23 sched_add+0x48c 2 0% 99% 0.00 385 sched lock 19 sched_add+0x48c 2 0% 99% 0.00 382 et_hw_mtx cpu_idleclock+0x37c 2 0% 99% 0.00 4595 callout 0xc000000002d1381c 1 0% 99% 0.00 33330 turnstile lock turnstile_lookup+0xd4 1 0% 99% 0.00 873 sched lock 9 (usermode) 1 0% 99% 0.00 276 sched lock 71 sched_add+0x48c 1 0% 99% 0.00 1475 sched lock 67 sched_add+0x48c 1 0% 99% 0.00 256 sched lock 67 (usermode) 1 0% 99% 0.00 753 sched lock 68 0xc000000002d3539c 1 0% 99% 0.00 16951 sched lock 67 0xc000000002d3539c 1 0% 99% 0.00 1392 sched lock 69 0xc000000002d3539c 1 0% 99% 0.00 256 sched lock 61 sched_add+0x48c 1 0% 99% 0.00 295 sched lock 63 sched_add+0x48c 1 0% 99% 0.00 1922 sched lock 62 0xc000000002d3539c 1 0% 99% 0.00 8808 sched lock 66 (usermode) 1 0% 99% 0.00 554 sched lock 57 (usermode) 1 0% 99% 0.00 731 sched lock 55 sched_add+0x48c 1 0% 99% 0.00 238 sched lock 53 sched_add+0x48c 1 0% 99% 0.00 463 sched lock 50 sched_add+0x48c 1 0% 99% 0.00 33627 sched lock 52 0xc000000002d35384 1 0% 99% 0.00 257 sched lock 43 sched_add+0x48c 1 0% 99% 0.00 195 sched lock 43 0xc000000002d3539c 1 0% 99% 0.00 1520 sched lock 46 sched_add+0x48c 1 0% 100% 0.00 1505 sched lock 47 sched_add+0x48c 1 0% 100% 0.00 263 sched lock 5 0xc000000002d3539c 1 0% 100% 0.00 201 sched lock 36 sched_add+0x48c 1 0% 100% 0.00 8891 sched lock 36 (usermode) 1 0% 100% 0.00 331105 sched lock 35 sched_switch+0xca0 1 0% 100% 0.00 543 sched lock 30 sched_add+0x48c 1 0% 100% 0.00 383 sched lock 31 0xc000000002d3539c 1 0% 100% 0.00 391 sched lock 39 sched_add+0x48c 1 0% 100% 0.00 2070 sched lock 38 (usermode) 1 0% 100% 0.00 33795 sched lock 36 0xc000000002d3539c 1 0% 100% 0.00 4568 sched lock 37 (usermode) 1 0% 100% 0.00 1476 sched lock 40 sched_add+0x48c 1 0% 100% 0.00 461 sched lock 4 (usermode) 1 0% 100% 0.00 1506 sched lock 42 sched_add+0x48c 1 0% 100% 0.00 478 sched lock 1 sched_add+0x48c 1 0% 100% 0.00 4638 sched lock 0 sched_switch+0x81c 1 0% 100% 0.00 1252 sched lock 0 sched_add+0x48c 1 0% 100% 0.00 445 callout (usermode) 1 0% 100% 0.00 275 callout 0xc000000002d12b30 1 0% 100% 0.00 375 et_hw_mtx cpu_new_callout+0x370 1 0% 100% 0.00 338 sched lock 20 sched_add+0x48c 1 0% 100% 0.00 194 sched lock 2 sched_add+0x48c 1 0% 100% 0.00 2162 sched lock 16 sched_add+0x48c 1 0% 100% 0.00 4560 sched lock 14 sched_add+0x48c 1 0% 100% 0.00 8782 sched lock 15 sched_add+0x48c 1 0% 100% 0.00 1478 sched lock 23 0xc000000002d35384 1 0% 100% 0.00 17184 sched lock 22 (usermode) 1 0% 100% 0.00 320 sched lock 22 sched_add+0x48c 1 0% 100% 0.00 17047 sched lock 21 sched_add+0x48c 1 0% 100% 0.00 8797 sched lock 25 sched_add+0x48c 1 0% 100% 0.00 53988 sched lock 27 (usermode) 1 0% 100% 0.00 17197 sched lock 3 sched_add+0x48c ------------------------------------------------------------------------------- Thread lock spin: 928 events in 139.370 seconds (7 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 416 45% 45% 0.00 6990 sleepq chain _thread_lock+0xf4 57 6% 51% 0.00 8732 sleepq chain (usermode) 36 4% 55% 0.00 6600 sched lock 45 _thread_lock+0xf4 30 3% 58% 0.00 16404 sched lock 10 _thread_lock+0xf4 29 3% 61% 0.00 13758 sched lock 30 _thread_lock+0xf4 29 3% 64% 0.00 9758 sched lock 15 _thread_lock+0xf4 24 3% 67% 0.00 7428 sched lock 20 _thread_lock+0xf4 21 2% 69% 0.00 11917 sched lock 5 _thread_lock+0xf4 21 2% 71% 0.00 7339 sched lock 35 _thread_lock+0xf4 19 2% 73% 0.00 5676 sched lock 40 _thread_lock+0xf4 18 2% 75% 0.00 6047 sched lock 60 _thread_lock+0xf4 17 2% 77% 0.00 7232 sched lock 70 _thread_lock+0xf4 17 2% 79% 0.00 8367 sched lock 65 _thread_lock+0xf4 16 2% 81% 0.00 9247 sched lock 50 _thread_lock+0xf4 14 2% 82% 0.00 8406 sched lock 55 _thread_lock+0xf4 12 1% 84% 0.00 8706 sched lock 25 _thread_lock+0xf4 11 1% 85% 0.00 10880 sched lock 0 _thread_lock+0xf4 6 1% 85% 0.00 29456 sched lock 40 (usermode) 6 1% 86% 0.00 17262 sched lock 14 _thread_lock+0xf4 5 1% 87% 0.00 10785 sched lock 66 (usermode) 5 1% 87% 0.00 4328 sched lock 5 (usermode) 5 1% 88% 0.00 3578 sched lock 15 (usermode) 4 0% 88% 0.00 7804 turnstile lock _thread_lock+0xf4 4 0% 89% 0.00 25450 sched lock 9 _thread_lock+0xf4 4 0% 89% 0.00 25746 sched lock 68 _thread_lock+0xf4 4 0% 89% 0.00 52596 sched lock 50 (usermode) 4 0% 90% 0.00 20469 sched lock 30 (usermode) 3 0% 90% 0.00 8970 sched lock 71 (usermode) 3 0% 91% 0.00 6523 sched lock 63 (usermode) 3 0% 91% 0.00 1928 sched lock 60 (usermode) 3 0% 91% 0.00 7240 sched lock 36 _thread_lock+0xf4 3 0% 91% 0.00 1671 sched lock 34 (usermode) 3 0% 92% 0.00 1688 sched lock 29 (usermode) 3 0% 92% 0.00 897 sched lock 10 (usermode) 2 0% 92% 0.00 1508 turnstile lock (usermode) 2 0% 93% 0.00 630 sched lock 9 (usermode) 2 0% 93% 0.00 33737 sched lock 71 _thread_lock+0xf4 2 0% 93% 0.00 9460 sched lock 70 (usermode) 2 0% 93% 0.00 1701 sched lock 68 (usermode) 2 0% 93% 0.00 17320 sched lock 67 (usermode) 2 0% 94% 0.00 6041 sched lock 66 _thread_lock+0xf4 2 0% 94% 0.00 14002 sched lock 62 (usermode) 2 0% 94% 0.00 3604 sched lock 57 (usermode) 2 0% 94% 0.00 596 sched lock 54 (usermode) 2 0% 95% 0.00 17170 sched lock 47 _thread_lock+0xf4 2 0% 95% 0.00 25200 sched lock 43 _thread_lock+0xf4 2 0% 95% 0.00 4654 sched lock 37 (usermode) 2 0% 95% 0.00 9171 sched lock 35 (usermode) 2 0% 95% 0.00 1120 sched lock 27 (usermode) 2 0% 96% 0.00 135203 sched lock 25 (usermode) 2 0% 96% 0.00 17452 sched lock 22 (usermode) 2 0% 96% 0.00 17261 sched lock 21 _thread_lock+0xf4 2 0% 96% 0.00 1679 sched lock 16 (usermode) 1 0% 96% 0.00 34000 sched lock 8 _thread_lock+0xf4 1 0% 96% 0.00 17264 sched lock 65 (usermode) 1 0% 97% 0.00 1658 sched lock 64 (usermode) 1 0% 97% 0.00 480 sched lock 56 _thread_lock+0xf4 1 0% 97% 0.00 648 sched lock 56 (usermode) 1 0% 97% 0.00 8997 sched lock 55 (usermode) 1 0% 97% 0.00 8983 sched lock 52 (usermode) 1 0% 97% 0.00 528 sched lock 52 _thread_lock+0xf4 1 0% 97% 0.00 1623 sched lock 6 (usermode) 1 0% 97% 0.00 1256 sched lock 59 (usermode) 1 0% 97% 0.00 654 sched lock 63 _thread_lock+0xf4 1 0% 98% 0.00 17297 sched lock 39 (usermode) 1 0% 98% 0.00 67121 sched lock 38 _thread_lock+0xf4 1 0% 98% 0.00 351 sched lock 37 _thread_lock+0xf4 1 0% 98% 0.00 414 sched lock 38 (usermode) 1 0% 98% 0.00 453 sched lock 31 (usermode) 1 0% 98% 0.00 2525 sched lock 32 (usermode) 1 0% 98% 0.00 32854 sched lock 31 _thread_lock+0xf4 1 0% 98% 0.00 1577 sched lock 48 _thread_lock+0xf4 1 0% 98% 0.00 733 sched lock 43 (usermode) 1 0% 98% 0.00 1722 sched lock 49 (usermode) 1 0% 99% 0.00 3510 sched lock 49 _thread_lock+0xf4 1 0% 99% 0.00 1586 sched lock 51 (usermode) 1 0% 99% 0.00 449 sched lock 51 _thread_lock+0xf4 1 0% 99% 0.00 1621 sched lock 14 (usermode) 1 0% 99% 0.00 33815 sched lock 1 (usermode) 1 0% 99% 0.00 1650 sched lock 12 (usermode) 1 0% 99% 0.00 17264 sched lock 0 (usermode) 1 0% 99% 0.00 728 sched lock 21 (usermode) 1 0% 99% 0.00 1002 sched lock 18 _thread_lock+0xf4 1 0% 100% 0.00 2682 sched lock 20 (usermode) 1 0% 100% 0.00 17124 sched lock 24 _thread_lock+0xf4 1 0% 100% 0.00 463 sched lock 22 _thread_lock+0xf4 1 0% 100% 0.00 2665 sched lock 24 (usermode) 1 0% 100% 0.00 426 sched lock 3 _thread_lock+0xf4 ------------------------------------------------------------------------------- R/W writer blocked by writer: 9 events in 139.370 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 4 44% 44% 0.00 290027 kernel vm object kmem_back_domain+0x340 2 22% 67% 0.00 289173 kernel vm object (usermode) 1 11% 78% 0.00 78280 vm object 0xc000000003119e88 1 11% 89% 0.00 748968 vm object vm_object_shadow+0x378 1 11% 100% 0.00 200885 kernel vm object 0xc0000000030eac08 ------------------------------------------------------------------------------- R/W writer blocked by readers: 5 events in 139.370 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2 40% 40% 0.00 168385 vm object 0xc0000000030f2c88 2 40% 80% 0.00 685360 vm object (usermode) 1 20% 100% 0.00 1011654 vm object vm_object_deallocate+0x800 ------------------------------------------------------------------------------- R/W writer spin on writer: 732 events in 139.370 seconds (5 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 227 31% 31% 0.00 697100 kernel vm object kmem_back_domain+0x340 214 29% 60% 0.00 621214 kernel vm object 0xc0000000030eac08 193 26% 87% 0.00 1611929 kernel vm object (usermode) 29 4% 91% 0.00 19699 vm object vm_fault_hold+0x1b28 21 3% 93% 0.00 37518 vm object (usermode) 9 1% 95% 0.00 106987 vm object 0xc000000003119e88 7 1% 96% 0.00 73265 vm object vm_object_shadow+0x378 5 1% 96% 0.00 45347 vm object vm_object_deallocate+0x800 5 1% 97% 0.00 75735 vm object 0xc0000000030f2c88 4 1% 98% 0.00 4049 vm object vm_object_collapse+0x710 4 1% 98% 0.00 3818 tcpinp (usermode) 3 0% 98% 0.00 33826 vm object vm_object_reference+0x134 3 0% 99% 0.00 44015 vm object vm_map_insert+0x758 2 0% 99% 0.00 37332 bufobj interlock bufobj_wdrop+0x13c 1 0% 99% 0.00 402 vm object vm_object_deallocate+0x814 1 0% 99% 0.00 2543 vm object vfs_busy_pages+0x2a8 1 0% 100% 0.00 34861 vm object vm_fault_hold+0x2700 1 0% 100% 0.00 385 bufobj interlock 0xc0000000030b3c04 1 0% 100% 0.00 926 tcpinp 0xc000000002eb6814 1 0% 100% 0.00 918 bufobj interlock (usermode) ------------------------------------------------------------------------------- R/W writer spin on readers: 154 events in 139.370 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 52 34% 34% 0.00 30820 vm object (usermode) 36 23% 57% 0.00 36809 vm object vmspace_fork+0x2d4 23 15% 72% 0.00 24525 vm object vm_fault_hold+0x1b28 12 8% 80% 0.00 36879 vm object vm_object_collapse+0x710 11 7% 87% 0.00 78822 vm object 0xc0000000030f2c88 6 4% 91% 0.00 26748 vm object vm_fault_hold+0x2700 5 3% 94% 0.00 18173 vm object vm_object_shadow+0x378 5 3% 97% 0.00 46480 vm object vm_object_deallocate+0x800 4 3% 100% 0.00 40490 vm object 0xc000000003119e88 ------------------------------------------------------------------------------- R/W reader spin on writer: 76 events in 139.370 seconds (1 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 58 76% 76% 0.00 26198 vm object __rw_rlock_int+0x9c 18 24% 100% 0.00 32967 vm object (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 1 events in 139.370 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 1 100% 100% 0.00 36907 vm object __rw_rlock_int+0x9c ------------------------------------------------------------------------------- SX exclusive block: 48 events in 139.370 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 21 44% 44% 0.00 58088 vq->vq_lock (usermode) 11 23% 67% 0.00 46178 vq->vq_lock vdev_queue_io+0x268 6 12% 79% 0.00 45111 dn->dn_struct_rwlock 0xc0000000025a6044 4 8% 88% 0.00 50941 dn->dn_struct_rwlock (usermode) 2 4% 92% 0.00 49667 spa->spa_alloc_locks[i] (usermode) 1 2% 94% 0.00 30666 vq->vq_lock 0xc000000002655fb4 1 2% 96% 0.00 57280 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 1 2% 98% 0.00 42243 scl->scl_lock spa_config_enter+0x140 1 2% 100% 0.00 41360 mc->mc_lock metaslab_class_throttle_unreserve+0x154 ------------------------------------------------------------------------------- SX exclusive spin: 19831 events in 139.370 seconds (142 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 4292 22% 22% 0.00 90111 tm->tm_lock trim_map_free+0x19c 2060 10% 32% 0.00 128916 vq->vq_lock (usermode) 2001 10% 42% 0.00 111977 vq->vq_lock vdev_queue_io+0x268 1342 7% 49% 0.00 29970 zio->io_lock (usermode) 1294 7% 55% 0.00 27346 zio->io_lock 0xc0000000026bf498 839 4% 60% 0.00 22743 zio->io_lock zio_add_child+0x274 784 4% 64% 0.00 48689 ms->ms_lock metaslab_alloc_dva+0x1138 709 4% 67% 0.00 33225 zio->io_lock 0xc0000000026c1474 615 3% 70% 0.00 136728 tm->tm_lock (usermode) 569 3% 73% 0.00 62804 spa->spa_alloc_locks[i] (usermode) 560 3% 76% 0.00 58095 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 485 2% 78% 0.00 62051 spa->spa_alloc_locks[i] 0xc0000000026bcf70 462 2% 81% 0.00 23662 vq->vq_lock vdev_queue_io_done+0x278 457 2% 83% 0.00 40450 mg->mg_lock metaslab_alloc_dva+0x3a8 427 2% 85% 0.00 62506 mg->mg_lock metaslab_alloc_dva+0x7d4 401 2% 87% 0.00 29824 vq->vq_lock vdev_queue_io_done+0x12c 391 2% 89% 0.00 12153 mc->mc_lock metaslab_class_throttle_unreserve+0x154 372 2% 91% 0.00 58652 mg->mg_lock (usermode) 275 1% 92% 0.00 53700 ms->ms_lock (usermode) 177 1% 93% 0.00 32509 scl->scl_lock spa_config_enter+0x140 174 1% 94% 0.00 19895 os->os_userused_lock 0xc0000000025b7700 164 1% 95% 0.00 23905 os->os_userused_lock 0xc0000000025b7898 155 1% 96% 0.00 26579 scl->scl_lock spa_config_exit+0x84 142 1% 97% 0.00 129548 vq->vq_lock 0xc000000002655fb4 98 0% 97% 0.00 18351 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 95 0% 98% 0.00 21552 os->os_userused_lock (usermode) 69 0% 98% 0.00 13351 mc->mc_lock (usermode) 61 0% 98% 0.00 3694 ms->ms_lock metaslab_free_concrete+0x2e8 61 0% 98% 0.00 12377 dp->dp_lock dsl_pool_undirty_space+0x18c 54 0% 99% 0.00 10125 dn->dn_struct_rwlock 0xc0000000025a6044 44 0% 99% 0.00 10436 db->db_mtx dmu_buf_will_dirty+0x2e8 27 0% 99% 0.00 11259 dd->dd_lock dsl_dir_diduse_space+0x270 25 0% 99% 0.00 9351 db->db_mtx (usermode) 20 0% 99% 0.00 14473 dd->dd_lock (usermode) 18 0% 99% 0.00 8080 dp->dp_lock (usermode) 11 0% 99% 0.00 88178 dn->dn_struct_rwlock (usermode) 10 0% 100% 0.00 2274 scl->scl_lock (usermode) 8 0% 100% 0.00 12963 as->as_lock aggsum_add+0x360 7 0% 100% 0.00 4813 zio->io_lock 0xc0000000026bda18 7 0% 100% 0.00 24406 ds->ds_lock dsl_dataset_block_kill+0x5e4 6 0% 100% 0.00 1181 ds->ds_lock dsl_dataset_block_kill+0x614 6 0% 100% 0.00 30954 ds->ds_lock dsl_dataset_block_born+0x3d4 6 0% 100% 0.00 3072 dn->dn_mtx 0xc0000000025a605c 6 0% 100% 0.00 17837 dd->dd_lock dsl_dir_tempreserve_clear+0x88 5 0% 100% 0.00 1190 bpl->bpl_lock bplist_append+0x158 5 0% 100% 0.00 1471 bpl->bpl_lock (usermode) 4 0% 100% 0.00 249 vd->vdev_stat_lock vdev_stat_update+0x348 3 0% 100% 0.00 1080 zio->io_lock 0xc0000000026bfc60 3 0% 100% 0.00 17016 zio->io_lock 0xc0000000026be2dc 3 0% 100% 0.00 3463 tm->tm_lock trim_map_write_done+0x188 3 0% 100% 0.00 2296 dn->dn_mtx dnode_diduse_space+0x184 3 0% 100% 0.00 22885 dd->dd_lock dsl_dir_willuse_space+0x204 2 0% 100% 0.00 8504 zio->io_lock 0xc0000000026bda48 2 0% 100% 0.00 6789 zio->io_lock 0xc0000000026bb8e0 2 0% 100% 0.00 17187 dp->dp_lock dsl_pool_dirty_space+0x17c 2 0% 100% 0.00 17938 as->as_lock (usermode) 1 0% 100% 0.00 238 zio->io_lock 0xc0000000026bf808 1 0% 100% 0.00 346 tm->tm_lock trim_map_write_start+0x3f8 1 0% 100% 0.00 4861 proctree sys_setpgid+0x5c8 1 0% 100% 0.00 9018 dp->dp_lock dsl_pool_need_dirty_delay+0x194 1 0% 100% 0.00 257 dd->dd_lock 0xc0000000025e75ec 1 0% 100% 0.00 1562 dd->dd_lock dsl_dir_sync+0x148 1 0% 100% 0.00 394 dn->dn_mtx dbuf_dirty+0xd38 1 0% 100% 0.00 898 db->db_mtx dbuf_read+0xa44 ------------------------------------------------------------------------------- SX shared spin: 22 events in 139.370 seconds (0 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 22 100% 100% 0.00 21130 dn->dn_struct_rwlock _sx_slock_int+0xa0 ------------------------------------------------------------------------------- root@fledgling:/usr/src #