root@fledgling:~ # l -s 7 cc -o stacktest stacktest.c Adaptive mutex spin: 84893 events in 21.178 seconds (4009 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 25033 29% 29% 0.00 132194 page pv (usermode) nsec ------ Time Distribution ------ count Stack 128 | 14 pmap_remove_pages+0x80 256 | 50 vmspace_exit+0x100 512 | 74 exit1+0x954 1024 | 363 sys_sys_exit+0x28 2048 | 479 trap+0xb80 4096 | 573 powerpc_interrupt+0x1a4 8192 | 566 16384 |@ 1311 32768 |@@@@@ 4460 65536 |@@@@@@ 5203 131072 |@@@@ 4121 262144 |@@@@ 3848 524288 |@@@ 2744 1048576 |@ 1135 2097152 | 90 4194304 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 21707 26% 55% 0.00 94291 page pv moea64_remove_pages+0x214 nsec ------ Time Distribution ------ count Stack 128 | 12 pmap_remove_pages+0x80 256 | 57 vmspace_exit+0x100 512 | 70 exit1+0x954 1024 | 424 sys_sys_exit+0x28 2048 | 507 trap+0xb80 4096 | 634 powerpc_interrupt+0x1a4 8192 | 635 16384 |@ 1386 32768 |@@@@@@ 4652 65536 |@@@@@@@ 5166 131072 |@@@@ 3566 262144 |@@@ 2569 524288 |@@ 1543 1048576 | 445 2097152 | 41 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12940 15% 70% 0.00 79167 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 128 | 14 moea64_enter_object+0x9c 256 | 27 pmap_enter_object+0xb0 512 | 26 0xc0000000030ec130 1024 | 154 vm_map_insert+0x6a8 2048 | 185 vm_map_fixed+0x120 4096 | 181 0xc000000002c47a14 8192 | 155 16384 | 333 32768 |@@@@ 1949 65536 |@@@@@@@@@@@ 4813 131072 |@@@@@@@ 3203 262144 |@@ 1257 524288 |@ 535 1048576 | 101 2097152 | 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6444 8% 78% 0.00 112048 page pv (usermode) nsec ------ Time Distribution ------ count Stack 128 | 1 moea64_enter_object+0x9c 256 | 11 pmap_enter_object+0xb0 512 | 10 0xc0000000030ec130 1024 | 66 vm_map_insert+0x6a8 2048 | 67 vm_map_fixed+0x120 4096 | 69 0xc000000002c47a14 8192 | 51 16384 | 157 32768 |@@@ 844 65536 |@@@@@@@@@ 2117 131072 |@@@@@@@ 1542 262144 |@@@ 820 524288 |@@ 488 1048576 | 174 2097152 | 26 4194304 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4199 5% 83% 0.00 55185 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 128 | 2 pmap_enter+0xbc 256 | 56 vm_fault_hold+0x20f4 512 | 80 vm_fault+0x94 1024 |@ 192 trap+0x10ac 2048 |@@ 400 powerpc_interrupt+0x1a4 4096 |@@@ 428 8192 |@@@ 431 16384 |@@@ 477 32768 |@@@@ 623 65536 |@@@ 533 131072 |@@@ 477 262144 |@@ 299 524288 |@ 159 1048576 | 40 2097152 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1760 2% 85% 0.00 60602 dp_sync_taskq 0xc000000002d74f5c nsec ------ Time Distribution ------ count Stack 128 | 6 taskqueue_thread_loop+0x158 256 | 35 fork_exit+0xdc 512 | 30 0xc000000003185c5c 1024 | 50 2048 | 30 4096 |@ 80 8192 |@@@ 184 16384 |@@@@@ 311 32768 |@@@@@ 320 65536 |@@@ 224 131072 |@@@ 234 262144 |@@@ 176 524288 |@ 65 1048576 | 15 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1148 1% 86% 0.00 97756 page pv (usermode) nsec ------ Time Distribution ------ count Stack 256 | 12 pmap_enter+0xbc 512 | 10 vm_fault_hold+0x20f4 1024 |@ 45 vm_fault+0x94 2048 |@@@ 120 trap+0x10ac 4096 |@@@ 117 powerpc_interrupt+0x1a4 8192 |@@ 94 16384 |@@ 104 32768 |@@@ 147 65536 |@@@ 118 131072 |@@ 113 262144 |@@@ 124 524288 |@@ 103 1048576 | 34 2097152 | 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1064 1% 88% 0.00 81365 dp_sync_taskq 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 128 | 3 _sleep+0x290 256 | 11 taskqueue_thread_loop+0x1b4 512 | 19 fork_exit+0xdc 1024 | 33 0xc000000003185c5c 2048 |@ 49 4096 |@ 38 8192 |@@ 84 16384 |@@@@ 144 32768 |@@@ 139 65536 |@@@ 137 131072 |@@@@ 156 262144 |@@@@ 175 524288 |@ 67 1048576 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 989 1% 89% 0.00 104662 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 256 | 12 _sleep+0x290 512 | 14 taskqueue_thread_loop+0x1b4 1024 | 28 fork_exit+0xdc 2048 |@ 37 0xc000000003185c5c 4096 |@ 33 8192 |@@ 72 16384 |@@@ 109 32768 |@@@@ 134 65536 |@@@ 103 131072 |@@@@ 154 262144 |@@@@@ 167 524288 |@@@ 108 1048576 | 18 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 734 1% 90% 0.00 74915 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 128 | 2 moea64_enter_object+0x9c 256 | 7 pmap_enter_object+0xb0 512 | 4 0xc0000000030ec130 1024 |@@ 50 vm_map_insert+0x6a8 2048 |@@ 73 vm_map_fixed+0x120 4096 |@@@ 95 vm_mmap_object+0x3ac 8192 |@@ 65 16384 |@@ 53 32768 |@@ 54 65536 |@@@@ 107 131072 |@@@ 85 262144 |@@ 71 524288 |@@ 57 1048576 | 11 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 561 1% 90% 0.00 46656 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 128 | 3 taskq_dispatch+0xb4 256 | 10 dmu_objset_sync+0x2b8 512 | 16 dsl_pool_sync+0x450 1024 |@ 30 spa_sync+0xc08 2048 |@ 30 0xc000000002635b28 4096 | 18 fork_exit+0xdc 8192 |@ 21 16384 |@@@ 69 32768 |@@@@@ 95 65536 |@@@@@ 96 131072 |@@@@@@ 122 262144 |@@ 50 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 450 1% 91% 0.00 79274 kernelpmap moea64_enter+0x918 nsec ------ Time Distribution ------ count Stack 256 | 1 pmap_enter+0xbc 512 | 2 kmem_back_domain+0x200 1024 | 4 kmem_malloc_domainset+0xd0 2048 | 8 uma_large_malloc_domain+0x148 4096 | 11 uma_large_malloc+0x28 8192 | 14 malloc+0x1a0 16384 |@@ 36 32768 |@@@ 52 65536 |@@@@@@@@ 122 131072 |@@@@@@@@ 126 262144 |@@@ 53 524288 |@ 20 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 377 0% 91% 0.00 44113 dp_sync_taskq taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 256 | 9 taskq_dispatch+0xb4 512 | 8 dmu_objset_sync+0x2b8 1024 |@ 22 dsl_pool_sync+0x450 2048 |@ 16 spa_sync+0xc08 4096 |@ 14 0xc000000002635b28 8192 | 11 fork_exit+0xdc 16384 |@@@ 43 32768 |@@@@@@ 82 65536 |@@@@@ 66 131072 |@@@@@ 75 262144 |@@ 31 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 343 0% 92% 0.00 32850 vm page vm_page_change_lock+0x160 nsec ------ Time Distribution ------ count Stack 128 | 1 vm_object_terminate+0x2b4 256 | 10 vm_object_deallocate+0x728 512 | 7 0xc0000000030ea1fc 1024 |@ 16 _vm_map_unlock+0xf8 2048 |@@@ 39 vmspace_exit+0x194 4096 |@ 18 exit1+0x954 8192 |@@ 23 16384 |@@ 34 32768 |@@@@@@ 79 65536 |@@@@@@ 71 131072 |@@ 33 262144 | 11 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 315 0% 92% 0.00 67991 dp_sync_taskq taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 256 | 6 taskq_dispatch+0xb4 512 | 8 dmu_objset_sync+0x2b8 1024 | 5 dsl_dataset_sync+0x1f0 2048 |@ 14 dsl_pool_sync+0x11c 4096 | 5 spa_sync+0xc08 8192 | 7 0xc000000002635b28 16384 |@@ 28 32768 |@@@@@@ 67 65536 |@@@ 32 131072 |@@@@@@@@ 93 262144 |@@@@ 46 524288 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 291 0% 92% 0.00 56314 zio_write_intr 0xc000000002d74f5c nsec ------ Time Distribution ------ count Stack 256 |@ 10 taskqueue_thread_loop+0x158 512 | 4 fork_exit+0xdc 1024 |@ 10 0xc000000003185c5c 2048 |@ 10 4096 | 7 8192 |@@ 25 16384 |@@@@ 40 32768 |@@@@@@@ 68 65536 |@@@@ 39 131072 |@@@ 35 262144 |@@@ 31 524288 |@ 12 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 281 0% 93% 0.00 72515 zio_write_intr 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 128 | 2 _sleep+0x290 256 | 8 taskqueue_thread_loop+0x1b4 512 |@ 10 fork_exit+0xdc 1024 |@@ 19 0xc000000003185c5c 2048 | 4 4096 | 5 8192 |@ 12 16384 |@@ 20 32768 |@@@@@@ 58 65536 |@@@@ 42 131072 |@@@@@ 48 262144 |@@@ 34 524288 |@@ 19 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 277 0% 93% 0.00 24725 vm active pagequeue 0xc0000000031072a8 nsec ------ Time Distribution ------ count Stack 256 | 4 vm_page_free_prep+0x158 512 | 7 vm_page_free_toq+0x34 1024 |@@@ 29 vm_page_free+0x2c 2048 |@@@@ 45 vm_object_terminate+0x2f8 4096 |@@ 23 vm_object_deallocate+0x728 8192 |@ 11 0xc0000000030ea1fc 16384 |@@ 27 32768 |@@@@@ 55 65536 |@@@@@ 52 131072 |@@ 20 262144 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 256 0% 93% 0.00 65960 zio_write_intr (usermode) nsec ------ Time Distribution ------ count Stack 128 | 1 256 | 5 512 | 5 1024 | 7 2048 | 2 4096 |@@ 20 8192 |@@ 23 16384 |@@@@ 37 32768 |@@@@ 41 65536 |@@@ 31 131072 |@@@@ 41 262144 |@@@ 28 524288 |@ 15 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 256 0% 94% 0.00 50165 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 128 | 1 pmap_kextract+0x80 256 | 6 _bus_dmamap_load_buffer+0x104 512 | 2 bus_dmamap_load+0xb8 1024 |@ 9 0xc000000002a0c85c 2048 |@ 13 nvme_qpair_submit_request+0xa0 4096 |@@@ 33 nvme_ctrlr_submit_io_request+0x40 8192 |@@@ 32 16384 |@@@@ 38 32768 |@@@ 30 65536 |@@@ 26 131072 |@@@ 33 262144 |@@ 24 524288 | 8 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 254 0% 94% 0.00 51204 dp_sync_taskq taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 128 | 3 taskq_dispatch+0xb4 256 | 4 dmu_objset_do_userquota_updates+0x1ac 512 | 7 dsl_pool_sync+0x27c 1024 |@ 10 spa_sync+0xc08 2048 |@ 9 0xc000000002635b28 4096 | 3 fork_exit+0xdc 8192 |@ 9 16384 |@@@@ 36 32768 |@@@@@ 47 65536 |@@@@@ 46 131072 |@@@@@@ 52 262144 |@@@ 27 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 252 0% 94% 0.00 80504 zio_write_issue 0xc000000002d74f5c nsec ------ Time Distribution ------ count Stack 256 | 6 taskqueue_thread_loop+0x158 512 | 6 fork_exit+0xdc 1024 | 4 0xc000000003185c5c 2048 |@ 14 4096 |@@ 21 8192 |@@@@ 35 16384 |@@@ 29 32768 |@@@ 33 65536 |@@@ 26 131072 |@@@@ 35 262144 |@@@ 28 524288 |@ 11 1048576 | 1 2097152 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 247 0% 94% 0.00 93089 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 128 | 1 256 | 4 512 | 0 1024 | 6 2048 |@ 10 4096 |@ 10 8192 |@@@ 25 16384 |@@@ 29 32768 |@@@@ 38 65536 |@@@@ 36 131072 |@@@@ 34 262144 |@@ 20 524288 |@@@ 30 1048576 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 239 0% 95% 0.00 97229 zio_write_issue 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 128 | 1 _sleep+0x290 256 | 1 taskqueue_thread_loop+0x1b4 512 | 5 fork_exit+0xdc 1024 | 6 0xc000000003185c5c 2048 | 2 4096 | 7 8192 |@ 15 16384 |@@@@ 37 32768 |@@@@@ 42 65536 |@@@@ 37 131072 |@@@ 30 262144 |@@@@ 34 524288 |@ 14 1048576 | 7 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 203 0% 95% 0.00 136169 page pv (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@ 14 moea64_enter_object+0x9c 2048 |@@ 16 pmap_enter_object+0xb0 4096 |@@ 17 0xc0000000030ec130 8192 |@@ 16 vm_map_insert+0x6a8 16384 |@ 8 vm_map_fixed+0x120 32768 |@@ 17 vm_mmap_object+0x3ac 65536 |@@@ 25 131072 |@@@ 23 262144 |@@@@ 32 524288 |@@@ 23 1048576 |@ 12 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 199 0% 95% 0.00 133563 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 128 | 1 pmap_kextract+0x80 256 | 2 free+0x44 512 | 5 zfs_kmem_free+0x28 1024 | 3 zio_buf_free+0xac 2048 |@ 7 abd_free+0x1b4 4096 | 5 0xc0000000026befc0 8192 |@@@ 21 16384 |@@@ 24 32768 |@@@ 23 65536 |@ 11 131072 |@@@@ 27 262144 |@@@@@ 38 524288 |@@@ 21 1048576 |@ 11 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 179 0% 95% 0.00 88872 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 256 | 2 pmap_enter+0xbc 512 | 5 kmem_back_domain+0x200 1024 |@@ 16 kmem_malloc_domainset+0xd0 2048 |@ 10 uma_large_malloc_domain+0x148 4096 |@@ 15 uma_large_malloc+0x28 8192 |@@@@ 27 malloc+0x1a0 16384 |@ 7 32768 |@@@ 20 65536 |@@@ 20 131072 |@@ 14 262144 |@@@ 21 524288 |@@@ 19 1048576 | 2 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 166 0% 96% 0.00 72622 zio_write_intr (usermode) nsec ------ Time Distribution ------ count Stack 256 | 4 taskq_dispatch_ent+0x50 512 |@ 10 spa_taskq_dispatch_ent+0xb0 1024 |@@ 14 0xc0000000026c145c 2048 | 4 0xc0000000026bf6ac 4096 | 2 zio_execute+0x19c 8192 |@ 6 taskq_dispatch_ent+0xa4 16384 |@@@ 19 32768 |@@@@ 25 65536 |@@@@@ 28 131072 |@@@@ 24 262144 |@@@ 18 524288 |@ 11 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 164 0% 96% 0.00 20768 vm active pagequeue 0xc0000000031072a8 nsec ------ Time Distribution ------ count Stack 256 |@ 9 vm_fault_hold+0x2220 512 | 4 vm_fault+0x94 1024 |@@ 13 trap+0x10ac 2048 |@@@@@@@ 40 powerpc_interrupt+0x1a4 4096 |@@@ 20 8192 |@@@ 18 16384 |@@ 12 32768 |@@ 16 65536 |@@@ 20 131072 |@ 6 262144 | 5 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 158 0% 96% 0.00 55541 pmap moea64_enter+0x918 nsec ------ Time Distribution ------ count Stack 1024 | 5 pmap_enter+0xbc 2048 | 1 vm_fault_hold+0x20f4 4096 |@ 8 vm_fault+0x94 8192 | 5 trap+0x10ac 16384 |@@@ 21 powerpc_interrupt+0x1a4 32768 |@@@@@@@ 38 65536 |@@@@@@ 35 131072 |@@@ 20 262144 |@@@@ 23 524288 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 141 0% 96% 0.00 72132 page pv moea64_remove+0x22c nsec ------ Time Distribution ------ count Stack 256 | 1 pmap_remove+0x98 512 | 1 0xc0000000030e89c0 1024 |@@ 11 kmem_free+0x44 2048 |@@@@ 20 uma_large_free+0x30 4096 |@@ 11 free+0x90 8192 |@@ 10 zfs_kmem_free+0x28 16384 |@@ 12 32768 |@@@ 17 65536 |@@ 13 131072 |@@@ 16 262144 |@@@@ 20 524288 |@ 6 1048576 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 136 0% 96% 0.00 120384 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 pmap_kextract+0x80 512 | 1 free+0x44 1024 | 4 zfs_kmem_free+0x28 2048 | 1 zio_buf_free+0xac 4096 |@ 7 abd_free+0x1b4 8192 |@@ 12 0xc0000000026befc0 16384 |@@@@ 21 32768 |@@ 11 65536 |@@@ 14 131072 |@@@@ 22 262144 |@@@@ 20 524288 |@@@ 16 1048576 |@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 125 0% 96% 0.00 46332 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 256 |@ 5 taskq_dispatch+0xb4 512 | 3 dmu_objset_sync+0x2b8 1024 | 3 dsl_dataset_sync+0x1f0 2048 |@ 5 dsl_pool_sync+0x300 4096 | 2 spa_sync+0xc08 8192 | 2 0xc000000002635b28 16384 |@@@ 16 32768 |@@@@@@ 25 65536 |@@@@@@ 26 131072 |@@@@@@ 29 262144 |@@ 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 110 0% 97% 0.00 47630 zio_write_intr taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 256 |@ 5 taskq_dispatch_ent+0x50 512 |@ 4 spa_taskq_dispatch_ent+0xb0 1024 |@ 4 zio_delay_interrupt+0x244 2048 |@@ 8 0xc0000000026508a8 4096 |@ 4 biodone+0x104 8192 |@@ 8 g_io_deliver+0x348 16384 |@@@ 12 32768 |@@@@@@ 25 65536 |@@@@ 15 131072 |@@@ 14 262144 |@ 7 524288 |@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 107 0% 97% 0.00 24917 vm page vm_page_change_lock+0x160 nsec ------ Time Distribution ------ count Stack 128 | 2 vm_object_page_remove+0xfc 256 | 3 0xc0000000030f0a40 512 | 3 vm_map_delete+0x1bc 1024 |@ 7 vmspace_exit+0x184 2048 |@@ 10 exit1+0x954 4096 |@ 6 sys_sys_exit+0x28 8192 |@ 7 16384 |@@@@@ 21 32768 |@@@@@ 21 65536 |@@@@@ 19 131072 |@ 6 262144 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 100 0% 97% 0.00 56644 page pv moea64_enter+0x9a8 nsec ------ Time Distribution ------ count Stack 512 | 2 pmap_enter+0xbc 1024 |@@ 8 vm_fault_hold+0x20f4 2048 |@@ 8 vm_fault+0x94 4096 |@@@@ 15 trap+0x10ac 8192 |@@@ 10 powerpc_interrupt+0x1a4 16384 |@@@@ 14 32768 |@@@@ 14 65536 |@ 4 131072 |@@ 8 262144 |@@@ 10 524288 |@@ 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 95 0% 97% 0.00 31171 vm page vm_fault_hold+0x2724 nsec ------ Time Distribution ------ count Stack 256 |@ 6 vm_fault+0x94 512 |@ 4 trap+0x10ac 1024 |@ 6 powerpc_interrupt+0x1a4 2048 |@@@@@ 17 4096 |@@ 9 8192 |@@ 7 16384 |@@ 7 32768 |@@@ 10 65536 |@@@@ 13 131072 |@@@ 11 262144 |@ 4 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 87 0% 97% 0.00 50108 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 256 | 2 taskq_dispatch+0xb4 512 |@ 4 dmu_objset_do_userquota_updates+0x1ac 1024 |@ 4 dsl_pool_sync+0x27c 2048 |@ 5 spa_sync+0xc08 4096 | 2 0xc000000002635b28 8192 | 2 fork_exit+0xdc 16384 |@@ 8 32768 |@@@@@@ 20 65536 |@@@@@ 16 131072 |@@@@ 13 262144 |@@@ 11 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 82 0% 97% 0.00 92068 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 128 | 1 pmap_enter+0xbc 256 | 0 kmem_back_domain+0x200 512 | 1 kmem_malloc_domainset+0xd0 1024 | 1 uma_large_malloc_domain+0x148 2048 | 0 uma_large_malloc+0x28 4096 |@ 3 malloc+0x1a0 8192 |@ 5 16384 |@ 3 32768 |@@ 6 65536 |@@@@@@ 19 131072 |@@@@@@@@@ 25 262144 |@@@@ 13 524288 |@ 4 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 80 0% 97% 0.00 83544 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 512 |@ 3 moea64_enter_object+0x9c 1024 |@ 4 pmap_enter_object+0xb0 2048 |@@@ 10 0xc0000000030ec130 4096 |@@@ 9 vm_map_insert+0x6a8 8192 |@@ 6 vm_map_find+0x4f0 16384 |@@@ 9 vm_map_find_min+0x98 32768 |@@@@ 12 65536 |@ 4 131072 |@@@ 8 262144 |@@ 6 524288 |@@ 7 1048576 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 79 0% 97% 0.00 86697 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 512 | 1 taskq_dispatch_ent+0x50 1024 | 2 spa_taskq_dispatch_ent+0xb0 2048 | 2 0xc0000000026bb048 4096 |@ 3 zio_execute+0x19c 8192 | 1 zio_nowait+0x118 16384 |@@ 7 dmu_objset_sync+0x358 32768 |@@@@@@ 17 65536 |@@@@ 12 131072 |@@@@@ 14 262144 |@@@@@ 14 524288 |@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 72 0% 97% 0.00 103440 page pv (usermode) nsec ------ Time Distribution ------ count Stack 1024 | 1 pmap_remove+0x98 2048 |@@@ 8 0xc0000000030e89c0 4096 |@@@ 9 kmem_free+0x44 8192 |@@@@ 10 uma_large_free+0x30 16384 |@ 4 free+0x90 32768 |@ 4 zfs_kmem_free+0x28 65536 |@@ 5 131072 |@@@@@ 13 262144 |@@@@ 10 524288 |@@ 6 1048576 | 1 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 70 0% 97% 0.00 12443 vnode interlock vget+0x2f8 nsec ------ Time Distribution ------ count Stack 256 |@@@ 7 cache_lookup+0xc30 512 |@@ 6 vfs_cache_lookup+0xf8 1024 |@@@@@@ 16 VOP_LOOKUP_APV+0x98 2048 |@@@@@ 12 lookup+0x800 4096 | 2 namei+0x538 8192 | 0 kern_statat+0xb0 16384 |@ 4 32768 |@@@@@@ 15 65536 |@@@ 7 131072 | 0 262144 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 69 0% 98% 0.00 36350 vm active pagequeue (usermode) nsec ------ Time Distribution ------ count Stack 128 | 1 vm_page_free_prep+0x158 256 | 1 vm_page_free_toq+0x34 512 | 1 vm_page_free+0x2c 1024 | 2 vm_object_page_remove+0x33c 2048 |@@@@@@ 14 0xc0000000030f0a40 4096 |@@@@ 10 vm_map_delete+0x1bc 8192 | 1 16384 |@@ 6 32768 |@@@@ 11 65536 |@@@@@ 13 131072 |@@ 5 262144 |@ 3 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 66 0% 98% 0.00 43493 dp_sync_taskq taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 256 | 1 taskq_dispatch+0xb4 512 | 2 dmu_objset_sync+0x2b8 1024 |@ 4 dsl_dataset_sync+0x1f0 2048 | 2 dsl_pool_sync+0x300 4096 |@ 3 spa_sync+0xc08 8192 |@ 3 0xc000000002635b28 16384 |@@ 6 32768 |@@@@ 9 65536 |@@@@@@@@ 19 131072 |@@@@@ 12 262144 |@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 63 0% 98% 0.00 64516 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 512 | 1 pmap_kextract+0x80 1024 | 1 free+0x44 2048 | 1 zfs_kmem_free+0x28 4096 | 1 zio_buf_free+0xac 8192 |@@ 5 abd_free+0x1b4 16384 |@@ 5 0xc000000002656114 32768 |@@@ 7 65536 |@@@@@@@@ 18 131072 |@@@@@@@ 16 262144 |@@@ 7 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 60 0% 98% 0.00 1346605 process lock ast+0x7e0 nsec ------ Time Distribution ------ count Stack 131072 | 1 262144 |@ 2 524288 |@@@ 7 1048576 |@@@@@@@@@ 18 2097152 |@@@@@@@@@@ 21 4194304 |@@@@@ 11 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 58 0% 98% 0.00 11278 vnode interlock vget+0x2f8 nsec ------ Time Distribution ------ count Stack 256 |@@ 5 cache_lookup+0xc30 512 |@@@ 6 vfs_cache_lookup+0xf8 1024 |@@@@ 8 zfs_freebsd_aclcheck+0x74 2048 |@@@@@@ 13 VOP_LOOKUP_APV+0x98 4096 |@ 3 lookup+0x800 8192 |@ 2 namei+0x538 16384 |@@ 5 32768 |@@@@@ 10 65536 |@@ 4 131072 |@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 56 0% 98% 0.00 70546 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@ 2 pmap_kextract+0x80 2048 | 1 _bus_dmamap_load_buffer+0x104 4096 |@ 3 bus_dmamap_load+0xb8 8192 |@ 3 0xc000000002a0c85c 16384 |@@@@@ 10 nvme_qpair_submit_request+0xa0 32768 |@@@@ 8 nvme_ctrlr_submit_io_request+0x40 65536 |@@@ 7 131072 |@@@@@@ 12 262144 |@@@ 7 524288 |@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 55 0% 98% 0.00 27031 struct mount vlist mtx vfs_msync+0x1d4 nsec ------ Time Distribution ------ count Stack 256 |@@@ 6 0xc000000002e24904 512 |@ 2 VOP_FSYNC_APV+0x98 1024 |@ 2 0xc000000002e2222c 2048 |@@ 4 fork_exit+0xdc 4096 |@ 2 0xc000000003185c5c 8192 | 0 16384 | 0 32768 |@@@@@@@ 13 65536 |@@@@@@@@@@@@@ 25 131072 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 55 0% 98% 0.00 22797 struct mount mtx vfs_busy+0x434 nsec ------ Time Distribution ------ count Stack 128 | 1 lookup+0xa68 256 |@@@ 6 namei+0x538 512 |@@ 4 kern_statat+0xb0 1024 | 0 sys_fstatat+0x40 2048 |@ 3 trap+0xb80 4096 |@ 2 powerpc_interrupt+0x1a4 8192 |@ 2 16384 |@@@@ 9 32768 |@@@@@@ 11 65536 |@@@@@@ 12 131072 |@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 52 0% 98% 0.00 66887 pmap (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@ 3 pmap_enter+0xbc 4096 | 1 vm_fault_hold+0x20f4 8192 |@ 3 vm_fault+0x94 16384 |@@ 4 trap+0x10ac 32768 |@@@@@ 10 powerpc_interrupt+0x1a4 65536 |@@@@@@@@@ 17 131072 |@ 3 262144 |@@@@ 8 524288 |@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 52 0% 98% 0.00 100832 page pv moea64_remove_pages+0x214 nsec ------ Time Distribution ------ count Stack 512 | 1 pmap_remove_pages+0x80 1024 | 1 exec_new_vmspace+0x304 2048 | 1 0xc000000002c46738 4096 |@@@@ 7 kern_execve+0x678 8192 |@@@@@ 9 sys_execve+0x80 16384 |@@ 4 trap+0xb80 32768 |@@ 5 65536 |@ 2 131072 |@@@@ 8 262144 |@@@@@ 9 524288 |@ 3 1048576 |@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 52 0% 98% 0.00 55093 page pv moea64_remove+0x22c nsec ------ Time Distribution ------ count Stack 256 | 1 pmap_remove+0x98 512 | 1 vm_map_delete+0x194 1024 |@@ 5 kern_munmap+0xc4 2048 |@ 3 sys_munmap+0x2c 4096 |@@@@ 8 trap+0xb80 8192 |@@@@ 7 powerpc_interrupt+0x1a4 16384 |@@@@@ 9 32768 |@ 2 65536 |@@@ 6 131072 | 1 262144 |@@@ 6 524288 |@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 46 0% 98% 0.00 95926 page pv (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 pmap_enter+0xbc 512 | 0 kmem_back_domain+0x200 1024 |@ 2 kmem_malloc_domainset+0xd0 2048 |@ 2 uma_large_malloc_domain+0x148 4096 |@@@@ 7 uma_large_malloc+0x28 8192 |@@@ 6 malloc+0x1a0 16384 |@ 3 32768 |@@ 4 65536 |@@ 4 131072 |@ 3 262144 |@@@ 5 524288 |@@@@@ 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 43 0% 98% 0.00 143670 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 512 | 1 taskq_dispatch_ent+0x50 1024 | 0 spa_taskq_dispatch_ent+0xb0 2048 | 1 zio_allocate_dispatch+0x224 4096 |@@ 3 0xc0000000026becfc 8192 |@@@@ 6 zio_execute+0x19c 16384 |@@@@ 6 taskq_dispatch_ent+0xa4 32768 |@@@@ 6 65536 |@@@ 5 131072 |@ 2 262144 |@@@ 5 524288 |@@ 3 1048576 |@@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 43 0% 98% 0.00 23559 vm active pagequeue (usermode) nsec ------ Time Distribution ------ count Stack 128 |@ 2 vm_page_free_prep+0x158 256 |@ 2 vm_page_free_toq+0x34 512 |@ 2 vm_page_free+0x2c 1024 |@ 2 vm_object_terminate+0x2f8 2048 |@@ 4 vm_object_deallocate+0x728 4096 |@@ 3 0xc0000000030ea1fc 8192 |@ 2 16384 |@@@@@ 8 32768 |@@@@ 7 65536 |@@@@@@ 9 131072 | 1 262144 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 41 0% 98% 0.00 66095 zio_write_intr taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 256 |@@ 3 taskq_dispatch_ent+0x50 512 | 1 spa_taskq_dispatch_ent+0xb0 1024 |@@ 3 0xc0000000026c145c 2048 | 0 0xc0000000026bf6ac 4096 | 0 zio_execute+0x19c 8192 | 0 taskq_dispatch_ent+0xa4 16384 |@@@ 5 32768 |@@@@@ 7 65536 |@@@@@@ 9 131072 |@@@ 5 262144 |@@@@ 6 524288 |@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 37 0% 98% 0.00 92173 page pv 0xc00000000313beb4 nsec ------ Time Distribution ------ count Stack 256 |@ 2 moea64_ts_referenced+0x24 512 | 1 pmap_ts_referenced+0x80 1024 | 1 0xc00000000310d228 2048 |@@@@ 6 0xc00000000310bb78 4096 |@@@@ 5 fork_exit+0xdc 8192 |@@ 3 0xc000000003185c5c 16384 |@ 2 32768 |@@ 3 65536 |@@ 3 131072 |@@ 3 262144 |@ 2 524288 |@@@@ 5 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 34 0% 99% 0.00 60986 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 1024 | 1 pmap_enter+0xbc 2048 |@@@@ 5 vm_fault_hold+0x20f4 4096 |@@@@@@@ 8 vm_fault+0x94 8192 |@@@@ 5 vm_map_wire_locked+0x36c 16384 |@@@@ 5 vm_map_wire+0x5c 32768 |@@ 3 vslock+0x80 65536 |@ 2 131072 | 0 262144 |@ 2 524288 |@ 2 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 33 0% 99% 0.00 31173 struct mount mtx vfs_unbusy+0x15c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@ 7 lookup+0xb24 512 |@@ 3 namei+0x538 1024 | 1 kern_statat+0xb0 2048 | 0 sys_fstatat+0x40 4096 | 0 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 1 32768 |@@@@@ 6 65536 |@@@@@@@@@@ 11 131072 |@@ 3 262144 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 32 0% 99% 0.00 47551 zio_write_intr (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 taskq_dispatch_ent+0x50 512 | 0 spa_taskq_dispatch_ent+0xb0 1024 |@@ 3 zio_delay_interrupt+0x244 2048 | 1 0xc0000000026508a8 4096 | 1 biodone+0x104 8192 |@ 2 g_io_deliver+0x348 16384 |@@ 3 32768 |@@@@@@ 7 65536 |@@@@@ 6 131072 |@@@@@ 6 262144 | 1 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 31 0% 99% 0.00 135708 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 512 | 1 _sleep+0x290 1024 |@ 2 taskqueue_thread_loop+0x1b4 2048 | 0 fork_exit+0xdc 4096 |@ 2 0xc000000003185c5c 8192 |@@@ 4 16384 |@ 2 32768 |@@@ 4 65536 |@@@ 4 131072 |@ 2 262144 |@@ 3 524288 |@@@@ 5 1048576 |@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 31 0% 99% 0.00 15498 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@ 3 cache_lookup+0xc30 512 | 0 vfs_cache_lookup+0xf8 1024 |@@@ 4 VOP_LOOKUP_APV+0x98 2048 |@@@@ 5 lookup+0x800 4096 |@@@ 4 namei+0x538 8192 | 1 kern_statat+0xb0 16384 |@ 2 32768 |@@@@@@ 7 65536 |@@ 3 131072 |@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 31 0% 99% 0.00 110859 page pv (usermode) nsec ------ Time Distribution ------ count Stack 1024 | 1 pmap_remove_pages+0x80 2048 |@ 2 exec_new_vmspace+0x304 4096 |@@@@@@ 7 0xc000000002c46738 8192 | 0 kern_execve+0x678 16384 | 1 sys_execve+0x80 32768 |@ 2 trap+0xb80 65536 |@ 2 131072 |@@ 3 262144 |@@@@@@@@ 9 524288 |@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 30 0% 99% 0.00 185142 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 512 |@ 1 taskq_dispatch_ent+0x50 1024 |@ 1 spa_taskq_dispatch_ent+0xb0 2048 |@@ 2 0xc0000000026bb048 4096 |@@@@ 4 zio_execute+0x19c 8192 |@@ 2 zio_nowait+0x118 16384 |@@ 2 0xc0000000025a4230 32768 |@@@@@@ 6 65536 |@@ 2 131072 |@ 1 262144 |@@ 2 524288 |@@@@ 4 1048576 |@@ 2 2097152 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 29 0% 99% 0.00 44669 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 256 |@ 1 512 | 0 1024 |@ 1 2048 | 0 4096 |@@ 2 8192 |@@@@ 4 16384 |@@@@ 4 32768 |@@@@@@ 6 65536 |@@@ 3 131072 |@@@@ 4 262144 |@@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 29 0% 99% 0.00 123328 zio_write_intr (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@ 4 _sleep+0x290 2048 | 0 taskqueue_thread_loop+0x1b4 4096 | 0 fork_exit+0xdc 8192 |@@ 2 0xc000000003185c5c 16384 |@@ 2 32768 |@@@ 3 65536 |@ 1 131072 |@@@@@@@@ 8 262144 |@@@ 3 524288 |@@@@@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 28 0% 99% 0.00 775300 process lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@ 1 65536 | 0 131072 |@ 1 262144 |@@@@@ 5 524288 |@@@@@@ 6 1048576 |@@@@ 4 2097152 |@@@@@@@@@@ 10 4194304 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 28 0% 99% 0.00 154972 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 16384 |@ 1 pmap_kextract+0x80 32768 |@@ 2 _bus_dmamap_load_buffer+0x21c 65536 |@@@@@@@ 7 bus_dmamap_load+0xb8 131072 |@@@@@@@@@@ 10 0xc000000002a0c85c 262144 |@@@ 3 nvme_qpair_submit_request+0xa0 524288 |@@@@ 4 nvme_ctrlr_submit_io_request+0x40 1048576 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 27 0% 99% 0.00 891118 process lock kern_thr_exit+0x12c nsec ------ Time Distribution ------ count Stack 16384 |@ 1 thread_suspend_check+0x61c 32768 |@ 1 ast+0x60c 65536 | 0 131072 |@@@ 3 262144 |@@@@ 4 524288 |@@@@ 4 1048576 |@@@ 3 2097152 |@@@@@@@@ 8 4194304 |@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 25 0% 99% 0.00 43596 umtxql (usermode) nsec ------ Time Distribution ------ count Stack 128 |@ 1 0xc000000002d1d95c 256 |@@ 2 0xc000000002d19a70 512 | 0 sys__umtx_op+0x48 1024 | 0 trap+0xb80 2048 | 0 powerpc_interrupt+0x1a4 4096 | 0 8192 |@@@@@@ 5 16384 |@@@ 3 32768 |@@@@ 4 65536 |@@@@ 4 131072 |@ 1 262144 |@@@@@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 23 0% 99% 0.00 26614 vnode interlock 0xc000000002e1ab8c nsec ------ Time Distribution ------ count Stack 16384 |@ 1 lookup+0xa90 32768 |@@@@@@@@@@@@@@@@@@@@@@ 17 namei+0x538 65536 |@@@@@@ 5 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 22 0% 99% 0.00 16114 vm active pagequeue (usermode) nsec ------ Time Distribution ------ count Stack 128 |@ 1 vm_fault_hold+0x2220 256 | 0 vm_fault+0x94 512 |@ 1 trap+0x10ac 1024 |@@ 2 powerpc_interrupt+0x1a4 2048 |@@@@@ 4 4096 |@@@@ 3 8192 |@ 1 16384 |@@@@@ 4 32768 |@@ 2 65536 |@@@@ 3 131072 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 21 0% 99% 0.00 41098 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 512 |@ 1 taskq_dispatch_ent+0x50 1024 |@ 1 spa_taskq_dispatch_ent+0xb0 2048 | 0 0xc0000000026bb048 4096 |@ 1 zio_execute+0x19c 8192 | 0 zio_nowait+0x118 16384 |@@ 2 0xc0000000025a3c80 32768 |@@@@@@@@ 6 65536 |@@@@@ 4 131072 |@@@@@@@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 21 0% 99% 0.00 2419 struct mount vlist mtx _vhold+0x474 nsec ------ Time Distribution ------ count Stack 256 |@@@@@ 4 cache_lookup+0xba8 512 |@@@@ 3 vfs_cache_lookup+0xf8 1024 |@@@@@ 4 zfs_freebsd_aclcheck+0x74 2048 |@@@@@@@ 5 VOP_LOOKUP_APV+0x98 4096 |@ 1 lookup+0x800 8192 |@@ 2 namei+0x538 16384 |@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 20 0% 99% 0.00 44388 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 256 |@ 1 taskq_dispatch_ent+0x50 512 | 0 spa_taskq_dispatch_ent+0xb0 1024 | 0 0xc0000000026c145c 2048 |@ 1 0xc0000000026bd95c 4096 | 0 zio_execute+0x19c 8192 |@@@@@@@ 5 taskq_dispatch_ent+0xa4 16384 |@@@@@@ 4 32768 |@@@ 2 65536 |@@@@ 3 131072 |@@@ 2 262144 |@ 1 524288 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19 0% 99% 0.00 40072 umtxql 0xc000000002d1ccbc nsec ------ Time Distribution ------ count Stack 2048 |@@@ 2 0xc000000002d1d95c 4096 | 0 0xc000000002d19a70 8192 |@@@ 2 sys__umtx_op+0x48 16384 |@@@@ 3 trap+0xb80 32768 |@@@@@@ 4 powerpc_interrupt+0x1a4 65536 |@@@@@@@ 5 131072 |@ 1 262144 |@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19 0% 99% 0.00 194631 page pv (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@ 1 pmap_remove+0x98 4096 |@@@ 2 vm_map_delete+0x194 8192 |@ 1 kern_munmap+0xc4 16384 | 0 sys_munmap+0x2c 32768 |@ 1 trap+0xb80 65536 |@@@ 2 powerpc_interrupt+0x1a4 131072 |@@@@ 3 262144 |@@@@@@ 4 524288 |@@@ 2 1048576 |@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 18 0% 99% 0.00 33477 vm page (usermode) nsec ------ Time Distribution ------ count Stack 256 |@ 1 vm_object_terminate+0x2b4 512 | 0 vm_object_deallocate+0x728 1024 |@ 1 0xc0000000030ea1fc 2048 |@ 1 _vm_map_unlock+0xf8 4096 | 0 vmspace_exit+0x194 8192 |@@@@@ 3 exit1+0x954 16384 | 0 32768 |@@@@@@@@ 5 65536 |@@@@@@ 4 131072 |@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 16 0% 99% 0.00 4611 vnode interlock vget+0x2f8 nsec ------ Time Distribution ------ count Stack 256 |@ 1 cache_lookup+0xc30 512 |@ 1 vfs_cache_lookup+0xf8 1024 |@@@@@@@ 4 VOP_LOOKUP_APV+0x98 2048 |@@@@@@@@@ 5 lookup+0x800 4096 | 0 namei+0x538 8192 | 0 vn_open_cred+0x2f8 16384 |@@@@@@@ 4 32768 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 16 0% 99% 0.00 1212603 process lock (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@ 3 1048576 |@@@@@@@ 4 2097152 |@@@@@@@@@@@ 6 4194304 |@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 0% 99% 0.00 26207 umtxql 0xc000000002d1afb4 nsec ------ Time Distribution ------ count Stack 1024 |@@ 1 sys__umtx_op+0x48 2048 | 0 trap+0xb80 4096 |@@@@ 2 powerpc_interrupt+0x1a4 8192 |@@@@@@@@@@@@@@ 7 16384 |@@ 1 32768 |@@ 1 65536 |@@ 1 131072 |@@ 1 262144 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 0% 99% 0.00 524429 process lock kern_thr_exit+0x24c nsec ------ Time Distribution ------ count Stack 8192 |@@ 1 thread_suspend_check+0x61c 16384 | 0 ast+0x60c 32768 | 0 65536 | 0 131072 | 0 262144 |@@@@@@@@ 4 524288 |@@@@@@@@ 4 1048576 |@@@@@@ 3 2097152 |@@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 0% 99% 0.00 33900 page pv (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@ 2 moea64_enter_quick+0x2c 4096 | 0 pmap_enter_quick+0xa4 8192 |@@@@ 2 0xc0000000030e470c 16384 |@@@@@@ 3 vm_fault_hold+0x2150 32768 |@@ 1 vm_fault+0x94 65536 |@@@@@@@@ 4 trap+0x10ac 131072 |@@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 0% 99% 0.00 145114 page pv (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@ 4 moea64_enter_object+0x9c 2048 | 0 pmap_enter_object+0xb0 4096 |@@@@ 2 0xc0000000030ec130 8192 | 0 vm_map_insert+0x6a8 16384 |@@ 1 vm_map_find+0x4f0 32768 |@@@@ 2 vm_map_find_min+0x98 65536 | 0 131072 |@@@@ 2 262144 | 0 524288 |@@@@@@ 3 1048576 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 0% 99% 0.00 64985 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@ 1 taskq_dispatch_ent+0x50 512 | 0 spa_taskq_dispatch_ent+0xb0 1024 |@@ 1 0xc0000000026bb048 2048 | 0 zio_execute+0x19c 4096 | 0 zio_nowait+0x118 8192 |@@@@ 2 0xc0000000025a4230 16384 |@@@@@@ 3 32768 |@@ 1 65536 |@@ 1 131072 |@@@@@@ 3 262144 |@@ 1 524288 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 0% 99% 0.00 12492 pipe mutex 0xc000000002d8cc5c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 5 0xc000000002d87de0 512 |@@@@ 2 sys_write+0xf4 1024 |@@ 1 trap+0xb80 2048 | 0 powerpc_interrupt+0x1a4 4096 | 0 8192 | 0 16384 | 0 32768 |@@@@@@@@ 4 65536 |@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 0% 99% 0.00 205592 page pv (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@ 2 pmap_enter+0xbc 2048 |@@@@@@ 3 vm_fault_hold+0x20f4 4096 | 0 vm_fault+0x94 8192 | 0 trap+0x10ac 16384 |@@@@ 2 powerpc_interrupt+0x1a4 32768 |@@ 1 65536 |@@ 1 131072 |@@ 1 262144 |@@@@ 2 524288 | 0 1048576 |@@ 1 2097152 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 0% 99% 0.00 67982 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 512 |@@ 1 pmap_enter+0xbc 1024 |@@ 1 vm_fault_hold+0x20f4 2048 |@@@@ 2 vm_fault+0x94 4096 |@@ 1 trap+0x1194 8192 |@@ 1 powerpc_interrupt+0x1a4 16384 |@@@@@@ 3 memcpy+0xd4 32768 |@@@@ 2 65536 | 0 131072 | 0 262144 |@@ 1 524288 | 0 1048576 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 0% 99% 0.00 32329 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 512 |@@ 1 pmap_enter+0xbc 1024 | 0 vm_fault_hold+0x20f4 2048 |@@@@ 2 vm_fault+0x94 4096 | 0 trap+0x1194 8192 |@@ 1 powerpc_interrupt+0x1a4 16384 |@@@@ 2 memcpy+0xb8 32768 |@@@@ 2 65536 |@@@@@@ 3 131072 |@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 99% 0.00 38131 umtxql 0xc000000002d1dae0 nsec ------ Time Distribution ------ count Stack 256 |@@ 1 0xc000000002d19a70 512 | 0 sys__umtx_op+0x48 1024 | 0 trap+0xb80 2048 |@@ 1 powerpc_interrupt+0x1a4 4096 | 0 8192 | 0 16384 |@@ 1 32768 |@@@@@@@@@@ 4 65536 |@@@@@@@ 3 131072 |@@ 1 262144 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 99% 0.00 2922046 process lock (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@ 1 2097152 | 0 4194304 |@@@@@@@@@@@@@@@@@@@@@@@@@@@ 11 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 99% 0.00 21766 pipe mutex 0xc000000002d8d764 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@ 3 0xc000000002d87de0 512 | 0 sys_write+0xf4 1024 |@@ 1 trap+0xb80 2048 |@@ 1 powerpc_interrupt+0x1a4 4096 | 0 8192 | 0 16384 | 0 32768 |@@@@@ 2 65536 |@@@@@@@@@@@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 99% 0.00 27729 pipe mutex 0xc000000002d8c150 nsec ------ Time Distribution ------ count Stack 8192 |@@ 1 0xc000000002d873c8 16384 | 0 sys_read+0xf4 32768 |@@@@@@@@@@@@@@@@@ 7 trap+0xb80 65536 |@@@@@@@@@@ 4 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 99% 0.00 28357 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@ 1 lookup+0x1d0 32768 |@@@@@@@@@@@@@@@@ 6 namei+0x538 65536 |@@@@@@@@@@ 4 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 99% 0.00 30539 struct mount vlist mtx (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@ 1 0xc000000002e24904 512 |@@ 1 VOP_FSYNC_APV+0x98 1024 |@@ 1 0xc000000002e2222c 2048 | 0 fork_exit+0xdc 4096 | 0 0xc000000003185c5c 8192 | 0 16384 | 0 32768 |@@ 1 65536 |@@@@@@@@@@@@@@@@@@@ 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 99% 0.00 86526 kernelpmap moea64_remove+0x420 nsec ------ Time Distribution ------ count Stack 16384 |@@ 1 pmap_remove+0x98 32768 | 0 0xc0000000030e89c0 65536 |@@@@@@@@@@@@@ 5 kmem_free+0x44 131072 |@@@@@ 2 uma_large_free+0x30 262144 |@@@@@@@@ 3 free+0x90 zfs_kmem_free+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 99% 0.00 241382 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 2048 |@@@ 1 taskq_dispatch_ent+0x50 4096 | 0 spa_taskq_dispatch_ent+0xb0 8192 |@@@ 1 zio_allocate_dispatch+0x224 16384 |@@@ 1 0xc0000000026becfc 32768 | 0 zio_execute+0x19c 65536 | 0 taskq_dispatch_ent+0xa4 131072 |@@@ 1 262144 |@@@@@@ 2 524288 |@@@@@@@@@ 3 1048576 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 99% 0.00 32321 vm page (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@ 1 4096 | 0 8192 |@@@@@@@@@ 3 16384 |@@@ 1 32768 |@@@ 1 65536 |@@@@@@@@@ 3 131072 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 99% 0.00 42832 umtxql (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@ 1 4096 | 0 8192 | 0 16384 | 0 32768 |@@@@@@@@@ 3 65536 |@@@@@@@@@@@@@@@ 5 131072 | 0 262144 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 99% 0.00 34865 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@ 2 lookup+0xa68 512 |@@@ 1 namei+0x538 1024 | 0 kern_statat+0xb0 2048 |@@@ 1 sys_fstatat+0x40 4096 |@@@ 1 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 0 32768 |@@@ 1 65536 |@@@@@@ 2 131072 |@@@ 1 262144 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 99% 0.00 89480 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 256 |@@@ 1 moea64_enter_quick+0x2c 512 | 0 pmap_enter_quick+0xa4 1024 |@@@ 1 0xc0000000030e470c 2048 |@@@@@@@@@@@@ 4 vm_fault_hold+0x252c 4096 |@@@ 1 vm_fault+0x94 8192 |@@@ 1 trap+0x10ac 16384 | 0 32768 | 0 65536 | 0 131072 | 0 262144 | 0 524288 |@@@ 1 1048576 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 100% 0.00 108738 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@ 1 taskq_dispatch+0xb4 32768 |@@@ 1 dmu_objset_sync+0x2b8 65536 |@@@ 1 dsl_dataset_sync+0x1f0 131072 |@@@@@@ 2 dsl_pool_sync+0x11c 262144 |@@@@@@@@@@@@@@@ 5 spa_sync+0xc08 0xc000000002635b28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 100% 0.00 6903 vm page 0xc0000000030e442c nsec ------ Time Distribution ------ count Stack 512 |@@@ 1 vm_fault_hold+0x944 1024 | 0 vm_fault+0x94 2048 |@@@@@@@@@@@@@@@@ 5 trap+0x10ac 4096 |@@@ 1 powerpc_interrupt+0x1a4 8192 | 0 16384 |@@@ 1 32768 | 0 65536 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 100% 0.00 30523 umtxql (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@ 1 0xc000000002d19a70 4096 |@@@ 1 sys__umtx_op+0x48 8192 | 0 trap+0xb80 16384 |@@@@@@ 2 powerpc_interrupt+0x1a4 32768 |@@@ 1 65536 |@@@@@@@@@@ 3 131072 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 100% 0.00 143479 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 512 |@@@ 1 pmap_enter+0xbc 1024 |@@@ 1 vm_fault_hold+0x135c 2048 |@@@ 1 vm_fault+0x94 4096 | 0 trap+0x10ac 8192 | 0 powerpc_interrupt+0x1a4 16384 |@@@ 1 32768 |@@@@@@ 2 65536 | 0 131072 | 0 262144 |@@@ 1 524288 |@@@ 1 1048576 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 100% 0.00 32171 ttymtx ttyoutq_read_uio+0x34c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@ 2 ttydisc_getc_uio+0x20c 512 | 0 0xc000000002dafae8 1024 | 0 0xc000000002d873c8 2048 | 0 sys_read+0xf4 4096 | 0 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 0 32768 | 0 65536 |@@@@@@@@@@@@@@@@@@@@@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 100% 0.00 583 struct mount vlist mtx _vdrop+0x6f4 nsec ------ Time Distribution ------ count Stack 128 |@@@ 1 0xc000000002e1aa98 256 |@@@@@@@ 2 0xc000000002e338d0 512 |@@@@@@@@@@@ 3 0xc000000002e31db8 1024 | 0 _fdrop+0x4c 2048 |@@@@@@@ 2 closef+0x39c 0xc000000002c68bf4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 100% 0.00 86294 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 2048 |@@@ 1 moea64_enter_quick+0x2c 4096 | 0 pmap_enter_quick+0xa4 8192 | 0 0xc0000000030e470c 16384 | 0 vm_fault_hold+0x2150 32768 |@@@ 1 vm_fault+0x94 65536 |@@@@@@@@@@@ 3 trap+0x10ac 131072 |@@@@@@@ 2 262144 | 0 524288 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 100% 0.00 74378 page pv (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@ 1 moea64_enter_quick+0x2c 8192 |@@@ 1 pmap_enter_quick+0xa4 16384 |@@@ 1 0xc0000000030e470c 32768 | 0 vm_fault_hold+0x252c 65536 |@@@ 1 vm_fault+0x94 131072 |@@@@@@@ 2 trap+0x10ac 262144 |@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 100% 0.00 65733 dp_sync_taskq taskqueue_quiesce+0x2e8 nsec ------ Time Distribution ------ count Stack 2048 |@@@ 1 taskq_wait+0x24 4096 | 0 dmu_objset_sync+0x30c 8192 | 0 dsl_dataset_sync+0x1f0 16384 | 0 dsl_pool_sync+0x11c 32768 |@@@@@@@ 2 spa_sync+0xc08 65536 |@@@@@@@ 2 0xc000000002635b28 131072 |@@@@@@@ 2 262144 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 100% 0.00 22363 umtxql 0xc000000002d1db10 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@ 3 0xc000000002d19a70 512 |@@@@ 1 sys__umtx_op+0x48 1024 | 0 trap+0xb80 2048 | 0 powerpc_interrupt+0x1a4 4096 | 0 8192 | 0 16384 | 0 32768 |@@@@ 1 65536 |@@@@ 1 131072 |@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 100% 0.00 54246 struct mount vlist mtx (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@ 4 cache_lookup+0xba8 1024 |@@@@@@@@ 2 vfs_cache_lookup+0xf8 2048 | 0 zfs_freebsd_aclcheck+0x74 4096 | 0 VOP_LOOKUP_APV+0x98 8192 | 0 lookup+0x800 16384 | 0 namei+0x538 32768 | 0 65536 | 0 131072 | 0 262144 | 0 524288 |@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 100% 0.00 27477 struct mount mtx vfs_unbusy+0x15c nsec ------ Time Distribution ------ count Stack 256 |@@@@ 1 lookup+0xb24 512 | 0 namei+0x538 1024 | 0 vn_open_cred+0x2f8 2048 | 0 vn_open+0x30 4096 | 0 kern_openat+0x274 8192 | 0 sys_openat+0x58 16384 | 0 32768 |@@@@@@@@@@@@ 3 65536 |@@@@@@@@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 100% 0.00 873 UPVO entry uma_zfree_arg+0x5c8 nsec ------ Time Distribution ------ count Stack 256 |@@@@ 1 moea64_remove_pages+0x3a8 512 | 0 pmap_remove_pages+0x80 1024 |@@@@@@@@@@@@@@@@@@@@@ 5 vmspace_exit+0x100 2048 |@@@@ 1 exit1+0x954 sys_sys_exit+0x28 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 33648 vnode interlock 0xc000000002e1ab8c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 2 lookup+0xa90 65536 |@@@@@@@@@@@@@@@@@@@@ 4 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 7609 vm active pagequeue 0xc0000000031072a8 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@ 1 vm_page_free_prep+0x158 2048 |@@@@@ 1 vm_page_free_toq+0x34 4096 |@@@@@ 1 vm_page_free+0x2c 8192 |@@@@@ 1 vm_object_page_remove+0x33c 16384 |@@@@@ 1 0xc0000000030f0a40 32768 |@@@@@ 1 vm_map_delete+0x1bc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 24159 umtxql 0xc000000002d1b260 nsec ------ Time Distribution ------ count Stack 256 |@@@@@ 1 sys__umtx_op+0x48 512 | 0 trap+0xb80 1024 | 0 powerpc_interrupt+0x1a4 2048 | 0 4096 | 0 8192 | 0 16384 |@@@@@ 1 32768 |@@@@@@@@@@ 2 65536 |@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 589 struct mount vlist mtx (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 3 0xc000000002e1aa98 512 | 0 0xc000000002e338d0 1024 |@@@@@@@@@@ 2 0xc000000002e31db8 2048 |@@@@@ 1 _fdrop+0x4c closef+0x39c 0xc000000002c68bf4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 3478 struct mount mtx vfs_busy+0x434 nsec ------ Time Distribution ------ count Stack 256 |@@@@@ 1 lookup+0xa68 512 |@@@@@ 1 namei+0x538 1024 | 0 vn_open_cred+0x2f8 2048 |@@@@@ 1 vn_open+0x30 4096 |@@@@@@@@@@ 2 kern_openat+0x274 8192 | 0 sys_openat+0x58 16384 |@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 157671 page pv moea64_remove+0x22c nsec ------ Time Distribution ------ count Stack 1024 |@@@@@ 1 moea64_qremove+0x60 2048 |@@@@@@@@@@ 2 pmap_qremove+0x8c 4096 | 0 0xc0000000030e6a98 8192 | 0 thread_reap+0x19c 16384 |@@@@@ 1 thread_wait+0xe0 32768 | 0 proc_reap+0x9a0 65536 | 0 131072 | 0 262144 | 0 524288 |@@@@@ 1 1048576 |@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 57135 metaslab_group_taskq (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 2 _sleep+0x290 65536 |@@@@@@@@@@ 2 taskqueue_thread_loop+0x1b4 131072 |@@@@@ 1 fork_exit+0xdc 262144 |@@@@@ 1 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 77122 dp_sync_taskq taskqueue_quiesce+0x2e8 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@ 2 taskq_wait+0x24 32768 | 0 dmu_objset_sync+0x30c 65536 | 0 dsl_pool_sync+0x450 131072 |@@@@@@@@@@@@@@@ 3 spa_sync+0xc08 262144 |@@@@@ 1 0xc000000002635b28 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 100% 0.00 41444 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@ 2 taskq_wait+0x24 16384 | 0 dmu_objset_sync+0x30c 32768 |@@@@@ 1 dsl_pool_sync+0x450 65536 |@@@@@@@@@@ 2 spa_sync+0xc08 131072 |@@@@@ 1 0xc000000002635b28 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 34522 vnode interlock 0xc000000002e1ab8c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@ 2 kern_statat+0x168 65536 |@@@@@@@@@@@@@@@@@@ 3 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 8706 ttymtx (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@ 3 0xc000000002da9ef4 512 | 0 0xc000000002b69224 1024 | 0 0xc000000002d87de0 2048 |@@@@@@ 1 sys_write+0xf4 4096 | 0 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 0 32768 | 0 65536 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 1070 pipe mutex 0xc000000002d8e004 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@ 2 kern_poll+0x5a4 2048 |@@@@@@@@@@@@@@@@@@ 3 sys_poll+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 6041 pipe mutex 0xc000000002d8e004 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@ 2 kern_poll+0x44c 512 | 0 sys_poll+0x80 1024 |@@@@@@ 1 trap+0xb80 2048 | 0 powerpc_interrupt+0x1a4 4096 | 0 8192 |@@@@@@ 1 16384 | 0 32768 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 143147 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@ 1 pmap_enter+0xbc 2048 | 0 vm_fault_hold+0x20f4 4096 | 0 vm_fault+0x94 8192 | 0 trap+0x1194 16384 | 0 powerpc_interrupt+0x1a4 32768 |@@@@@@ 1 suword+0x90 65536 |@@@@@@ 1 131072 |@@@@@@ 1 262144 | 0 524288 | 0 1048576 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 193820 page pv (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@ 1 moea64_qremove+0x60 16384 |@@@@@@ 1 pmap_qremove+0x8c 32768 |@@@@@@ 1 0xc0000000030e6a98 65536 | 0 thread_reap+0x19c 131072 | 0 thread_wait+0xe0 262144 | 0 proc_reap+0x9a0 524288 |@@@@@@ 1 1048576 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 193317 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@ 1 pmap_kextract+0x80 131072 |@@@@@@ 1 free+0x44 262144 |@@@@@@@@@@@@ 2 zfs_kmem_free+0x28 524288 |@@@@@@ 1 zio_buf_free+0xac abd_free+0x1b4 0xc000000002656114 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 100% 0.00 1051 UPVO entry (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@ 3 moea64_remove_pages+0x3a8 2048 |@@@@@@@@@@@@ 2 pmap_remove_pages+0x80 vmspace_exit+0x100 exit1+0x954 sys_sys_exit+0x28 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 19632 vnode interlock 0xc0000000030c6708 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@ 1 VOP_GETATTR_APV+0x98 8192 | 0 vn_stat+0xb0 16384 |@@@@@@@ 1 kern_statat+0xdc 32768 |@@@@@@@ 1 sys_fstatat+0x40 65536 |@@@@@@@ 1 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 47905 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@ 1 lookup+0xa90 65536 |@@@@@@@@@@@@@@@ 2 namei+0x538 131072 |@@@@@@@ 1 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 22900 vm active pagequeue vm_page_dequeue+0x1b4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 2 vm_page_alloc_domain_after+0x28c 4096 | 0 vm_page_alloc+0xa4 8192 | 0 vm_fault_hold+0x1660 16384 | 0 vm_fault+0x94 32768 |@@@@@@@ 1 trap+0x10ac 65536 |@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 13357 vm active pagequeue vm_page_dequeue+0x1b4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@ 3 vm_page_deactivate+0x68 8192 | 0 0xc0000000030e4374 16384 | 0 vm_fault_hold+0x944 32768 | 0 vm_fault+0x94 65536 |@@@@@@@ 1 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 1466 select mtxpool 0xc000000002d8b728 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@ 1 kern_poll+0x51c 1024 |@@@@@@@ 1 sys_poll+0x80 2048 |@@@@@@@ 1 trap+0xb80 4096 |@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 661 select mtxpool selrecord+0x1dc nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@ 1 0xc000000002d8df68 512 |@@@@@@@ 1 kern_poll+0x44c 1024 |@@@@@@@ 1 sys_poll+0x80 2048 |@@@@@@@ 1 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 35896 page pv moea64_remove+0x22c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@ 1 pmap_remove+0x98 512 | 0 vm_map_delete+0x194 1024 |@@@@@@@ 1 vm_map_remove+0x88 2048 | 0 0xc000000002d8ef14 4096 | 0 0xc000000002d8e724 8192 | 0 _fdrop+0x4c 16384 | 0 32768 |@@@@@@@ 1 65536 | 0 131072 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 4941 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 2 pmap_enter+0xbc 8192 |@@@@@@@@@@@@@@@ 2 vm_fault_hold+0x20f4 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 memcpy+0x11c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 110646 page pv (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@ 1 pmap_enter+0xbc 4096 |@@@@@@@ 1 vm_fault_hold+0x135c 8192 | 0 vm_fault+0x94 16384 | 0 trap+0x10ac 32768 | 0 powerpc_interrupt+0x1a4 65536 | 0 131072 |@@@@@@@ 1 262144 | 0 524288 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 33944 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@ 1 pmap_kextract+0x80 8192 |@@@@@@@ 1 0xc00000000316650c 16384 | 0 0xc000000003165c18 32768 | 0 0xc000000002d12720 65536 |@@@@@@@ 1 softclock+0xdc 131072 |@@@@@@@ 1 0xc000000002c922c4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 100% 0.00 817 UPVO entry uma_zalloc_arg+0x98c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@ 1 moea64_enter+0xa4 1024 |@@@@@@@@@@@@@@@@@@@@@@ 3 moea64_enter_object+0x9c pmap_enter_object+0xb0 0xc0000000030ec130 vm_map_insert+0x6a8 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 34901 vnode interlock 0xc000000002e1ab8c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1 lookup+0x1d0 65536 |@@@@@@@@@@@@@@@@@@@@ 2 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 3984 vnode interlock 0xc000000002e02370 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@ 2 VOP_UNSET_TEXT_APV+0x98 512 | 0 vm_map_entry_set_vnode_text+0x100 1024 | 0 0xc0000000030ea1e4 2048 | 0 _vm_map_unlock+0xf8 4096 | 0 vmspace_exit+0x194 8192 | 0 exit1+0x954 16384 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 34644 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1 lookup+0xe88 65536 |@@@@@@@@@@@@@@@@@@@@ 2 namei+0x538 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 149116 vm page (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@ 1 vm_object_page_remove+0xfc 131072 |@@@@@@@@@@ 1 0xc0000000030f0a40 262144 | 0 vm_map_delete+0x1bc 524288 |@@@@@@@@@@ 1 vmspace_exit+0x184 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 29190 ttymtx ttydisc_write+0x388 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 1 0xc000000002da9ef4 512 | 0 0xc000000002b69224 1024 | 0 0xc000000002d87de0 2048 | 0 sys_write+0xf4 4096 | 0 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 0 32768 | 0 65536 |@@@@@@@@@@@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 8838 ttymtx 0xc000000002dafc0c nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@ 1 0xc000000002d873c8 2048 | 0 sys_read+0xf4 4096 |@@@@@@@@@@ 1 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 0 32768 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 180 ttymtx (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 3 0xc000000002b69224 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 7831 struct mount vlist mtx _vdrop+0x6f4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 1 0xc000000002e1aa98 1024 |@@@@@@@@@@ 1 namei+0x7d4 2048 | 0 kern_accessat+0x124 4096 | 0 sys_access+0x34 8192 | 0 trap+0xb80 16384 | 0 powerpc_interrupt+0x1a4 32768 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 40677 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 3 lookup+0xb24 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 27895 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 1 vn_start_write+0x160 1024 | 0 vn_close+0xd0 2048 | 0 0xc000000002e31db8 4096 | 0 _fdrop+0x4c 8192 | 0 closef+0x39c 16384 | 0 0xc000000002c68bf4 32768 | 0 65536 |@@@@@@@@@@@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 67573 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@ 1 pmap_kextract+0x80 8192 | 0 free+0x44 16384 | 0 zfs_kmem_free+0x28 32768 | 0 zio_buf_free+0xac 65536 | 0 abd_return_buf_copy+0x160 131072 |@@@@@@@@@@@@@@@@@@@@ 2 0xc0000000026bfd20 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 100% 0.00 199310 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@ 1 pmap_remove+0x98 131072 | 0 0xc0000000030e89c0 262144 |@@@@@@@@@@ 1 kmem_free+0x44 524288 |@@@@@@@@@@ 1 uma_large_free+0x30 free+0x90 zfs_kmem_free+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 5365 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 1024 | 0 spa_taskq_dispatch_ent+0xb0 2048 | 0 0xc0000000026bb048 4096 | 0 zio_execute+0x19c 8192 | 0 zio_nowait+0x118 16384 |@@@@@@@@@@@@@@@ 1 dmu_objset_sync+0x3cc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 20000 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 1024 | 0 spa_taskq_dispatch_ent+0xb0 2048 | 0 0xc0000000026bb048 4096 | 0 zio_execute+0x19c 8192 | 0 zio_nowait+0x118 16384 | 0 dmu_objset_sync+0x358 32768 | 0 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 1751 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 2048 | 0 spa_taskq_dispatch_ent+0xb0 4096 |@@@@@@@@@@@@@@@ 1 0xc0000000026bb048 zio_execute+0x19c zio_nowait+0x118 0xc0000000025a3c80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 178 zio_free_issue_2 taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 0xc0000000026bb048 zio_execute+0x19c zio_nowait+0x118 0xc000000002624e40 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 26271 vnode interlock 0xc0000000030c612c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 VOP_CLOSE_APV+0x98 1024 | 0 kern_execve+0x954 2048 | 0 sys_execve+0x80 4096 | 0 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 0 32768 | 0 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 25031 vnode interlock 0xc000000002e1ab8c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 lookup+0x1d0 namei+0x538 kern_readlinkat+0xac sys_readlink+0x38 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 41820 vnode interlock 0xc000000002e1ab8c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 lookup+0x1d0 namei+0x538 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 21193 vnode interlock 0xc000000002e1ab70 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 lookup+0xe88 namei+0x538 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 41934 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 lookup+0x1d0 namei+0x538 kern_accessat+0x124 sys_access+0x34 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 83507 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 lookup+0x1d0 131072 |@@@@@@@@@@@@@@@ 1 namei+0x538 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 161314 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 cache_lookup+0xba8 16384 | 0 vfs_cache_lookup+0xf8 32768 | 0 zfs_freebsd_aclcheck+0x74 65536 | 0 VOP_LOOKUP_APV+0x98 131072 | 0 lookup+0x800 262144 | 0 namei+0x538 524288 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 22291 vm page vm_imgact_map_page+0x43c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 0xc000000002c4733c 1024 | 0 0xc000000002c4684c 2048 | 0 kern_execve+0x678 4096 | 0 sys_execve+0x80 8192 | 0 trap+0xb80 16384 | 0 powerpc_interrupt+0x1a4 32768 | 0 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 1718 vm page (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 kern_execve+0x8e4 1024 | 0 sys_execve+0x80 2048 | 0 trap+0xb80 4096 |@@@@@@@@@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 11893 vm active pagequeue (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 4096 | 0 8192 | 0 16384 | 0 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 44384 umtxql 0xc000000002d1c97c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 0xc000000002d19938 sys__umtx_op+0x48 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 11379 umtxql 0xc000000002d1c7f0 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 0xc000000002d19938 512 | 0 sys__umtx_op+0x48 1024 | 0 trap+0xb80 2048 | 0 powerpc_interrupt+0x1a4 4096 | 0 8192 | 0 16384 | 0 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 14130 umtxql kern_umtx_wake+0x324 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 0xc000000002d1aaec 16384 | 0 sys__umtx_op+0x48 32768 |@@@@@@@@@@@@@@@ 1 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 67142 umtxql (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 0xc000000002d19a70 131072 |@@@@@@@@@@@@@@@ 1 sys__umtx_op+0x48 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 8712 ttymtx 0xc000000002db0ba0 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 kern_select+0xb14 16384 |@@@@@@@@@@@@@@@ 1 sys_select+0x84 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 8122 ttymtx 0xc000000002da9fc4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 0xc000000002b69224 16384 |@@@@@@@@@@@@@@@ 1 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 22360 struct mount vlist mtx _vdrop+0x6f4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 0xc000000002e1aa98 kern_statat+0x168 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 3019 struct mount mtx vfs_busy+0x434 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 lookup+0xa68 512 | 0 namei+0x538 1024 | 0 kern_accessat+0x124 2048 | 0 sys_access+0x34 4096 | 0 trap+0xb80 8192 |@@@@@@@@@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 237 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 lookup+0xb24 512 |@@@@@@@@@@@@@@@ 1 namei+0x538 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 1307 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 vop_stdgetwritemount+0x48 VOP_GETWRITEMOUNT_APV+0x98 vn_start_write+0x68 vn_close+0xd0 0xc000000002e31db8 _fdrop+0x4c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 2426 page pv (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 pmap_remove+0x98 4096 |@@@@@@@@@@@@@@@ 1 vm_map_delete+0x194 vm_map_remove+0x88 0xc000000002d8ef14 0xc000000002d8e724 _fdrop+0x4c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 15649 page pv (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 pmap_enter+0xbc 16384 | 0 vm_fault_hold+0x20f4 32768 |@@@@@@@@@@@@@@@ 1 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 memcpy+0xd4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 149477 nvme qpair lock 0xc000000002a0bae4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 nvme_qpair_process_completions+0x134 65536 | 0 0xc000000002a0bff0 131072 | 0 0xc000000002c922c4 262144 | 0 fork_exit+0xdc 524288 |@@@@@@@@@@@@@@@ 1 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 22545 metaslab_group_taskq taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 taskq_dispatch+0xb4 2048 | 0 metaslab_sync_reassess+0x2d8 4096 | 0 vdev_sync_done+0x8c 8192 | 0 spa_sync+0x12bc 16384 | 0 0xc000000002635b28 32768 | 0 fork_exit+0xdc 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 32640 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 65536 |@@@@@@@@@@@@@@@ 1 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac 0xc00000000258931c 0xc000000002589110 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 32133 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 65536 |@@@@@@@@@@@@@@@ 1 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac 0xc00000000258931c arc_buf_destroy+0x190 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 5923 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 4096 | 0 free+0x44 8192 | 0 0xc000000002524658 16384 |@@@@@@@@@@@@@@@ 1 0xc000000002511878 xpt_done_direct+0x60 0xc000000002a0e1ec ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 62357 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 131072 |@@@@@@@@@@@@@@@ 1 _bus_dmamap_load_buffer+0x21c bus_dmamap_load+0xb8 0xc000000002a0c85c nvme_qpair_submit_request+0xa0 nvme_ctrlr_submit_io_request+0x40 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 16850 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 32768 |@@@@@@@@@@@@@@@ 1 uma_large_malloc_domain+0x184 uma_large_malloc+0x28 malloc+0x1a0 zfs_kmem_alloc+0x2c 0xc0000000026bb318 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 93032 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 65536 | 0 uma_large_malloc_domain+0x15c 131072 | 0 uma_large_malloc+0x28 262144 |@@@@@@@@@@@@@@@ 1 malloc+0x1a0 zfs_kmem_alloc+0x2c 0xc0000000026bb318 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 31827 dp_sync_taskq taskqueue_quiesce+0x2e8 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 taskq_wait+0x24 65536 |@@@@@@@@@@@@@@@ 1 dsl_pool_sync+0x2a4 spa_sync+0xc08 0xc000000002635b28 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 32585 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 taskq_wait+0x24 65536 |@@@@@@@@@@@@@@@ 1 dmu_objset_sync+0x30c dsl_dataset_sync+0x1f0 dsl_pool_sync+0x300 spa_sync+0xc08 0xc000000002635b28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 10440 CAM device lock 0xc000000002524924 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 0xc000000002511878 512 | 0 xpt_done_direct+0x60 1024 | 0 0xc000000002a0e1ec 2048 | 0 0xc000000002a0b8c8 4096 | 0 nvme_qpair_process_completions+0x134 8192 | 0 0xc000000002a0bff0 16384 | 0 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 11345 CAM device lock (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 0xc00000000250e564 1024 | 0 cam_iosched_schedule+0x48 2048 | 0 0xc000000002523e78 4096 | 0 0xc000000002c1fd20 8192 | 0 g_io_request+0x400 16384 | 0 0xc000000002c32eac 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 20618 CAM device lock 0xc000000002523fac nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 0xc000000002c1fd20 4096 | 0 g_io_request+0x400 8192 | 0 0xc000000002c32eac 16384 | 0 g_io_request+0x400 32768 | 0 0xc000000002c26d8c 65536 |@@@@@@@@@@@@@@@ 1 g_io_request+0x400 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 140 zio_free_issue_0 (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1099 vnode interlock 0xc0000000030c7358 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 VOP_MARKATIME_APV+0x98 vfs_mark_atime+0x58 kern_execve+0x1518 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 38965 vnode interlock 0xc000000002e1ab8c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0xc54 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 334 zio_free_issue_7 taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 0xc0000000026bb048 zio_execute+0x19c zio_nowait+0x118 0xc000000002624e40 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 599 zio_free_issue_5 taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 0xc0000000026bb048 zio_execute+0x19c zio_nowait+0x118 0xc000000002624e40 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1231 zio_free_issue_4 (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1084 zio_free_issue_7 0xc000000002d74f5c nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 934 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0x1d0 namei+0x824 kern_accessat+0x124 sys_access+0x34 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23244 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0x1d0 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21581 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0xe88 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 834 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 cache_lookup+0xc30 vfs_cache_lookup+0xf8 zfs_freebsd_aclcheck+0x74 VOP_LOOKUP_APV+0x98 lookup+0x800 namei+0x538 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 39519 vnode interlock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0xa90 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 20863 vm page vm_fault_hold+0x2724 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 suword+0x90 suword+0x7c exec_copyout_strings+0x1c8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3074 vm page vm_page_change_lock+0x160 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc00000000310d1e0 0xc00000000310bb78 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 15887 vm page free queue (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030d9364 uma_zfree_arg+0x69c vm_page_free_toq+0x1d4 vm_page_free+0x2c vm_object_terminate+0x2f8 vm_object_deallocate+0x728 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 216 vnode interlock vget+0x2f8 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 cache_lookup+0xc30 vfs_cache_lookup+0xf8 VOP_LOOKUP_APV+0x98 lookup+0x800 namei+0x538 vn_open_cred+0x158 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 893 vnode interlock vget+0x2f8 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 cache_lookup+0xc30 vfs_cache_lookup+0xf8 VOP_LOOKUP_APV+0x98 lookup+0x800 namei+0x538 kern_accessat+0x124 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23189 vnode interlock 0xc000000002e02370 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 VOP_UNSET_TEXT_APV+0x98 kern_execve+0x980 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 41638 vnode interlock 0xc000000002e1ab70 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0xa90 namei+0x538 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 88994 vnode interlock 0xc000000002e1ab70 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0x1d0 namei+0x538 vn_open_cred+0x2f8 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 44290 vnode interlock 0xc000000002e1ab70 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0x1d0 namei+0x538 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 41499 vnode interlock 0xc000000002e1ab70 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_statat+0x168 sys_fstatat+0x40 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23139 umtxql (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11257 umtxql (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 0xc000000002d1d6e4 0xc000000002d19a70 sys__umtx_op+0x48 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 171 ttymtx 0xc000000002db0ba0 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_select+0xdd4 sys_select+0x84 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 225 ttymtx 0xc000000002daa684 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002b694d4 kern_select+0xb14 sys_select+0x84 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 652 struct mount vlist mtx _vdrop+0x6f4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002e1aa98 kern_readlinkat+0x184 sys_readlink+0x38 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 42298 ttymtx (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ttydisc_getc_uio+0x20c 0xc000000002dafae8 0xc000000002d873c8 sys_read+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 81218 struct mount mtx vfs_unbusy+0x15c nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0xb24 namei+0x538 kern_accessat+0x124 sys_access+0x34 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 41494 struct mount mtx vfs_rel+0x128 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vn_start_write+0x160 vn_close+0xd0 0xc000000002e31db8 _fdrop+0x4c closef+0x39c 0xc000000002c68bf4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 167 struct mount mtx vfs_ref+0x114 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vop_stdgetwritemount+0x48 VOP_GETWRITEMOUNT_APV+0x98 vn_start_write+0x68 0xc000000002e37c18 0xc000000002e37790 0xc000000002e33f34 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 191 struct mount vlist mtx (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002e1aa98 kern_readlinkat+0x184 sys_readlink+0x38 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11358 struct mount vlist mtx (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002e1aa98 namei+0x7d4 kern_accessat+0x124 sys_access+0x34 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1035 struct mount vlist mtx _vhold+0x474 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vref+0x3c zfs_zget+0x1e8 zfs_dirent_lookup+0x1dc zfs_dirlook+0xdc 0xc0000000026a00f0 0xc0000000026a0a08 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23062 vm active pagequeue (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_page_alloc_domain_after+0x28c vm_page_alloc+0xa4 vm_fault_hold+0x1660 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11342 umtxql (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d19a70 sys__umtx_op+0x48 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 29362 vm active pagequeue 0xc00000000310d62c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc00000000310bb78 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 88909 vm active pagequeue 0xc0000000031072a8 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_fault_hold+0x2220 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 suword+0x90 suword+0x7c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22965 vm active pagequeue vm_page_dequeue+0x1b4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_page_alloc_domain_after+0x28c vm_page_alloc+0xa4 vm_fault_hold+0x1660 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 40410 vm page (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 memcpy+0xb8 copyout+0xa0 exec_copyout_strings+0xbc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1039 vm page (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_execve+0x37c sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 20818 vm page (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_fault_hold+0x944 vm_fault+0x94 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5956 vm page vm_fault_hold+0x2724 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 memcpy+0xb8 copyout+0xa0 exec_copyout_strings+0xbc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11028 page pv (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_enter+0xbc vm_fault_hold+0x20f4 vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 suword+0x90 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 312576 kernelpmap moea64_remove+0x420 nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 moea64_qremove+0x60 pmap_qremove+0x8c allocbuf+0x104 brelse+0x298 bufdone+0x440 0xc000000002c2bfb0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 81444 metaslab_group_taskq 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 321 nvme qpair lock nvme_qpair_submit_request+0x120 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 nvme_ctrlr_submit_io_request+0x40 0xc000000002a0e10c 0xc00000000250c200 xpt_action_default+0x9b4 0xc000000002525078 xpt_action+0x44 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 85103 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 0xc00000000316651c 0xc000000003165c18 0xc000000002d12720 softclock+0xdc 0xc000000002c922c4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 41866 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac 0xc00000000258931c arc_buf_destroy+0x190 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 20090 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 uma_large_malloc_domain+0x184 uma_large_malloc+0x28 malloc+0x1a0 zfs_kmem_alloc+0x2c 0xc0000000026bb318 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11444 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 0xc000000002d76228 taskqueue_quiesce+0x1ec taskq_wait+0x24 dmu_objset_sync+0x30c dsl_dataset_sync+0x1f0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21408 dp_sync_taskq (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_quiesce+0x1c4 taskq_wait+0x24 dmu_objset_sync+0x30c dsl_pool_sync+0x450 spa_sync+0xc08 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22478 dp_sync_taskq 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_quiesce+0x1c4 taskq_wait+0x24 dmu_objset_sync+0x30c dsl_pool_sync+0x450 spa_sync+0xc08 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 20025 dp_sync_taskq 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 0xc000000002d76228 taskqueue_quiesce+0x1ec taskq_wait+0x24 dmu_objset_sync+0x30c dsl_pool_sync+0x450 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21281 dp_sync_taskq taskqueue_quiesce+0x2e8 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskq_wait+0x24 dmu_objset_sync+0x30c dsl_dataset_sync+0x1f0 dsl_pool_sync+0x300 spa_sync+0xc08 0xc000000002635b28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11393 kernel arena domain vmem_xfree+0x39c nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vmem_free+0x34 kmem_free+0x58 uma_large_free+0x30 free+0x90 zfs_kmem_free+0x28 zio_buf_free+0xac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3111 kernel arena domain vmem_xalloc+0x8c0 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vmem_alloc+0x84 kmem_malloc_domainset+0xac uma_large_malloc_domain+0x148 uma_large_malloc+0x28 malloc+0x1a0 zfs_kmem_alloc+0x2c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23122 page pv moea64_page_wired_mappings+0x1b8 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_page_wired_mappings+0x80 brelse+0x5c0 0xc000000003093c94 softdep_setup_freeblocks+0x944 ffs_truncate+0x744 0xc0000000030c6e84 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23743 page pv moea64_remove+0x22c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 moea64_qremove+0x60 pmap_qremove+0x8c allocbuf+0x104 brelse+0x298 bufdone+0x440 0xc000000002c2bfb0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1357 page pv (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_page_wired_mappings+0x80 brelse+0x5c0 0xc000000003093a1c softdep_setup_freeblocks+0x944 ffs_truncate+0x744 ufs_inactive+0x29c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5900 page pv (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_page_wired_mappings+0x80 brelse+0x5c0 0xc000000003093c94 softdep_setup_freeblocks+0x944 ffs_truncate+0x744 ufs_inactive+0x29c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11385 page pv moea64_enter+0x904 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_enter+0xbc vm_fault_hold+0x135c vm_fault+0x94 trap+0x1194 powerpc_interrupt+0x1a4 memcpy+0xd4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 865 process lock 0xc000000002c86d88 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_wait6+0x564 sys_wait4+0xd8 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23505 process lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 268 pipe mutex 0xc000000002d8f10c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d8e724 _fdrop+0x4c closef+0x39c 0xc000000002c6c64c fdescfree+0x628 exit1+0x830 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22952 pipe mutex 0xc000000002d8bf18 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d873c8 sys_read+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 965 pipe mutex (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_poll+0x5a4 sys_poll+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 44425 pipe mutex (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d873c8 sys_read+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 39429 pipe mutex (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 20931 so_snd tcp_do_segment+0x3ca0 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 tcp_input+0xe34 ip_input+0x1cc netisr_dispatch_src+0x14c netisr_dispatch+0x28 ether_demux+0x1a0 0xc000000002e6470c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 182 so_snd sosend_generic+0x3bc nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sosend+0x68 0xc000000002d940c4 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1326 sellck 0xc000000002d8af60 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d8d5b0 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5288 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vop_stdgetwritemount+0x48 VOP_GETWRITEMOUNT_APV+0x98 vn_start_write+0x68 0xc000000002e37c18 0xc000000002e37790 0xc000000002e33f34 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1045 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 lookup+0xb24 namei+0x538 kern_accessat+0x124 sys_access+0x34 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 986 struct mount mtx (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 insmntque+0x2c zfs_mknode+0x96c 0xc0000000026a105c VOP_CREATE_APV+0x98 vn_open_cred+0x398 vn_open+0x30 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11438 struct mount mtx vfs_ref+0x114 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vop_stdgetwritemount+0x48 VOP_GETWRITEMOUNT_APV+0x98 vn_start_write+0x68 vn_close+0xd0 0xc000000002e31db8 _fdrop+0x4c ------------------------------------------------------------------------------- Adaptive mutex block: 16 events in 21.178 seconds (1 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 6% 0.00 19123 zio_write_issue 0xc000000002d74f5c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 12% 0.00 46415 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 0xc0000000026c145c 0xc0000000026bd95c zio_execute+0x19c taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 19% 0.00 125612 zio_write_issue taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 0xc0000000026bb048 zio_execute+0x19c zio_nowait+0x118 0xc0000000025a4230 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 25% 0.00 55624 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 31% 0.00 20295 zio_write_issue 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 38% 0.00 17933 zio_write_intr (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 zio_delay_interrupt+0x244 0xc0000000026508a8 biodone+0x104 g_io_deliver+0x348 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 44% 0.00 79392 page pv moea64_remove_pages+0x214 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_remove_pages+0x80 vmspace_exit+0x100 exit1+0x954 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 50% 0.00 21725 zio_write_intr 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 56% 0.00 116383 zio_write_issue (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 62% 0.00 240176 dp_sync_taskq 0xc000000002d74f5c nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 69% 0.00 143619 dp_sync_taskq taskqueue_enqueue+0x158 nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 taskq_dispatch+0xb4 dmu_objset_sync+0x2b8 dsl_pool_sync+0x450 spa_sync+0xc08 0xc000000002635b28 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 75% 0.00 147567 dp_sync_taskq 0xc000000002cbb898 nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 81% 0.00 74445 kernelpmap moea64_enter+0x918 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_enter+0xbc kmem_back_domain+0x200 kmem_malloc_domainset+0xd0 uma_large_malloc_domain+0x148 uma_large_malloc+0x28 malloc+0x1a0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 88% 0.00 88996 kernelpmap (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac abd_free+0x1b4 0xc0000000026befc0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 94% 0.00 103220 kernelpmap moea64_kextract+0x174 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 pmap_kextract+0x80 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac abd_free+0x1b4 0xc000000002656114 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 6% 100% 0.00 54064 nvme qpair lock 0xc000000002a0bae4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 nvme_qpair_process_completions+0x134 0xc000000002a0bff0 0xc000000002c922c4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Spin lock spin: 27943 events in 21.178 seconds (1319 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4013 14% 14% 0.00 1333690 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 10 __rw_wlock_hard+0x2cc 512 | 18 vm_fault_hold+0x1b28 1024 | 42 vm_fault+0x94 2048 | 52 trap+0x10ac 4096 | 58 powerpc_interrupt+0x1a4 8192 | 79 16384 | 57 32768 | 60 65536 | 93 131072 |@ 179 262144 |@@ 348 524288 |@@@@@@ 828 1048576 |@@@@@@@@@ 1243 2097152 |@@@@ 606 4194304 | 116 8388608 | 64 16777216 | 91 33554432 | 69 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3459 12% 27% 0.00 1254181 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 4 __rw_wlock_hard+0x2cc 512 | 20 vm_fault_hold+0x1b28 1024 | 26 vm_fault+0x94 2048 | 39 trap+0x10ac 4096 | 39 powerpc_interrupt+0x1a4 8192 | 54 16384 | 56 32768 | 79 65536 | 51 131072 |@ 140 262144 |@@ 318 524288 |@@@@@@ 720 1048576 |@@@@@@@@ 987 2097152 |@@@@@ 610 4194304 |@ 130 8388608 | 71 16777216 | 90 33554432 | 25 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1815 6% 33% 0.00 1420313 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 6 __rw_wlock_hard+0x2cc 512 | 7 vm_object_reference+0x134 1024 | 12 0xc000000002c479cc 2048 | 19 0xc000000002c472c0 4096 | 28 0xc000000002c4684c 8192 | 33 kern_execve+0x678 16384 | 35 32768 | 25 65536 | 38 131072 | 52 262144 |@@ 131 524288 |@@@@@@ 395 1048576 |@@@@@@@@ 539 2097152 |@@@@@ 324 4194304 |@ 61 8388608 | 38 16777216 | 39 33554432 | 33 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1637 6% 39% 0.00 1264122 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wlock_hard+0x2cc 512 | 6 vm_object_reference+0x134 1024 | 12 0xc000000002c479cc 2048 | 20 0xc000000002c472c0 4096 | 16 0xc000000002c4684c 8192 | 30 kern_execve+0x678 16384 | 33 32768 | 27 65536 | 23 131072 | 52 262144 |@@ 127 524288 |@@@@@@ 334 1048576 |@@@@@@@@@ 511 2097152 |@@@@@ 307 4194304 | 42 8388608 | 36 16777216 | 53 33554432 | 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1379 5% 44% 0.00 1314729 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wlock_hard+0x2cc 512 | 8 vm_map_insert+0x758 1024 | 8 vm_map_fixed+0x120 2048 | 11 0xc000000002c47a14 4096 | 21 0xc000000002c472c0 8192 | 14 0xc000000002c4684c 16384 | 20 32768 | 22 65536 | 21 131072 |@ 51 262144 |@@ 104 524288 |@@@@@@ 299 1048576 |@@@@@@@@@ 455 2097152 |@@@@ 222 4194304 |@ 52 8388608 | 25 16777216 | 25 33554432 | 20 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1340 5% 49% 0.00 1336435 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 6 __rw_wlock_hard+0x2cc 512 | 17 vm_object_deallocate+0x800 1024 | 13 0xc0000000030ea1fc 2048 | 33 _vm_map_unlock+0xf8 4096 | 28 vmspace_exit+0x194 8192 | 27 exit1+0x954 16384 | 23 32768 | 25 65536 | 36 131072 | 31 262144 |@@ 96 524288 |@@@@@@ 288 1048576 |@@@@@@@@ 370 2097152 |@@@@@ 229 4194304 | 40 8388608 | 29 16777216 | 31 33554432 | 18 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1272 5% 53% 0.00 1244382 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 3 __rw_wlock_hard+0x2cc 512 | 7 vm_object_deallocate+0x800 1024 | 11 0xc0000000030ea1fc 2048 | 16 _vm_map_unlock+0xf8 4096 | 21 vmspace_exit+0x194 8192 | 20 exit1+0x954 16384 | 18 32768 | 29 65536 | 27 131072 | 37 262144 |@ 84 524288 |@@@@@@ 265 1048576 |@@@@@@@@@ 384 2097152 |@@@@@ 231 4194304 |@ 49 8388608 | 29 16777216 | 38 33554432 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1071 4% 57% 0.00 1331100 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 7 __rw_wlock_hard+0x2cc 512 | 7 0xc0000000030f0be8 1024 | 14 vm_map_delete+0x1bc 2048 | 18 vmspace_exit+0x184 4096 | 15 exit1+0x954 8192 | 11 sys_sys_exit+0x28 16384 | 22 32768 | 20 65536 | 16 131072 | 28 262144 |@ 71 524288 |@@@@@ 210 1048576 |@@@@@@@@@ 333 2097152 |@@@@@ 205 4194304 | 29 8388608 | 25 16777216 | 28 33554432 | 12 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 891 3% 60% 0.00 1481528 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 2 __rw_wlock_hard+0x2cc 512 | 3 0xc0000000030f0be8 1024 | 3 vm_map_delete+0x1bc 2048 | 10 vmspace_exit+0x184 4096 | 15 exit1+0x954 8192 | 25 sys_sys_exit+0x28 16384 | 15 32768 | 8 65536 | 11 131072 | 20 262144 |@@ 65 524288 |@@@@@@ 184 1048576 |@@@@@@@@ 255 2097152 |@@@@@ 177 4194304 |@ 33 8388608 | 25 16777216 |@ 31 33554432 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 780 3% 63% 0.00 1312414 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 512 | 1 __rw_wlock_hard+0x2cc 1024 | 5 vm_map_insert+0x758 2048 | 8 vm_map_fixed+0x120 4096 | 6 0xc000000002c47a14 8192 | 10 0xc000000002c472c0 16384 | 4 0xc000000002c4684c 32768 | 4 65536 | 10 131072 | 25 262144 |@@ 61 524288 |@@@@@@ 165 1048576 |@@@@@@@@@ 258 2097152 |@@@@@ 149 4194304 |@ 30 8388608 | 16 16777216 | 24 33554432 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 762 3% 66% 0.00 1096199 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 2 __rw_wlock_hard+0x2cc 512 | 3 vm_object_deallocate+0x800 1024 | 1 kern_execve+0x9d0 2048 | 6 sys_execve+0x80 4096 | 16 trap+0xb80 8192 | 11 powerpc_interrupt+0x1a4 16384 | 19 32768 | 24 65536 | 15 131072 |@ 31 262144 |@@ 75 524288 |@@@@@@ 164 1048576 |@@@@@@@@@ 233 2097152 |@@@@ 109 4194304 | 21 8388608 | 13 16777216 | 13 33554432 | 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 706 3% 68% 0.00 1205719 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 3 __rw_wlock_hard+0x2cc 512 | 1 vnode_create_vobject+0x18c 1024 | 4 0xc0000000030c5fe0 2048 | 11 VOP_OPEN_APV+0x98 4096 | 5 exec_check_permissions+0x164 8192 | 12 kern_execve+0x348 16384 | 5 32768 | 5 65536 | 16 131072 | 17 262144 |@@ 58 524288 |@@@@@ 125 1048576 |@@@@@@@@@ 223 2097152 |@@@@@@ 159 4194304 |@ 29 8388608 | 13 16777216 | 19 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 689 2% 71% 0.00 1408392 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wlock_hard+0x2cc 512 | 4 vnode_create_vobject+0x18c 1024 | 6 0xc0000000030c5fe0 2048 | 10 VOP_OPEN_APV+0x98 4096 | 9 exec_check_permissions+0x164 8192 | 7 kern_execve+0x348 16384 | 9 32768 | 6 65536 | 9 131072 | 17 262144 |@ 42 524288 |@@@@@@ 143 1048576 |@@@@@@@@@@ 236 2097152 |@@@@@ 118 4194304 |@ 30 8388608 | 16 16777216 | 16 33554432 | 10 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 656 2% 73% 0.00 1258907 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 4 __rw_wlock_hard+0x2cc 512 | 4 vm_object_deallocate+0x814 1024 | 9 0xc0000000030ea1fc 2048 | 11 _vm_map_unlock+0xf8 4096 | 12 vmspace_exit+0x194 8192 | 11 exit1+0x954 16384 | 14 32768 | 10 65536 | 17 131072 |@ 22 262144 |@@ 50 524288 |@@@@@@ 143 1048576 |@@@@@@@@ 188 2097152 |@@@@ 103 4194304 |@ 23 8388608 | 12 16777216 | 15 33554432 | 8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 656 2% 76% 0.00 1073672 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 512 | 4 __rw_wlock_hard+0x2cc 1024 | 3 vm_imgact_map_page+0x3f0 2048 | 5 0xc000000002c4733c 4096 | 9 0xc000000002c4684c 8192 | 11 kern_execve+0x678 16384 | 13 sys_execve+0x80 32768 | 15 65536 | 19 131072 |@ 23 262144 |@@ 65 524288 |@@@@@@@ 158 1048576 |@@@@@@@@ 191 2097152 |@@@@ 100 4194304 | 16 8388608 | 8 16777216 | 9 33554432 | 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 604 2% 78% 0.00 960261 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 4 0xc000000002ce6044 512 | 6 __rw_rlock_int+0x9c 1024 | 8 0xc0000000030ebe78 2048 |@ 21 vm_map_insert+0x6a8 4096 |@ 34 vm_map_fixed+0x120 8192 |@ 31 0xc000000002c47a14 16384 |@ 35 32768 |@ 30 65536 |@ 25 131072 |@ 28 262144 |@@@ 67 524288 |@@@@@@ 129 1048576 |@@@@@ 119 2097152 |@ 23 4194304 | 15 8388608 | 4 16777216 |@ 21 33554432 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 488 2% 80% 0.00 1190576 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wlock_hard+0x2cc 512 | 1 vm_object_shadow+0x378 1024 | 3 vm_map_lookup+0xb80 2048 | 7 vm_fault_hold+0xf8 4096 | 10 vm_fault+0x94 8192 | 12 trap+0x10ac 16384 | 8 32768 | 8 65536 | 14 131072 | 14 262144 |@@ 33 524288 |@@@@@@ 111 1048576 |@@@@@@@@ 142 2097152 |@@@@ 80 4194304 |@ 18 8388608 | 9 16777216 | 16 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 487 2% 81% 0.00 1087119 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 2 __rw_wlock_hard+0x2cc 512 | 2 vm_object_shadow+0x390 1024 | 4 vm_map_lookup+0xb80 2048 | 8 vm_fault_hold+0xf8 4096 | 6 vm_fault+0x94 8192 | 9 trap+0x10ac 16384 | 5 32768 | 6 65536 | 10 131072 |@ 20 262144 |@@ 35 524288 |@@@@@ 94 1048576 |@@@@@@@@@ 153 2097152 |@@@@@@ 104 4194304 | 10 8388608 | 8 16777216 | 10 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 478 2% 83% 0.00 705127 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 4 0xc000000002ce6044 512 | 8 __rw_rlock_int+0x9c 1024 | 5 vm_map_entry_set_vnode_text+0x4c 2048 |@ 24 0xc0000000030ea1e4 4096 |@ 24 _vm_map_unlock+0xf8 8192 |@ 28 vmspace_exit+0x194 16384 |@ 22 32768 |@ 29 65536 | 12 131072 |@ 29 262144 |@@@ 62 524288 |@@@@@@@ 113 1048576 |@@@@ 78 2097152 |@ 18 4194304 | 9 8388608 | 2 16777216 | 7 33554432 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 465 2% 85% 0.00 1359315 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wlock_hard+0x2cc 512 | 3 vm_imgact_map_page+0x3f0 1024 | 1 0xc000000002c4733c 2048 | 7 0xc000000002c4684c 4096 | 5 kern_execve+0x678 8192 | 4 sys_execve+0x80 16384 | 4 32768 | 4 65536 | 11 131072 | 14 262144 |@@ 41 524288 |@@@@@@ 102 1048576 |@@@@@@@@@ 143 2097152 |@@@@@ 79 4194304 | 14 8388608 | 14 16777216 | 14 33554432 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 446 2% 86% 0.00 1241729 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 512 | 2 __rw_wlock_hard+0x2cc 1024 | 2 vm_object_deallocate+0x814 2048 | 5 0xc0000000030ea1fc 4096 | 4 _vm_map_unlock+0xf8 8192 | 8 vmspace_exit+0x194 16384 | 5 exit1+0x954 32768 | 3 65536 | 7 131072 | 9 262144 |@@ 39 524288 |@@@@ 73 1048576 |@@@@@@@@@ 145 2097152 |@@@@@@ 99 4194304 |@ 25 8388608 | 9 16777216 | 9 33554432 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 437 2% 88% 0.00 1546937 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wlock_hard+0x2cc 512 | 2 vm_object_shadow+0x390 1024 | 1 vm_map_lookup+0xb80 2048 | 4 vm_fault_hold+0xf8 4096 | 8 vm_fault+0x94 8192 | 6 trap+0x10ac 16384 | 12 32768 | 7 65536 | 11 131072 | 10 262144 |@@ 43 524288 |@@@@@@ 94 1048576 |@@@@@@@ 110 2097152 |@@@@@ 75 4194304 |@ 19 8388608 | 12 16777216 | 13 33554432 | 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 390 1% 89% 0.00 1238076 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 512 | 2 __rw_wlock_hard+0x2cc 1024 | 3 vm_object_shadow+0x378 2048 | 2 vm_map_lookup+0xb80 4096 | 6 vm_fault_hold+0xf8 8192 | 7 vm_fault+0x94 16384 | 11 trap+0x10ac 32768 | 6 65536 | 5 131072 |@ 17 262144 |@@ 33 524288 |@@@@@@ 87 1048576 |@@@@@@@@@ 125 2097152 |@@@@ 54 4194304 |@ 13 8388608 | 7 16777216 | 7 33554432 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 333 1% 90% 0.00 1139512 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wlock_hard+0x2cc 512 | 1 vm_object_reference+0x134 1024 | 1 kern_execve+0x370 2048 | 2 sys_execve+0x80 4096 | 2 trap+0xb80 8192 | 3 powerpc_interrupt+0x1a4 16384 | 7 32768 | 1 65536 | 11 131072 |@ 14 262144 |@@ 27 524288 |@@@@@@ 73 1048576 |@@@@@@@@@ 102 2097152 |@@@@@ 62 4194304 | 11 8388608 | 6 16777216 | 8 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 303 1% 91% 0.00 1494347 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 512 | 1 __rw_wlock_hard+0x2cc 1024 | 1 vm_object_deallocate+0x800 2048 | 2 kern_execve+0x9d0 4096 | 1 sys_execve+0x80 8192 | 3 trap+0xb80 16384 | 7 powerpc_interrupt+0x1a4 32768 | 7 65536 | 4 131072 |@ 11 262144 |@@ 24 524288 |@@@@@ 58 1048576 |@@@@@@@@ 89 2097152 |@@@@@ 54 4194304 |@ 15 8388608 |@ 13 16777216 |@ 12 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 240 1% 92% 0.00 1619397 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 3 0xc000000002ce6044 512 | 4 __rw_rlock_int+0x9c 1024 | 1 0xc0000000030ebe78 2048 | 7 vm_map_insert+0x6a8 4096 | 6 vm_map_fixed+0x120 8192 |@ 11 0xc000000002c47a14 16384 |@ 11 32768 | 5 65536 | 6 131072 |@ 9 262144 |@@@ 31 524288 |@@@@@ 46 1048576 |@@@@@ 43 2097152 |@@@ 28 4194304 | 3 8388608 |@ 10 16777216 |@ 14 33554432 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 234 1% 93% 0.00 8788 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@ 13 _sleep+0x118 512 |@@ 18 taskqueue_thread_loop+0x1b4 1024 | 6 fork_exit+0xdc 2048 |@@@@ 35 0xc000000003185c5c 4096 |@@@@@ 40 8192 |@@@@@@@ 62 16384 |@@ 17 32768 |@@@ 26 65536 |@@ 16 131072 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 210 1% 94% 0.00 1200713 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 512 | 1 __rw_wlock_hard+0x2cc 1024 | 1 vm_object_reference+0x134 2048 | 1 kern_execve+0x370 4096 | 6 sys_execve+0x80 8192 | 2 trap+0xb80 16384 | 2 powerpc_interrupt+0x1a4 32768 | 2 65536 | 6 131072 |@ 8 262144 |@@ 20 524288 |@@@@@@ 43 1048576 |@@@@@@@@@ 64 2097152 |@@@@ 34 4194304 | 6 8388608 |@ 9 16777216 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 128 0% 94% 0.00 1241904 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 0xc000000002ce6044 512 | 1 __rw_rlock_int+0x9c 1024 | 1 vm_map_entry_set_vnode_text+0x4c 2048 | 4 0xc0000000030ea1e4 4096 |@@ 9 _vm_map_unlock+0xf8 8192 |@@ 9 vmspace_exit+0x194 16384 | 3 32768 | 2 65536 | 3 131072 | 2 262144 |@@ 11 524288 |@@@@ 21 1048576 |@@@@@@@ 33 2097152 |@@ 12 4194304 | 4 8388608 |@ 7 16777216 |@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 125 0% 95% 0.00 7575 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@ 9 _sleep+0x118 512 |@@ 9 taskqueue_thread_loop+0x1b4 1024 |@@@ 13 fork_exit+0xdc 2048 |@@@@ 19 0xc000000003185c5c 4096 |@@@@ 20 8192 |@@@@@@@@@ 41 16384 | 4 32768 |@ 7 65536 | 2 131072 | 0 262144 | 0 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 118 0% 95% 0.00 14977 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@ 7 wakeup_one+0x2c 512 |@@ 11 taskqueue_thread_enqueue+0x24 1024 |@@ 8 0xc000000002d740c8 2048 |@@@@ 18 taskqueue_enqueue+0x128 4096 |@@ 9 taskq_dispatch_ent+0x50 8192 |@@@@@ 22 spa_taskq_dispatch_ent+0xb0 16384 |@@@@ 19 32768 |@@@ 14 65536 |@ 4 131072 | 2 262144 | 3 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 106 0% 96% 0.00 1323192 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 512 | 1 __rw_wlock_hard+0x2cc 1024 | 0 exec_map_first_page+0x780 2048 | 1 kern_execve+0x37c 4096 | 2 sys_execve+0x80 8192 |@ 5 trap+0xb80 16384 | 2 powerpc_interrupt+0x1a4 32768 | 1 65536 | 0 131072 | 1 262144 |@@ 8 524288 |@@@@@@ 23 1048576 |@@@@@@@@ 31 2097152 |@@@@@@ 23 4194304 |@ 4 8388608 | 0 16777216 | 3 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 97 0% 96% 0.00 1215456 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 512 | 2 __rw_wlock_hard+0x2cc 1024 | 0 exec_map_first_page+0x780 2048 | 2 kern_execve+0x37c 4096 | 3 sys_execve+0x80 8192 | 0 trap+0xb80 16384 | 2 powerpc_interrupt+0x1a4 32768 | 2 65536 | 3 131072 | 2 262144 |@ 5 524288 |@@@@@ 18 1048576 |@@@@@@@@@ 30 2097152 |@@@@@ 17 4194304 |@ 5 8388608 |@ 5 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 93 0% 96% 0.00 676816 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wunlock_hard+0xb0 512 | 1 vm_fault_hold+0xe9c 1024 |@ 4 vm_fault+0x94 2048 | 3 trap+0x10ac 4096 |@@ 7 powerpc_interrupt+0x1a4 8192 |@@ 9 16384 |@ 4 32768 |@ 4 65536 | 3 131072 |@ 4 262144 |@@ 9 524288 |@@@@ 14 1048576 |@@@@@@ 19 2097152 |@ 5 4194304 | 3 8388608 | 2 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 69 0% 97% 0.00 507494 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 256 | 1 __rw_wunlock_hard+0xb0 512 | 2 vm_object_deallocate+0xb9c 1024 | 0 0xc0000000030ea1fc 2048 |@@ 5 _vm_map_unlock+0xf8 4096 | 2 vmspace_exit+0x194 8192 |@ 3 exit1+0x954 16384 | 2 32768 | 1 65536 |@ 4 131072 |@@@ 7 262144 |@@@ 8 524288 |@@@ 9 1048576 |@@@@@@@@@ 21 2097152 | 2 4194304 | 1 8388608 | 0 16777216 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 64 0% 97% 0.00 1076845 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 256 | 1 0xc000000002ce688c 512 | 2 _rw_runlock_cookie_int+0xa8 1024 | 0 vm_map_entry_set_vnode_text+0x10c 2048 |@ 4 0xc0000000030ea1e4 4096 |@@ 5 _vm_map_unlock+0xf8 8192 | 2 vmspace_exit+0x194 16384 |@ 3 32768 |@@ 5 65536 |@@ 5 131072 | 2 262144 | 1 524288 |@@@@@ 12 1048576 |@@@@@@ 14 2097152 |@@ 5 4194304 | 0 8388608 | 0 16777216 | 2 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 63 0% 97% 0.00 23418 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@ 6 wakeup_one+0x2c 512 |@ 3 taskqueue_thread_enqueue+0x24 1024 |@ 3 0xc000000002d740c8 2048 |@@ 5 taskqueue_enqueue+0x128 4096 |@@ 5 taskq_dispatch_ent+0x50 8192 |@@@@ 10 spa_taskq_dispatch_ent+0xb0 16384 |@@@ 7 32768 |@@ 5 65536 |@@@@@@ 13 131072 |@ 4 262144 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 38 0% 97% 0.00 320737 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 512 |@ 2 __rw_wunlock_hard+0xb0 1024 | 1 0xc0000000030f0b08 2048 | 1 vm_map_delete+0x1bc 4096 | 1 vmspace_exit+0x184 8192 |@@@ 4 exit1+0x954 16384 |@ 2 sys_sys_exit+0x28 32768 |@ 2 65536 | 1 131072 | 1 262144 |@@@ 5 524288 |@@@@@@@ 9 1048576 |@@@@@@ 8 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 30 0% 97% 0.00 11520 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 10 _sx_xlock_hard+0x354 512 |@ 1 _vm_map_lock+0x138 1024 | 0 vm_map_find+0xf0 2048 |@ 1 vm_map_find_min+0x98 4096 |@@@@@ 5 vm_mmap_object+0x46c 8192 |@@@@@@ 6 kern_mmap+0x450 16384 | 0 32768 |@@@@@ 5 65536 |@ 1 131072 | 0 262144 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 27 0% 97% 0.00 1083 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 512 |@@ 2 0xc000000002cad69c 1024 |@@@@@@@@@@@@@ 12 lockmgr_unlock_fast_path+0x13c 2048 |@@@@@@@@@@@@@ 12 vop_stdunlock+0x34 4096 |@ 1 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 lookup+0x1d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 26 0% 97% 0.00 1492 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@ 4 wakeup_one+0x2c 512 |@@@@@@@@@ 8 taskqueue_thread_enqueue+0x24 1024 |@@@@@@@@ 7 0xc000000002d740c8 2048 |@@@@@@ 6 taskqueue_enqueue+0x128 4096 | 0 taskq_dispatch+0xb4 8192 | 0 dmu_objset_sync+0x2b8 16384 | 0 32768 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 23 0% 98% 0.00 635 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@ 9 __rw_wlock_hard+0x2cc 512 |@@@@@ 4 vm_fault_hold+0x1b28 1024 |@@@@@@ 5 vm_fault+0x94 2048 |@@@@@@ 5 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 22 0% 98% 0.00 1094959 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 2048 |@ 1 __rw_wunlock_hard+0xb0 4096 |@ 1 vm_object_deallocate+0x830 8192 |@@@@ 3 0xc0000000030ea1fc 16384 |@ 1 _vm_map_unlock+0xf8 32768 | 0 vmspace_exit+0x194 65536 |@ 1 exit1+0x954 131072 |@ 1 262144 |@@@@@ 4 524288 |@@@@ 3 1048576 |@@@@ 3 2097152 |@@ 2 4194304 |@ 1 8388608 | 0 16777216 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 22 0% 98% 0.00 1406 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@ 3 0xc000000002cad69c 2048 |@@@@@@@@@@@@@@@@@@@@@@@ 17 lockmgr_unlock_fast_path+0x13c 4096 |@@ 2 vop_stdunlock+0x34 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 lookup+0xa90 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 21 0% 98% 0.00 1134291 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@ 1 __rw_wunlock_hard+0xb0 16384 | 0 vm_fault_hold+0xe9c 32768 |@ 1 vm_fault+0x94 65536 | 0 trap+0x10ac 131072 |@@@@ 3 powerpc_interrupt+0x1a4 262144 |@@@@@@@ 5 524288 |@@@@@ 4 1048576 |@@@@@@@ 5 2097152 | 0 4194304 |@ 1 8388608 | 0 16777216 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 16 0% 98% 0.00 862 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@ 5 __rw_wlock_hard+0x2cc 512 |@@@@@@@@@ 5 vm_fault_hold+0x1b28 1024 |@ 1 vm_fault+0x94 2048 |@@@@@ 3 trap+0x10ac 4096 |@@@ 2 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 0% 98% 0.00 522 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@ 7 __rw_wlock_hard+0x2cc 512 |@@@@@@ 3 vm_object_reference+0x134 1024 | 0 0xc000000002c479cc 2048 |@@@@@@@@@@ 5 0xc000000002c472c0 0xc000000002c4684c kern_execve+0x678 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 0% 98% 0.00 5596 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@ 6 _sx_xlock_hard+0x354 512 |@@ 1 _vm_map_lock+0x138 1024 | 0 vm_map_find+0xf0 2048 | 0 vm_map_find_min+0x98 4096 |@@@@ 2 vm_mmap_object+0x46c 8192 |@@@@ 2 kern_mmap+0x450 16384 |@@ 1 32768 |@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 98% 0.00 280681 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 512 |@@ 1 __rw_wunlock_hard+0xb0 1024 | 0 vm_map_insert+0x2a4 2048 | 0 vm_map_fixed+0x120 4096 |@@ 1 0xc000000002c47a14 8192 |@@@@@ 2 0xc000000002c472c0 16384 | 0 0xc000000002c4684c 32768 |@@@@@ 2 65536 |@@ 1 131072 | 0 262144 | 0 524288 |@@@@@ 2 1048576 |@@@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 98% 0.00 3442079 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@ 1 0xc000000002ce688c 1024 | 0 _rw_runlock_cookie_int+0xa8 2048 | 0 vm_map_entry_set_vnode_text+0x10c 4096 |@@ 1 0xc0000000030ea1e4 8192 | 0 _vm_map_unlock+0xf8 16384 |@@ 1 vmspace_exit+0x194 32768 | 0 65536 | 0 131072 | 0 262144 |@@ 1 524288 | 0 1048576 |@@@@@@@@@@ 4 2097152 | 0 4194304 |@@ 1 8388608 |@@ 1 16777216 |@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 98% 0.00 1088 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@ 1 0xc000000002cad69c 512 | 0 lockmgr_unlock_fast_path+0x13c 1024 |@@@@@@@@@@ 4 vop_stdunlock+0x34 2048 |@@@@@@@@@@@@@@@ 6 VOP_UNLOCK_APV+0x98 4096 |@@ 1 0xc000000002e1aa88 kern_statat+0x168 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 98% 0.00 18783 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@ 5 _sx_xlock_hard+0x354 512 |@@ 1 _vm_map_lock_upgrade+0x180 1024 | 0 vm_map_lookup+0xd68 2048 | 0 vm_fault_hold+0xf8 4096 | 0 vm_fault+0x94 8192 | 0 trap+0x10ac 16384 | 0 32768 |@@@@@@@@@@ 4 65536 |@@ 1 131072 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 98% 0.00 1225 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@ 1 0xc000000002cad69c 512 | 0 lockmgr_unlock_fast_path+0x13c 1024 |@@@@@@@ 3 vop_stdunlock+0x34 2048 |@@@@@@@@@@@@@@@@@ 7 VOP_UNLOCK_APV+0x98 4096 |@@ 1 0xc000000002e1aa88 lookup+0x1d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 98% 0.00 842 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@ 5 0xc000000002ce6044 512 |@@ 1 __rw_rlock_int+0x9c 1024 |@@ 1 0xc0000000030ebe78 2048 |@@@@@@@@ 3 vm_map_insert+0x6a8 4096 |@@ 1 vm_map_fixed+0x120 0xc000000002c47a14 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 98% 0.00 1218186 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 2048 |@@ 1 __rw_wunlock_hard+0xb0 4096 | 0 vm_object_reference+0x14c 8192 |@@ 1 0xc000000002c479cc 16384 |@@ 1 0xc000000002c472c0 32768 | 0 0xc000000002c4684c 65536 | 0 kern_execve+0x678 131072 | 0 262144 |@@ 1 524288 |@@@@@ 2 1048576 |@@@@@ 2 2097152 |@@ 1 4194304 |@@ 1 8388608 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 98% 0.00 8858 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@ 2 wakeup_one+0x2c 512 |@@@@@@@@@@ 4 taskqueue_thread_enqueue+0x24 1024 | 0 0xc000000002d740c8 2048 |@@@@@@@@ 3 taskqueue_enqueue+0x128 4096 |@@ 1 taskq_dispatch+0xb4 8192 | 0 dmu_objset_sync+0x2b8 16384 | 0 32768 | 0 65536 | 0 131072 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 98% 0.00 243500 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 1024 |@@@ 1 __rw_wunlock_hard+0xb0 2048 | 0 vm_object_deallocate+0xb9c 4096 | 0 kern_execve+0x9d0 8192 |@@@@@@@@@@@@ 4 sys_execve+0x80 16384 | 0 trap+0xb80 32768 | 0 powerpc_interrupt+0x1a4 65536 | 0 131072 |@@@ 1 262144 | 0 524288 |@@@@@@ 2 1048576 |@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 98% 0.00 1423 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@ 1 0xc000000002cad69c 1024 |@@@ 1 lockmgr_unlock_fast_path+0x13c 2048 |@@@@@@@@@@@@@@@@@@@@@@@@ 8 vop_stdunlock+0x34 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 lookup+0xa90 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 98% 0.00 835 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@ 2 __rw_wlock_hard+0x2cc 512 |@@@ 1 vm_object_deallocate+0x800 1024 |@@@ 1 kern_execve+0x9d0 2048 |@@@@@@@@@@@@@@@@ 5 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 98% 0.00 303890 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 4096 |@@@ 1 __rw_wunlock_hard+0xb0 8192 |@@@ 1 vnode_create_vobject+0x3f0 16384 | 0 0xc0000000030c5fe0 32768 | 0 VOP_OPEN_APV+0x98 65536 | 0 exec_check_permissions+0x164 131072 |@@@ 1 kern_execve+0x348 262144 |@@@ 1 524288 |@@@@@@@@@@ 3 1048576 |@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 98% 0.00 106237 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 2048 |@@@ 1 __rw_wunlock_hard+0xb0 4096 |@@@@@@ 2 vm_object_shadow+0x3c0 8192 |@@@@@@ 2 vm_map_lookup+0xb80 16384 | 0 vm_fault_hold+0xf8 32768 | 0 vm_fault+0x94 65536 | 0 trap+0x10ac 131072 |@@@ 1 262144 |@@@ 1 524288 |@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 98% 0.00 9663 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@ 1 wakeup+0x2c 512 | 0 0xc000000002d7515c 1024 |@@@ 1 taskqueue_thread_loop+0x158 2048 |@@@@@@ 2 fork_exit+0xdc 4096 |@@@ 1 0xc000000003185c5c 8192 |@@@ 1 16384 |@@@ 1 32768 |@@@ 1 65536 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 98% 0.00 125873 sched lock 25 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 4096 |@@@ 1 sched_switch+0x830 8192 | 0 mi_switch+0x158 16384 | 0 0xc000000002d6e2b4 32768 | 0 sleepq_wait+0x4c 65536 |@@@ 1 _sleep+0x248 131072 |@@@@@@ 2 taskqueue_thread_loop+0x1b4 262144 |@@@@@@@@@@@@@@@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 99% 0.00 78446 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 256 |@@@ 1 sched_idletd+0x2b4 512 | 0 fork_exit+0xdc 1024 | 0 0xc000000003185c5c 2048 | 0 0xc000000002500350 4096 |@@@@@@ 2 8192 |@@@ 1 16384 | 0 32768 |@@@@@@ 2 65536 | 0 131072 |@@@ 1 262144 |@@@ 1 524288 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 99% 0.00 765 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@ 2 0xc000000002ce6044 512 |@@@@@@@ 2 __rw_rlock_int+0x9c 1024 |@@@@@@@ 2 vm_map_entry_set_vnode_text+0x4c 2048 |@@@@@@@ 2 0xc0000000030ea1e4 _vm_map_unlock+0xf8 vmspace_exit+0x194 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 99% 0.00 467 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@ 3 __rw_wlock_hard+0x2cc 512 |@@@@@@@@@@@ 3 vm_object_reference+0x134 1024 |@@@ 1 0xc000000002c479cc 2048 |@@@ 1 0xc000000002c472c0 0xc000000002c4684c kern_execve+0x678 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 99% 0.00 1279 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@ 2 __rw_wlock_hard+0x2cc 512 |@@@@@@@@ 2 0xc0000000030f0be8 1024 | 0 vm_map_delete+0x1bc 2048 |@@@@@@@@ 2 vmspace_exit+0x184 4096 | 0 exit1+0x954 8192 |@@@@ 1 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 99% 0.00 815 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@ 1 __rw_wlock_hard+0x2cc 512 |@@@@ 1 vm_map_insert+0x758 1024 |@@@@@@@@@@@@ 3 vm_map_fixed+0x120 2048 |@@@@@@@@ 2 0xc000000002c47a14 0xc000000002c472c0 0xc000000002c4684c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 99% 0.00 1308 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@ 2 0xc000000002cad69c 2048 |@@@@@@@@@@@@@@@@@@@@@ 5 lockmgr_unlock_fast_path+0x13c vop_stdunlock+0x34 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 lookup+0xe88 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 99% 0.00 102356 sched lock 25 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@ 1 sched_idletd+0x2b4 65536 |@@@@ 1 fork_exit+0xdc 131072 |@@@@@@@@@@@@ 3 0xc000000003185c5c 262144 |@@@@@@@@ 2 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 99% 0.00 1044 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@ 1 __rw_wlock_hard+0x2cc 512 |@@@@@@@@@@ 2 vm_object_deallocate+0x800 1024 | 0 0xc0000000030ea1fc 2048 |@@@@@@@@@@ 2 _vm_map_unlock+0xf8 4096 |@@@@@ 1 vmspace_exit+0x194 exit1+0x954 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 99% 0.00 1038 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 2 __rw_wlock_hard+0x2cc 1024 |@@@@@ 1 vm_map_insert+0x758 2048 |@@@@@@@@@@@@@@@ 3 vm_map_fixed+0x120 0xc000000002c47a14 0xc000000002c472c0 0xc000000002c4684c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 99% 0.00 1226 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@ 1 __rw_wlock_hard+0x2cc 512 |@@@@@ 1 vnode_create_vobject+0x18c 1024 |@@@@@ 1 0xc0000000030c5fe0 2048 |@@@@@@@@@@ 2 VOP_OPEN_APV+0x98 4096 |@@@@@ 1 exec_check_permissions+0x164 kern_execve+0x348 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 99% 0.00 702 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 2 __rw_wlock_hard+0x2cc 512 |@@@@@ 1 vm_object_deallocate+0x800 1024 |@@@@@ 1 0xc0000000030ea1fc 2048 |@@@@@@@@@@ 2 _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 99% 0.00 140097 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 512 |@@@@@ 1 __rw_wunlock_hard+0xb0 1024 | 0 vm_object_shadow+0x3a8 2048 | 0 vm_map_lookup+0xb80 4096 |@@@@@ 1 vm_fault_hold+0xf8 8192 | 0 vm_fault+0x94 16384 | 0 trap+0x10ac 32768 | 0 65536 |@@@@@ 1 131072 |@@@@@ 1 262144 | 0 524288 |@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 99% 0.00 18048 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@ 3 sched_switch+0x830 32768 |@@@@@@@@@@ 2 mi_switch+0x158 65536 |@@@@@ 1 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 832 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@ 2 __rw_wlock_hard+0x2cc 512 | 0 vnode_create_vobject+0x18c 1024 |@@@@@@ 1 0xc0000000030c5fe0 2048 |@@@@@@@@@@@@ 2 VOP_OPEN_APV+0x98 exec_check_permissions+0x164 kern_execve+0x348 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 565 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@ 1 __rw_wlock_hard+0x2cc 512 |@@@@@@ 1 vm_imgact_map_page+0x3f0 1024 |@@@@@@@@@@@@@@@@@@ 3 0xc000000002c4733c 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 439967 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@ 1 __rw_wunlock_hard+0xb0 4096 |@@@@@@ 1 vm_imgact_map_page+0x408 8192 | 0 0xc000000002c4733c 16384 | 0 0xc000000002c4684c 32768 | 0 kern_execve+0x678 65536 | 0 sys_execve+0x80 131072 | 0 262144 |@@@@@@ 1 524288 | 0 1048576 |@@@@@@ 1 2097152 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 4337921 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@ 1 __rw_wunlock_hard+0xb0 1024 | 0 vnode_create_vobject+0x3f0 2048 | 0 0xc0000000030c5fe0 4096 | 0 VOP_OPEN_APV+0x98 8192 | 0 exec_check_permissions+0x164 16384 | 0 kern_execve+0x348 32768 | 0 65536 | 0 131072 | 0 262144 | 0 524288 | 0 1048576 |@@@@@@ 1 2097152 |@@@@@@ 1 4194304 | 0 8388608 |@@@@@@ 1 16777216 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 5788568 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@ 2 __rw_wunlock_hard+0xb0 2097152 | 0 vm_object_reference+0x14c 4194304 |@@@@@@ 1 0xc000000002c479cc 8388608 |@@@@@@ 1 0xc000000002c472c0 16777216 | 0 0xc000000002c4684c 33554432 |@@@@@@ 1 kern_execve+0x678 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 1713 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@ 1 0xc000000002cad69c 2048 |@@@@@@@@@@@@@@@@@@ 3 lockmgr_unlock_fast_path+0x13c 4096 |@@@@@@ 1 vop_stdunlock+0x34 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 lookup+0xc54 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 41489 sched lock 65 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@ 1 sched_switch+0x830 32768 |@@@@@@ 1 mi_switch+0x158 65536 |@@@@@@@@@@@@ 2 0xc000000002d6e2b4 131072 |@@@@@@ 1 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 99% 0.00 56810 sched lock 30 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@ 1 sched_switch+0x830 32768 | 0 mi_switch+0x158 65536 |@@@@@@@@@@@@@@@@@@ 3 0xc000000002d6e2b4 131072 | 0 sleepq_wait+0x4c 262144 |@@@@@@ 1 _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 242 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 2 __rw_wlock_hard+0x2cc 512 |@@@@@@@@@@@@@@@ 2 vm_object_deallocate+0x814 0xc0000000030ea1fc _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 107477 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@ 1 __rw_wunlock_hard+0xb0 2048 | 0 vm_object_reference+0x14c 4096 | 0 kern_execve+0x370 8192 |@@@@@@@@@@@@@@@ 2 sys_execve+0x80 16384 | 0 trap+0xb80 32768 | 0 powerpc_interrupt+0x1a4 65536 | 0 131072 | 0 262144 | 0 524288 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 18949 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@ 1 0xc000000002cac3fc 32768 |@@@@@@@@@@@@@@@@@@@@@@ 3 lockmgr_lock_fast_path+0xe4 vop_stdlock+0x3c VOP_LOCK1_APV+0x98 _vn_lock+0x70 vget+0x6c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 42191 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@ 1 sched_idletd+0x2b4 1024 | 0 fork_exit+0xdc 2048 | 0 0xc000000003185c5c 4096 | 0 0xc000000002500350 8192 | 0 16384 | 0 32768 | 0 65536 |@@@@@@@@@@@@@@@ 2 131072 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 22787 sched lock 70 (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@ 1 sched_switch+0x830 16384 | 0 mi_switch+0x158 32768 |@@@@@@@@@@@@@@@ 2 0xc000000002d6e2b4 65536 |@@@@@@@ 1 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 90352 sched lock 25 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@ 1 sched_idletd+0x2b4 512 | 0 fork_exit+0xdc 1024 | 0 0xc000000003185c5c 2048 | 0 0xc000000002500350 4096 |@@@@@@@ 1 8192 | 0 16384 | 0 32768 | 0 65536 | 0 131072 |@@@@@@@ 1 262144 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 354 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 3 __rw_wlock_hard+0x2cc vm_object_shadow+0x378 vm_map_lookup+0xb80 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 1112 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@ 1 0xc000000002ce6044 2048 |@@@@@@@@@@@@@@@@@@@@ 2 __rw_rlock_int+0x9c 0xc0000000030ebe78 vm_map_insert+0x6a8 vm_map_fixed+0x120 0xc000000002c47a14 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 268 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc 512 |@@@@@@@@@@@@@@@@@@@@ 2 vm_object_shadow+0x378 vm_map_lookup+0xb80 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 755 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc 512 | 0 vm_object_deallocate+0x814 1024 |@@@@@@@@@@ 1 0xc0000000030ea1fc 2048 |@@@@@@@@@@ 1 _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 503 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@ 2 wakeup_one+0x2c 512 | 0 taskqueue_thread_enqueue+0x24 1024 | 0 0xc000000002d740c8 2048 |@@@@@@@@@@ 1 taskqueue_enqueue+0x128 taskq_dispatch+0xb4 dmu_objset_do_userquota_updates+0x1ac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 22339 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 1 _sx_xlock_hard+0x354 512 | 0 _vm_map_lock+0x138 1024 | 0 vm_map_protect+0xc4 2048 | 0 sys_mprotect+0x80 4096 | 0 trap+0xb80 8192 | 0 powerpc_interrupt+0x1a4 16384 | 0 32768 |@@@@@@@@@@ 1 65536 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 39108 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1 0xc000000002cac3fc 65536 |@@@@@@@@@@@@@@@@@@@@ 2 __lockmgr_args+0x1b4 0xc0000000030b328c VOP_LOCK1_APV+0x98 _vn_lock+0x70 vget+0x6c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 48564 sched lock 65 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1 sched_idletd+0x2b4 65536 |@@@@@@@@@@ 1 fork_exit+0xdc 131072 |@@@@@@@@@@ 1 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 3975 sched lock 50 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@ 2 mi_switch+0x158 8192 |@@@@@@@@@@ 1 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 146582 sched lock 5 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@ 1 sched_switch+0x830 131072 |@@@@@@@@@@ 1 mi_switch+0x158 262144 | 0 0xc000000002d6e2b4 524288 |@@@@@@@@@@ 1 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 26302 sched lock 30 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 1 sched_idletd+0x2b4 512 | 0 fork_exit+0xdc 1024 |@@@@@@@@@@ 1 0xc000000003185c5c 2048 | 0 0xc000000002500350 4096 | 0 8192 | 0 16384 | 0 32768 | 0 65536 | 0 131072 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 76786 sched lock 30 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@ 1 sched_switch+0x830 4096 | 0 mi_switch+0x158 8192 | 0 0xc000000002d6e2b4 16384 | 0 sleepq_wait+0x4c 32768 | 0 _sleep+0x248 65536 |@@@@@@@@@@ 1 taskqueue_thread_loop+0x1b4 131072 | 0 262144 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 732 sched lock 28 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 1 sched_wakeup+0x198 512 | 0 setrunnable+0x7c 1024 |@@@@@@@@@@@@@@@@@@@@ 2 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 32917 sched lock 25 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@ 1 sched_switch+0x830 4096 | 0 mi_switch+0x158 8192 | 0 0xc000000002d6e2b4 16384 | 0 sleepq_wait+0x4c 32768 |@@@@@@@@@@ 1 _sleep+0x248 65536 | 0 taskqueue_thread_loop+0x1b4 131072 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 18664 sched lock 25 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@ 2 mi_switch+0x158 16384 | 0 0xc000000002d6e2b4 32768 | 0 sleepq_wait+0x4c 65536 |@@@@@@@@@@ 1 _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 198702 sched lock 15 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@ 1 sched_idletd+0x2b4 32768 | 0 fork_exit+0xdc 65536 | 0 0xc000000003185c5c 131072 | 0 0xc000000002500350 262144 |@@@@@@@@@@ 1 524288 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 50793 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 1 sched_switch+0x830 1024 | 0 mi_switch+0x158 2048 | 0 turnstile_wait+0x404 4096 | 0 __rw_wlock_hard+0x444 8192 |@@@@@@@@@@ 1 vm_object_deallocate+0x814 16384 | 0 0xc0000000030ea1fc 32768 | 0 65536 | 0 131072 | 0 262144 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 13071 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@ 2 sched_switch+0x830 4096 | 0 mi_switch+0x158 8192 | 0 turnstile_wait+0x404 16384 | 0 __rw_wlock_hard+0x444 32768 | 0 vm_fault_hold+0x1b28 65536 |@@@@@@@@@@ 1 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 28332 sched lock 10 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@ 1 sched_idletd+0x2b4 16384 | 0 fork_exit+0xdc 32768 | 0 0xc000000003185c5c 65536 |@@@@@@@@@@@@@@@@@@@@ 2 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 265 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc 512 |@@@@@@@@@@@@@@@ 1 vm_imgact_map_page+0x3f0 0xc000000002c4733c 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 6070 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc 2048 | 0 0xc000000003097404 4096 | 0 softdep_freefile+0x6f0 8192 | 0 ffs_vfree+0x44 16384 |@@@@@@@@@@@@@@@ 1 ufs_inactive+0x598 VOP_INACTIVE_APV+0x98 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 1076 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 0xc000000002ce6044 512 | 0 __rw_rlock_int+0x9c 1024 | 0 vm_map_entry_set_vnode_text+0x4c 2048 |@@@@@@@@@@@@@@@ 1 0xc0000000030ea1e4 _vm_map_unlock+0xf8 vmspace_exit+0x194 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 212 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 __rw_wlock_hard+0x2cc 0xc0000000030f0be8 vm_map_delete+0x1bc vmspace_exit+0x184 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 2631832 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@@@@ 1 0xc000000002ce688c 2097152 | 0 _rw_runlock_cookie_int+0xa8 4194304 | 0 0xc0000000030ec13c 8388608 |@@@@@@@@@@@@@@@ 1 vm_map_insert+0x6a8 vm_map_fixed+0x120 0xc000000002c47a14 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 7398017 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@@@@ 1 __rw_wunlock_hard+0xb0 2097152 | 0 vm_object_deallocate+0x830 4194304 | 0 0xc0000000030ea1fc 8388608 | 0 _vm_map_unlock+0xf8 16777216 |@@@@@@@@@@@@@@@ 1 vmspace_exit+0x194 exit1+0x954 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 17456270 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@ 1 __rw_wunlock_hard+0xb0 33554432 |@@@@@@@@@@@@@@@ 1 0xc0000000030f0b08 vm_map_delete+0x1bc vmspace_exit+0x184 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 26054 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 __rw_wunlock_hard+0xb0 0xc00000000309f9a0 0xc000000003085264 0xc000000003089c88 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 302 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 cv_broadcastpri+0x40 512 |@@@@@@@@@@@@@@@ 1 0xc000000002d8affc 0xc000000002d8d5b0 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 21987 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 wakeup+0x2c 0xc000000002d7515c taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 1995 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 0xc000000002cad69c 2048 | 0 lockmgr_unlock_fast_path+0x13c 4096 |@@@@@@@@@@@@@@@ 1 vop_stdunlock+0x34 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 lookup+0xe88 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 59496 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 131072 |@@@@@@@@@@@@@@@ 1 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_reference+0x134 0xc000000002c479cc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 25429 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 32768 | 0 mi_switch+0x158 65536 |@@@@@@@@@@@@@@@ 1 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 31979 sched lock 70 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 mi_switch+0x158 65536 |@@@@@@@@@@@@@@@ 1 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 32969 sched lock 65 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 65536 |@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 11401 sched lock 55 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 1024 | 0 fork_exit+0xdc 2048 | 0 0xc000000003185c5c 4096 | 0 0xc000000002500350 8192 | 0 16384 | 0 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 98481 sched lock 52 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 32297 sched lock 5 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 65536 |@@@@@@@@@@@@@@@ 1 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 4439 sched lock 5 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 1 mi_switch+0x158 8192 |@@@@@@@@@@@@@@@ 1 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 43939 sched lock 45 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 8192 | 0 mi_switch+0x158 16384 | 0 0xc000000002d6e2b4 32768 | 0 sleepq_wait+0x4c 65536 | 0 _sleep+0x248 131072 |@@@@@@@@@@@@@@@ 1 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 41189 sched lock 40 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 8192 | 0 mi_switch+0x158 16384 | 0 0xc000000002d6e2b4 32768 | 0 sleepq_wait+0x4c 65536 | 0 _sleep+0x248 131072 |@@@@@@@@@@@@@@@ 1 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 44587 sched lock 35 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 2048 | 0 fork_exit+0xdc 4096 | 0 0xc000000003185c5c 8192 | 0 0xc000000002500350 16384 | 0 32768 | 0 65536 | 0 131072 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 18580 sched lock 30 (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 4096 | 0 mi_switch+0x158 8192 | 0 0xc000000002d6e2b4 16384 | 0 sleepq_wait+0x4c 32768 | 0 _sleep+0x248 65536 |@@@@@@@@@@@@@@@ 1 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 141297 sched lock 30 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 65536 | 0 mi_switch+0x158 131072 | 0 0xc000000002d6e2b4 262144 | 0 sleepq_wait+0x4c 524288 |@@@@@@@@@@@@@@@ 1 _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 40480 sched lock 20 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 4096 | 0 mi_switch+0x158 8192 | 0 0xc000000002d6e2b4 16384 | 0 sleepq_wait+0x4c 32768 | 0 _sleep+0x248 65536 | 0 taskqueue_thread_loop+0x1b4 131072 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 190490 sched lock 15 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 131072 | 0 mi_switch+0x158 262144 | 0 turnstile_wait+0x404 524288 |@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 175220 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 262144 | 0 mi_switch+0x158 524288 |@@@@@@@@@@@@@@@ 1 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_reference+0x134 0xc000000002c479cc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 20681 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_map_insert+0x758 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 45245 sched lock 0 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 4096 | 0 fork_exit+0xdc 8192 | 0 0xc000000003185c5c 16384 | 0 0xc000000002500350 32768 | 0 65536 | 0 131072 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 620 sched lock 0 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 sched_switch+0x830 1024 |@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 39699 turnstile lock turnstile_lookup+0xd4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_unlock_sleep+0xc8 moea64_remove_pages+0x1e8 pmap_remove_pages+0x80 vmspace_exit+0x100 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 371 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc vm_object_reference+0x134 kern_execve+0x370 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 44341 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc kmem_back_domain+0x340 kmem_malloc_domainset+0xd0 uma_large_malloc_domain+0x148 uma_large_malloc+0x28 malloc+0x1a0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 40246 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_lock_sleep+0x138 taskqueue_enqueue+0x158 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 0xc0000000026bb048 zio_execute+0x19c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 79601 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_lock_sleep+0x138 taskqueue_enqueue+0x158 taskq_dispatch+0xb4 dmu_objset_sync+0x2b8 dsl_pool_sync+0x450 spa_sync+0xc08 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 201 turnstile lock turnstile_trywait+0x150 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc exec_map_first_page+0x780 kern_execve+0x37c sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 729 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc vm_object_shadow+0x390 vm_map_lookup+0xb80 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 44316 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_unlock_sleep+0xc8 moea64_kextract+0x18c pmap_kextract+0x80 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 41958 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_lock_sleep+0x138 0xc000000002d74f5c taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 220 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc exec_map_first_page+0x780 kern_execve+0x37c sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 240 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc vm_object_reference+0x134 kern_execve+0x370 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 88128 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wlock_hard+0x2cc kmem_back_domain+0x340 kmem_malloc_domainset+0xd0 uma_large_malloc_domain+0x148 uma_large_malloc+0x28 malloc+0x1a0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 551 turnstile chain turnstile_trywait+0x208 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_lock_sleep+0x138 moea64_kextract+0x174 pmap_kextract+0x80 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 682 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wunlock_hard+0xb0 exec_map_first_page+0x7fc kern_execve+0x37c sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 90057 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_unlock_sleep+0xb0 0xc000000002d75050 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 94370 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_unlock_sleep+0xb0 0xc000000002cbb96c _sleep+0x168 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 42728 turnstile chain turnstile_chain_lock+0xdc nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_unlock_sleep+0xb0 0xc000000002d74078 taskqueue_enqueue+0x128 taskq_dispatch_ent+0x50 spa_taskq_dispatch_ent+0xb0 0xc0000000026bb048 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 14202453 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wunlock_hard+0xb0 vm_map_insert+0x2a4 vm_map_fixed+0x120 0xc000000002c47a14 0xc000000002c472c0 0xc000000002c4684c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 189 td_contested turnstile_broadcast+0x1bc nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_unlock_sleep+0xd8 0xc000000002cbb96c _sleep+0x168 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 260 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 wakeup_one+0x2c taskqueue_thread_enqueue+0x24 0xc000000002d740c8 taskqueue_enqueue+0x128 taskq_dispatch+0xb4 metaslab_sync_reassess+0x2d8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 364 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002cfbe18 _sx_sunlock_int+0xa8 vm_map_lookup_done+0xa0 0xc0000000030e3cb0 vm_fault_hold+0x22f0 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22443 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xunlock_hard+0xb0 vdev_queue_io_done+0x220 0xc0000000026be130 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11409 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xunlock_hard+0xb0 _vm_map_unlock+0x118 vm_map_find+0x520 vm_map_find_min+0x98 vm_mmap_object+0x46c kern_mmap+0x450 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11432 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 wakeup+0x2c 0xc000000002cd8fec 0xc000000002cd9174 0xc000000002e31530 0xc000000002d873c8 sys_read+0xf4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22113 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sleep+0x118 0xc00000000310bf3c 0xc00000000310bb78 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1783 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 wakeup+0x2c 0xc000000002d8bf90 0xc000000002d873c8 sys_read+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 234515 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_lock_sleep+0x138 0xc000000002cbb898 _sleep+0x290 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 10263928 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wunlock_hard+0xb0 vm_object_deallocate+0xb9c 0xc0000000030ea1fc _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 6219029 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 8388608 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __rw_wunlock_hard+0xb0 vm_object_shadow+0x3c0 vm_map_lookup+0xb80 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 286681 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_lock_sleep+0x138 0xc000000002d74f5c taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 40863 turnstile chain (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 __mtx_lock_sleep+0x138 moea64_kextract+0x174 pmap_kextract+0x80 free+0x44 zfs_kmem_free+0x28 zio_buf_free+0xac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 9653 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_imgact_map_page+0x3f0 0xc000000002c4733c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 127115 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 89596 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sx_xlock_hard+0x4d8 vdev_queue_io+0x268 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 80534 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sx_xlock_hard+0x4d8 vdev_queue_io_done+0x12c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 39750 sched lock 70 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 9904 sched lock 70 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 480 sched lock 67 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x34c 0xc000000002ce6978 _rw_runlock_cookie_int+0xa8 0xc0000000030ec13c vm_map_insert+0x6a8 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 161015 sched lock 65 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 0xc0000000030f0be8 vm_map_delete+0x1bc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1189 sched lock 65 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_reference+0x134 0xc000000002c479cc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3177 sched lock 68 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 148 sched lock 68 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x34c 0xc000000002ce6978 _rw_runlock_cookie_int+0xa8 0xc0000000030ec13c vm_map_insert+0x6a8 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 10514 sched lock 7 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3036 sched lock 60 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 494 sched lock 6 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x34c 0xc000000002ce6978 _rw_runlock_cookie_int+0xa8 vm_map_entry_set_vnode_text+0x10c 0xc0000000030ea1e4 _vm_map_unlock+0xf8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 19706 sched lock 55 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 170 sched lock 53 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x34c 0xc000000002ce6978 _rw_runlock_cookie_int+0xa8 0xc0000000030ec13c vm_map_insert+0x6a8 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 260 sched lock 53 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1480 sched lock 55 (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 80908 sched lock 55 (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sx_xlock_hard+0x4d8 vdev_queue_io+0x268 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 20829 sched lock 63 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 2621 sched lock 61 (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 0xc000000002d6ebbc sleepq_wait_sig+0x2c _cv_wait_sig+0x14c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11159 sched lock 60 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 989 sched lock 60 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 443 sched lock 65 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5814 sched lock 65 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 523 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 wakeup+0x2c 0xc000000002d8d260 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5955 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xlock_hard+0x354 _vm_map_lock+0x138 kern_munmap+0x9c sys_munmap+0x2c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 490 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002cfbe18 _sx_sunlock_int+0xa8 vm_map_lookup_done+0xa0 0xc0000000030e3cb0 vm_fault_hold+0x22f0 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 174 sched lock 71 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 39425 sched lock 71 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_timedwait+0x4c _sleep+0x224 uma_reclaim_worker+0x168 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 348 sched lock 70 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_deallocate+0x814 0xc0000000030ea1fc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1919 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002cad69c lockmgr_unlock_fast_path+0x13c vop_stdunlock+0x34 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 vn_vptocnp+0x44c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 44317 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _cv_timedwait_sbt+0xa8 fork_rfppwait+0x12c trap+0x1028 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 2941 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002cad69c __lockmgr_args+0x424 0xc000000002dee228 bqrelse+0x18c ffs_vgetf+0x414 ffs_vget+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 750 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xlock_hard+0x354 vdev_queue_io+0x268 0xc0000000026bdce0 zio_execute+0x19c zio_nowait+0x118 0xc000000002653d80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 252 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xlock_hard+0x354 0xc0000000025a6044 0xc00000000258e800 0xc0000000026bd790 zio_execute+0x19c taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5427 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xlock_hard+0x354 vdev_queue_io_done+0x278 0xc0000000026be130 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 230 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xlock_hard+0x354 _vm_map_lock+0x138 kern_munmap+0x9c sys_munmap+0x2c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 44407 sleepq chain sleepq_lock+0xe4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 _sx_xlock_hard+0x354 _vm_map_lock_upgrade+0x180 vm_map_lookup+0xd68 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 162 sched lock 21 (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 160 sched lock 20 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 0xc0000000030f0be8 vm_map_delete+0x1bc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1744 sched lock 20 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 982 sched lock 20 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_timedwait+0x4c _sleep+0x224 0xc000000003089d44 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 13324 sched lock 20 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 0xc000000002ce6134 __rw_rlock_int+0x9c 0xc0000000030ebe78 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 148098 sched lock 15 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_imgact_map_page+0x3f0 0xc000000002c4733c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 190256 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 0xc0000000030f0be8 vm_map_delete+0x1bc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 9420 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 0xc000000002ce6134 __rw_rlock_int+0x9c vm_map_entry_set_vnode_text+0x4c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 17301 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vnode_create_vobject+0x18c 0xc0000000030c5fe0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 6562 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 0xc000000002ce6134 __rw_rlock_int+0x9c 0xc0000000030ebe78 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 526679 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 682 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 0xc000000002d6ebbc sleepq_wait_sig+0x2c _cv_wait_sig+0x14c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 656 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 44362 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 68444 sched lock 10 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_reference+0x134 kern_execve+0x370 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 121606 sched lock 10 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __mtx_lock_sleep+0x1d8 0xc000000002d74f5c taskqueue_thread_loop+0x158 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 75748 sched lock 12 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 10659 sched lock 15 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 9710 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 323493 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_shadow+0x390 vm_map_lookup+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 165232 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 118502 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_quiesce+0x1c4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5876 sched lock 15 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22158 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 49305 sched lock 15 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_timedwait+0x4c _cv_timedwait_sbt+0x17c fork_rfppwait+0x12c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 182 sched lock 0 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x34c 0xc000000002ce6978 _rw_runlock_cookie_int+0xa8 0xc0000000030ec13c vm_map_insert+0x6a8 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 986 sched lock 0 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_deallocate+0x800 0xc0000000030ea1fc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 791 sched lock 0 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 0xc000000002ce6134 __rw_rlock_int+0x9c 0xc0000000030ebe78 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 24816 callout 0xc000000002d1384c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 callout_reset_sbt_on+0xb8 sleepq_set_timeout_sbt+0x104 _cv_timedwait_sbt+0x114 0xc00000000259c048 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 676 callout 0xc000000002d1384c nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 callout_reset_sbt_on+0xb8 sleepq_set_timeout_sbt+0x104 _cv_timedwait_sbt+0x114 fork_rfppwait+0x12c trap+0x1028 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 396 sched lock 0 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 0xc000000002d6def4 0xc000000002d12720 callout_process+0x21c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 81978 sched lock 0 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_map_insert+0x758 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 961 sched lock 0 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 45499 sched lock 0 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_timedwait+0x4c _cv_timedwait_sbt+0x17c fork_rfppwait+0x12c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3165 sched lock 0 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 exec_map_first_page+0x780 kern_execve+0x37c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 658 sched lock 1 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 0xc000000002d6def4 0xc000000002d12720 callout_process+0x21c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22338 sched lock 10 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 10584 sched lock 10 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 80758 sched lock 10 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1029 sched lock 35 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 9278 sched lock 35 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_reference+0x134 0xc000000002c479cc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 6261 sched lock 33 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3090 sched lock 35 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 35783 sched lock 30 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 19902 sched lock 30 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 272753 sched lock 3 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 590 sched lock 26 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 76763 sched lock 30 sched_switch+0xca0 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 10204 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_shadow+0x390 vm_map_lookup+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11362 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 0xc000000002d6ebbc sleepq_wait_sig+0x2c _cv_wait_sig+0x14c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 2850 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_deallocate+0x800 0xc0000000030ea1fc _vm_map_unlock+0xf8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 425 sched lock 23 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_reference+0x134 0xc000000002c479cc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1265 sched lock 21 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 10381 sched lock 23 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1428 sched lock 24 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 44271 sched lock 25 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 75989 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 79504 sched lock 25 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_object_reference+0x134 0xc000000002c479cc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 639 sched lock 25 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_timedwait+0x4c _sleep+0x224 0xc000000002df2a30 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 149111 sched lock 25 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 24804 sched lock 48 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x34c __rw_wunlock_hard+0x108 vm_object_deallocate+0x830 0xc0000000030ea1fc _vm_map_unlock+0xf8 vmspace_exit+0x194 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 40274 sched lock 46 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 139066 sched lock 45 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 35846 sched lock 45 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 turnstile_wait+0x404 __rw_wlock_hard+0x444 vm_fault_hold+0x1b28 vm_fault+0x94 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 20855 sched lock 43 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 0xc000000002d6ebbc sleepq_timedwait_sig+0x30 _cv_timedwait_sig_sbt+0x17c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 43425 sched lock 41 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1657 sched lock 43 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_broadcast+0x11c cv_broadcastpri+0x80 0xc000000002d8affc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 90133 sched lock 43 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 77294 sched lock 40 (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5761 sched lock 40 (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 994 sched lock 39 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_lend_prio+0xcc 0xc000000002d7ace0 turnstile_wait+0x3e4 __mtx_lock_sleep+0x1d8 0xc000000002cbb898 _sleep+0x290 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 189 sched lock 4 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_broadcast+0x11c wakeup+0x44 exit1+0x17b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1753 sched lock 40 sched_switch+0x81c nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 11036 sched lock 40 (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21175 sched lock 40 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 44374 sched lock 50 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 2990 sched lock 5 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_timedwait+0x4c _sleep+0x224 0xc000000003089d44 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 82999 sched lock 5 (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 609 sched lock 49 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_timedwait+0x4c _cv_timedwait_sbt+0x17c fork_rfppwait+0x12c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 22404 sched lock 5 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 314 sched lock 51 sched_add+0x48c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 0xc000000002d6def4 0xc000000002d12720 callout_process+0x21c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 121523 sched lock 50 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21395 sched lock 50 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002c9255c fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 48176 sched lock 52 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _cv_wait+0x14c 0xc000000002b2f478 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 475 sched lock 52 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_wakeup+0x198 setrunnable+0x7c 0xc000000002d6f770 sleepq_signal+0xd0 wakeup_one+0x44 taskqueue_thread_enqueue+0x24 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 10612 sched lock 52 0xc000000002d353b4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_switch+0x830 mi_switch+0x158 0xc000000002d6e2b4 sleepq_wait+0x4c _sleep+0x248 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 81381 sched lock 52 0xc000000002d353cc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x2b4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Thread lock spin: 337 events in 21.178 seconds (16 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 29 9% 9% 0.00 899 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 256 |@ 1 sleepq_signal+0xbc 512 |@ 1 cv_signal+0x9c 1024 |@@@@@@@@@@@@@@@@@@@@@@@@ 24 usb_proc_msignal+0x1a4 2048 |@@@ 3 usbd_transfer_start+0x210 usbd_do_request_flags+0x6f4 usbd_req_get_port_status+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 22 7% 15% 0.00 4692 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@ 1 sleepq_broadcast+0x108 2048 | 0 wakeup+0x44 4096 |@@@@@@@@@ 7 kern_umtx_wake+0x2f8 8192 |@@@@@@@@@@@@@@@@@@@ 14 0xc000000002d19998 sys__umtx_op+0x48 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 3% 18% 0.00 3363 sched lock 25 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@ 1 sleepq_add+0xec 2048 |@@@@@@@@ 3 _sleep+0x188 4096 |@@@@@@@@ 3 taskqueue_thread_loop+0x1b4 8192 |@@@@@@@@@@ 4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 3% 21% 0.00 3413 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@ 1 sleepq_broadcast+0x108 1024 | 0 cv_broadcastpri+0x80 2048 |@@@@@@@@@@ 3 0xc000000002d8affc 4096 |@@@ 1 0xc000000002d8d5b0 8192 |@@@@@@@@@@@@@ 4 0xc000000002d87de0 sys_write+0xf4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 3% 24% 0.00 3371 sched lock 70 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@ 2 sleepq_add+0xec 4096 |@@@@@@@@@@@@@@@@ 5 _sleep+0x188 8192 |@@@@@@ 2 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 2% 26% 0.00 4989 sched lock 50 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@ 1 sleepq_add+0xec 1024 | 0 _sleep+0x188 2048 | 0 taskqueue_thread_loop+0x1b4 4096 | 0 fork_exit+0xdc 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@ 6 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 2% 28% 0.00 19677 sched lock 25 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@ 1 0xc000000002c92514 8192 |@@@@@@@@@@@@ 3 fork_exit+0xdc 16384 |@@@@ 1 0xc000000003185c5c 32768 |@@@@ 1 65536 | 0 131072 |@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 2% 30% 0.00 3739 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@ 1 sleepq_broadcast+0x108 4096 |@@@@@@@@@@@@@@@ 3 wakeup+0x44 8192 |@@@@@@@@@@ 2 0xc000000002d1b4d8 sys__umtx_op+0x48 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 2% 31% 0.00 3787 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@ 1 sleepq_broadcast+0x108 1024 | 0 wakeup+0x44 2048 |@@@@@ 1 kern_umtx_wake+0x2f8 4096 |@@@@@ 1 0xc000000002d19998 8192 |@@@@@@@@@@@@@@@ 3 sys__umtx_op+0x48 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 2% 33% 0.00 3436 sched lock 30 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 3 sleepq_add+0xec 4096 | 0 _sleep+0x188 8192 |@@@@@@@@@@@@@@@ 3 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 2% 35% 0.00 18103 sched lock 30 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@ 1 0xc000000002c92514 16384 |@@@@@@@@@@ 2 fork_exit+0xdc 32768 |@@@@@@@@@@ 2 0xc000000003185c5c 65536 |@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 1% 36% 0.00 18988 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@ 4 sleepq_broadcast+0x108 16384 | 0 _sx_xunlock_hard+0x108 32768 | 0 vdev_queue_io_done+0x220 65536 | 0 0xc0000000026be130 131072 |@@@@@@ 1 zio_execute+0x19c taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 1% 38% 0.00 2393 sched lock 60 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@ 1 sleepq_add+0xec 2048 |@@@@@@@@@@@@ 2 _sleep+0x188 4096 |@@@@@@ 1 taskqueue_thread_loop+0x1b4 8192 |@@@@@@ 1 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 39% 0.00 4165 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@ 1 sleepq_broadcast+0x108 4096 |@@@@@@@ 1 cv_broadcastpri+0x80 8192 |@@@@@@@@@@@@@@@ 2 0xc000000002635aa0 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 40% 0.00 3857 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@ 1 sleepq_broadcast+0x108 2048 | 0 wakeup+0x44 4096 |@@@@@@@ 1 kern_umtx_wake+0x2f8 8192 |@@@@@@@@@@@@@@@ 2 0xc000000002d1aaec sys__umtx_op+0x48 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 42% 0.00 18173 sched lock 70 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@ 1 0xc000000002c92514 32768 |@@@@@@@@@@@@@@@@@@@@@@ 3 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 43% 0.00 32931 sched lock 65 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@ 2 0xc000000002c92514 32768 |@@@@@@@ 1 fork_exit+0xdc 65536 | 0 0xc000000003185c5c 131072 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 44% 0.00 5877 sched lock 55 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 2 sleepq_add+0xec 8192 |@@@@@@@ 1 _sleep+0x188 16384 |@@@@@@@ 1 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 45% 0.00 5174 sched lock 40 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@ 1 sleepq_add+0xec 8192 |@@@@@@@@@@@@@@@@@@@@@@ 3 _sleep+0x188 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 46% 0.00 11705 sched lock 15 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@ 1 0xc000000002c92514 16384 |@@@@@@@@@@@@@@@@@@@@@@ 3 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 1% 47% 0.00 3199 sched lock 10 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@ 1 sleepq_add+0xec 2048 | 0 _sleep+0x188 4096 |@@@@@@@@@@@@@@@ 2 taskqueue_thread_loop+0x1b4 8192 |@@@@@@@ 1 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 48% 0.00 43772 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@ 1 sleepq_broadcast+0x108 8192 |@@@@@@@@@@ 1 wakeup+0x44 16384 | 0 0xc000000002d7515c 32768 | 0 taskqueue_thread_loop+0x158 65536 | 0 fork_exit+0xdc 131072 |@@@@@@@@@@ 1 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 49% 0.00 1763 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 1 sleepq_broadcast+0x108 1024 | 0 wakeup+0x44 2048 |@@@@@@@@@@ 1 0xc000000002d7515c 4096 |@@@@@@@@@@ 1 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 50% 0.00 13815 sched lock 70 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@ 1 sched_preempt+0x44 2048 | 0 powerpc_ipi_handler+0xfc 4096 | 0 intr_event_handle+0x104 8192 | 0 powerpc_dispatch_intr+0xe4 16384 | 0 0xc00000000316df7c 32768 |@@@@@@@@@@@@@@@@@@@@ 2 powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 51% 0.00 5679 sched lock 65 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@ 2 sleepq_add+0xec 8192 | 0 _sleep+0x188 16384 |@@@@@@@@@@ 1 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 52% 0.00 7798 sched lock 52 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 1 sched_preempt+0x44 1024 |@@@@@@@@@@ 1 powerpc_ipi_handler+0xfc 2048 | 0 intr_event_handle+0x104 4096 | 0 powerpc_dispatch_intr+0xe4 8192 | 0 0xc00000000316df7c 16384 | 0 powerpc_interrupt+0x138 32768 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 53% 0.00 14855 sched lock 5 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@ 2 0xc000000002c92514 32768 |@@@@@@@@@@ 1 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 54% 0.00 1265 sched lock 45 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@ 1 sleepq_add+0xec 1024 | 0 _sleep+0x188 2048 |@@@@@@@@@@@@@@@@@@@@ 2 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 55% 0.00 4942 sched lock 15 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@ 1 sleepq_add+0xec 8192 |@@@@@@@@@@@@@@@@@@@@ 2 _sleep+0x188 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 1% 55% 0.00 12194 sched lock 0 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@ 1 0xc000000002c92514 4096 | 0 fork_exit+0xdc 8192 | 0 0xc000000003185c5c 16384 |@@@@@@@@@@ 1 32768 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 56% 0.00 6489 turnstile lock _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 turnstile_unpend+0x310 0xc000000002ce6978 _rw_runlock_cookie_int+0xa8 vm_map_entry_set_vnode_text+0x10c 0xc0000000030ea1e4 _vm_map_unlock+0xf8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 57% 0.00 4529 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 8192 |@@@@@@@@@@@@@@@ 1 wakeup+0x44 kern_umtx_wake+0x2f8 0xc000000002d1aaec sys__umtx_op+0x48 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 57% 0.00 6565 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 4096 | 0 0xc000000002cfbef4 8192 | 0 _sx_sunlock_int+0xa8 16384 |@@@@@@@@@@@@@@@ 1 vm_map_lookup_done+0xa0 0xc0000000030e3cb0 vm_fault_hold+0x22f0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 58% 0.00 124839 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 262144 |@@@@@@@@@@@@@@@ 1 _sx_xunlock_hard+0x108 vdev_queue_io+0x280 0xc0000000026bdce0 zio_execute+0x19c zio_nowait+0x118 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 58% 0.00 5962 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 4096 | 0 0xc000000002cad710 8192 | 0 __lockmgr_args+0x424 16384 |@@@@@@@@@@@@@@@ 1 0xc000000002dee228 bqrelse+0x18c bdwrite+0x2a8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 59% 0.00 3330 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 0xc000000002d6de34 2048 | 0 0xc000000002d12720 4096 | 0 callout_process+0x21c 8192 |@@@@@@@@@@@@@@@ 1 0xc000000003132100 cpu_activeclock+0xa8 cpu_idle+0x84 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 60% 0.00 336 sched lock 70 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 60% 0.00 20046 sched lock 65 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 1024 | 0 powerpc_ipi_handler+0xfc 2048 | 0 intr_event_handle+0x104 4096 | 0 powerpc_dispatch_intr+0xe4 8192 | 0 0xc00000000316df7c 16384 | 0 powerpc_interrupt+0x138 32768 | 0 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 61% 0.00 711 sched lock 61 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 1024 | 0 powerpc_ipi_handler+0xfc 2048 |@@@@@@@@@@@@@@@ 1 intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 61% 0.00 12192 sched lock 55 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 1 0xc000000002c92514 8192 | 0 fork_exit+0xdc 16384 | 0 0xc000000003185c5c 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 62% 0.00 308 sched lock 55 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 63% 0.00 760 sched lock 5 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 63% 0.00 10724 sched lock 45 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 0xc000000002c92514 1024 | 0 fork_exit+0xdc 2048 | 0 0xc000000003185c5c 4096 | 0 8192 | 0 16384 | 0 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 64% 0.00 22361 sched lock 41 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 statclock+0x124 1024 | 0 0xc000000003132054 2048 | 0 cpu_activeclock+0xa8 4096 | 0 cpu_idle+0x84 8192 | 0 sched_idletd+0x3d4 16384 | 0 fork_exit+0xdc 32768 | 0 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 64% 0.00 11540 sched lock 40 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 0xc000000002c92514 4096 | 0 fork_exit+0xdc 8192 | 0 0xc000000003185c5c 16384 | 0 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 65% 0.00 21530 sched lock 25 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 2048 | 0 powerpc_ipi_handler+0xfc 4096 | 0 intr_event_handle+0x104 8192 | 0 powerpc_dispatch_intr+0xe4 16384 | 0 0xc00000000316df7c 32768 | 0 powerpc_interrupt+0x138 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 66% 0.00 858 sched lock 21 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 2048 |@@@@@@@@@@@@@@@ 1 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 66% 0.00 612 sched lock 15 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 67% 0.00 41999 sched lock 15 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 67% 0.00 386 sched lock 13 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 68% 0.00 44421 sched lock 13 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 69% 0.00 1924 sched lock 10 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 0xc000000002c92514 2048 | 0 fork_exit+0xdc 4096 |@@@@@@@@@@@@@@@ 1 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 69% 0.00 5748 sched lock 0 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 2048 | 0 powerpc_ipi_handler+0xfc 4096 | 0 intr_event_handle+0x104 8192 | 0 powerpc_dispatch_intr+0xe4 16384 |@@@@@@@@@@@@@@@ 1 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 1% 70% 0.00 20155 sched lock 0 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 0xc000000002c90f84 1024 | 0 swi_sched+0xac 2048 | 0 callout_process+0x338 4096 | 0 0xc000000003132100 8192 | 0 0xc000000003132b28 16384 | 0 decr_intr+0x110 32768 | 0 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 70% 0.00 1028 turnstile lock _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x310 0xc000000002ce6978 _rw_runlock_cookie_int+0xa8 0xc0000000030ec13c vm_map_insert+0x6a8 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 70% 0.00 80869 turnstile lock _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x310 __mtx_unlock_sleep+0xe4 0xc000000002d75050 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 71% 0.00 5548 turnstile lock _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x310 __mtx_unlock_sleep+0xe4 0xc000000002cbb96c _sleep+0x168 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 71% 0.00 10942 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 wakeup+0x44 0xc00000000309aa64 0xc000000003087890 bufdone+0x3b8 0xc000000002c2bfb0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 71% 0.00 5841 turnstile lock (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 turnstile_unpend+0x310 __mtx_unlock_sleep+0xe4 0xc000000002cbb96c _sleep+0x168 taskqueue_thread_loop+0x1b4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 72% 0.00 5929 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 wakeup+0x44 0xc000000002d8bf90 0xc000000002d873c8 sys_read+0xf4 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 72% 0.00 10020 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 0xc000000002cad73c lockmgr_unlock_fast_path+0x13c vop_stdunlock+0x34 VOP_UNLOCK_APV+0x98 0xc000000002e1aa88 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 72% 0.00 3097 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 cv_broadcastpri+0x80 0xc000000002d8affc 0xc000000002daf64c ttydisc_write+0x43c 0xc000000002da9ef4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 72% 0.00 22604 sleepq chain _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d6de34 0xc000000002d12720 callout_process+0x21c 0xc000000003132100 0xc000000003132b28 decr_intr+0x110 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 73% 0.00 469 sleepq chain (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_broadcast+0x108 wakeup+0x44 0xc000000002d751a0 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 73% 0.00 512 sched lock 8 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 73% 0.00 1102 sched lock 71 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c cpu_idle+0xc0 sched_idletd+0x3d4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 74% 0.00 730 sched lock 71 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 74% 0.00 40291 sched lock 9 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d7ac84 turnstile_wait+0x3e4 __mtx_lock_sleep+0x1d8 0xc000000002d74f5c taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 74% 0.00 43932 sched lock 7 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 74% 0.00 306 sched lock 7 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 75% 0.00 44516 sched lock 68 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 75% 0.00 21005 sched lock 65 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_wait+0x3c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 75% 0.00 39615 sched lock 68 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 76% 0.00 1790 sched lock 68 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d7ac84 turnstile_wait+0x3e4 __rw_wlock_hard+0x444 0xc0000000030e8b68 kmem_free+0x44 uma_large_free+0x30 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 76% 0.00 6970 sched lock 65 (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 76% 0.00 353 sched lock 63 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 77% 0.00 41074 sched lock 61 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_userret_slowpath+0x30 userret+0x188 trap+0xd4c powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 77% 0.00 3488 sched lock 65 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 0xc000000003132b28 decr_intr+0x110 powerpc_interrupt+0xa4 0xc0000000030a2860 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 77% 0.00 461 sched lock 55 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 77% 0.00 809 sched lock 54 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_add+0xec _sleep+0x188 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 78% 0.00 498 sched lock 54 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 78% 0.00 80085 sched lock 55 (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 78% 0.00 1117 sched lock 55 (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c powerpc_interrupt+0x26c dtrace_interrupt_enable+0xc dtrace_probe+0x5e4 _sx_xlock_hard+0xa68 vdev_queue_io+0x268 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 79% 0.00 44404 sched lock 60 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 79% 0.00 355 sched lock 60 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 79% 0.00 721 sched lock 59 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 80% 0.00 29883 sched lock 6 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 80% 0.00 5894 sched lock 60 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c92514 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 80% 0.00 1176 sched lock 60 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c90f84 intr_event_handle+0x16c powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 0xc000000003174a68 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 80% 0.00 326 sched lock 60 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 81% 0.00 44436 sched lock 54 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 81% 0.00 762 sched lock 54 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 81% 0.00 506 sched lock 54 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 82% 0.00 36051 sched lock 53 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 82% 0.00 44383 sched lock 54 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 82% 0.00 11749 sched lock 51 (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 82% 0.00 39976 sched lock 53 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 83% 0.00 537 sched lock 49 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 83% 0.00 377 sched lock 47 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_idletd+0x108 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 83% 0.00 1869 sched lock 46 (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c cpu_idle+0xc0 sched_idletd+0x3d4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 84% 0.00 5671 sched lock 46 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_add+0xec _sleep+0x188 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 84% 0.00 3052 sched lock 5 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_add+0xec _sleep+0x188 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 84% 0.00 39032 sched lock 50 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 85% 0.00 1061 sched lock 40 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d7ac84 turnstile_wait+0x3e4 __mtx_lock_sleep+0x1d8 moea64_enter+0x918 pmap_enter+0xbc kmem_back_domain+0x200 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 85% 0.00 6369 sched lock 40 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_wait+0x3c _sleep+0x248 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 85% 0.00 79131 sched lock 4 (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 85% 0.00 3142 sched lock 35 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_add+0xec _sleep+0x188 taskqueue_thread_loop+0x1b4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 86% 0.00 957 sched lock 40 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c90f84 intr_event_handle+0x16c powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 0xc000000003174a68 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 86% 0.00 22886 sched lock 43 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 86% 0.00 955 sched lock 42 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c cpu_idle+0xc0 sched_idletd+0x3d4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 87% 0.00 527 sched lock 41 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c cpu_idle+0xc0 sched_idletd+0x3d4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 87% 0.00 479 sched lock 41 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 87% 0.00 3433 sched lock 43 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sleepq_add+0xec _cv_timedwait_sig_sbt+0xfc 0xc000000002d8a0c0 kern_poll+0x4c0 sys_poll+0x80 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 88% 0.00 1976 sched lock 43 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_userret_slowpath+0x30 userret+0x188 trap+0xd4c powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 88% 0.00 660 sched lock 45 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c powerpc_interrupt+0x26c lock_delay+0x88 lockstat_nsecs+0x4c __rw_wlock_hard+0x480 0xc0000000030e8b68 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 88% 0.00 19830 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 88% 0.00 41535 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c powerpc_interrupt+0x26c spinlock_exit+0x5c 0xe0000000cacc73b0 sleepq_release+0x74 wakeup_one+0x54 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 89% 0.00 15240 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c90f84 intr_event_handle+0x16c powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 lock_delay+0x88 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 89% 0.00 470 sched lock 23 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 89% 0.00 21395 sched lock 25 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 0xc000000003132b28 decr_intr+0x110 powerpc_interrupt+0xa4 spinlock_exit+0x5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 90% 0.00 1105 sched lock 22 (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 90% 0.00 2146 sched lock 22 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d7ac84 turnstile_wait+0x3e4 __mtx_lock_sleep+0x1d8 0xc000000002cbb898 _sleep+0x290 taskqueue_thread_loop+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 90% 0.00 41777 sched lock 23 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 91% 0.00 79169 sched lock 19 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 91% 0.00 351 sched lock 19 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 91% 0.00 1313 sched lock 16 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 91% 0.00 561 sched lock 17 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 92% 0.00 657 sched lock 2 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 92% 0.00 505 sched lock 2 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 92% 0.00 83428 sched lock 20 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c90f84 intr_event_handle+0x16c powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 0xc000000003174a68 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 93% 0.00 20283 sched lock 20 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c92514 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 93% 0.00 39646 sched lock 26 (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 93% 0.00 41683 sched lock 25 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c powerpc_interrupt+0x26c 0xc0000000026be49c 0xc0000000026be484 zio_execute+0x19c taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 93% 0.00 357 sched lock 25 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c90f84 intr_event_handle+0x16c powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 fget_unlocked+0x114 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 94% 0.00 589 sched lock 25 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c 0xc0000000031405ec 0xc00000000313a5b0 moea64_remove_pages+0x100 pmap_remove_pages+0x80 vmspace_exit+0x100 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 94% 0.00 20091 sched lock 25 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c cpu_idle+0xc0 sched_idletd+0x3d4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 94% 0.00 1148 sched lock 30 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c90f84 intr_event_handle+0x16c powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 0xc000000003174a68 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 95% 0.00 303 sched lock 30 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 95% 0.00 378 sched lock 27 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 95% 0.00 22509 sched lock 28 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 96% 0.00 44472 sched lock 30 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c cpu_idle+0xc0 sched_idletd+0x3d4 fork_exit+0xdc 0xc000000003185c5c 0xc000000002500350 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 96% 0.00 670 sched lock 31 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 96% 0.00 441 sched lock 31 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 96% 0.00 469471 sched lock 1 (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 97% 0.00 981 sched lock 0 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002d7ac84 turnstile_wait+0x3e4 __mtx_lock_sleep+0x1d8 moea64_remove_pages+0x214 pmap_remove_pages+0x80 vmspace_exit+0x100 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 97% 0.00 146966 sched lock 0 (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 97% 0.00 777 sched lock 0 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c cpu_idle+0xc0 sched_idletd+0x3d4 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 98% 0.00 39843 sched lock 14 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 98% 0.00 44384 sched lock 12 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 statclock+0x124 0xc000000003132054 cpu_activeclock+0xa8 cpu_idle+0x84 sched_idletd+0x3d4 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 98% 0.00 314 sched lock 12 (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 554 sched lock 12 (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 20904 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 331 sched lock 14 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_exit_thread+0x30 sched_exit+0x24 exit1+0x17d8 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 289576 sched lock 15 (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 critical_exit_preempt+0x4c powerpc_interrupt+0x26c 0xc0000000030ec034 0xc0000000030ebfcc vm_map_insert+0x6a8 vm_map_fixed+0x120 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 829 sched lock 15 _thread_lock+0xf4 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1828 sched lock 16 (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sched_preempt+0x44 powerpc_ipi_handler+0xfc intr_event_handle+0x104 powerpc_dispatch_intr+0xe4 0xc00000000316df7c powerpc_interrupt+0x138 ------------------------------------------------------------------------------- R/W writer blocked by writer: 952 events in 21.178 seconds (45 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 199 21% 21% 0.00 508695935 vm object vm_fault_hold+0x1b28 nsec ------ Time Distribution ------ count Stack 131072 | 1 vm_fault+0x94 262144 | 0 trap+0x10ac 524288 | 2 powerpc_interrupt+0x1a4 1048576 | 1 2097152 | 0 4194304 | 0 8388608 | 0 16777216 | 3 33554432 |@@ 15 67108864 |@ 9 134217728 | 5 268435456 |@@@@@@ 41 536870912 |@@@@@@ 42 1073741824 |@@@@@@@@@ 62 2147483648 |@@ 17 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 151 16% 37% 0.00 498824479 vm object vm_map_insert+0x758 nsec ------ Time Distribution ------ count Stack 16777216 | 3 vm_map_fixed+0x120 33554432 |@@ 11 0xc000000002c47a14 67108864 |@@ 11 0xc000000002c472c0 134217728 |@ 8 0xc000000002c4684c 268435456 |@@@@@ 27 kern_execve+0x678 536870912 |@@@@@@@ 36 sys_execve+0x80 1073741824 |@@@@@@@ 36 2147483648 |@@@ 18 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 118 12% 49% 0.00 367769728 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 2097152 | 1 0xc0000000030ea1fc 4194304 | 0 _vm_map_unlock+0xf8 8388608 | 0 vmspace_exit+0x194 16777216 |@ 4 exit1+0x954 33554432 |@@@@ 18 sys_sys_exit+0x28 67108864 |@@ 8 trap+0xb80 134217728 |@ 7 268435456 |@@@@@@@@ 32 536870912 |@@@@ 17 1073741824 |@@@@@ 22 2147483648 |@@ 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 78 8% 57% 0.00 624460613 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 262144 | 1 0xc000000002c479cc 524288 | 0 0xc000000002c472c0 1048576 | 0 0xc000000002c4684c 2097152 | 0 kern_execve+0x678 4194304 | 0 sys_execve+0x80 8388608 | 0 trap+0xb80 16777216 | 1 33554432 |@ 4 67108864 |@@ 6 134217728 | 1 268435456 |@@@@ 12 536870912 |@@@@@@ 17 1073741824 |@@@@@@@@ 21 2147483648 |@@@@@ 13 4294967296 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 70 7% 65% 0.00 534935774 vm object 0xc0000000030f0be8 nsec ------ Time Distribution ------ count Stack 262144 | 1 vm_map_delete+0x1bc 524288 | 0 vmspace_exit+0x184 1048576 | 0 exit1+0x954 2097152 | 0 sys_sys_exit+0x28 4194304 | 0 trap+0xb80 8388608 | 0 powerpc_interrupt+0x1a4 16777216 |@ 3 33554432 |@ 4 67108864 | 1 134217728 | 1 268435456 |@@@@@@ 16 536870912 |@@@@@@ 15 1073741824 |@@@@@@@@@ 21 2147483648 |@@@ 7 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 66 7% 72% 0.00 403067013 vm object vm_object_shadow+0x390 nsec ------ Time Distribution ------ count Stack 32768 | 1 vm_map_lookup+0xb80 65536 | 0 vm_fault_hold+0xf8 131072 | 1 vm_fault+0x94 262144 | 1 trap+0x10ac 524288 | 2 powerpc_interrupt+0x1a4 1048576 | 1 2097152 | 0 4194304 | 1 8388608 | 0 16777216 | 0 33554432 |@@ 6 67108864 |@@ 6 134217728 |@ 3 268435456 |@@@@@ 11 536870912 |@@@@@@ 14 1073741824 |@@@@@ 13 2147483648 |@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 58 6% 78% 0.00 438720916 vm object vm_object_shadow+0x378 nsec ------ Time Distribution ------ count Stack 65536 | 1 vm_map_lookup+0xb80 131072 | 0 vm_fault_hold+0xf8 262144 | 1 vm_fault+0x94 524288 |@ 2 trap+0x10ac 1048576 | 0 powerpc_interrupt+0x1a4 2097152 | 0 4194304 |@ 3 8388608 | 0 16777216 | 0 33554432 |@ 2 67108864 |@ 2 134217728 |@ 2 268435456 |@@@@@ 11 536870912 |@@@@@ 11 1073741824 |@@@@@@@@@@ 20 2147483648 |@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 44 5% 82% 0.00 409193984 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 524288 | 1 kern_execve+0x9d0 1048576 | 0 sys_execve+0x80 2097152 | 0 trap+0xb80 4194304 | 0 powerpc_interrupt+0x1a4 8388608 | 0 16777216 | 0 33554432 |@@ 4 67108864 |@@@ 5 134217728 | 1 268435456 |@@@@ 6 536870912 |@@@@@@@@@@ 16 1073741824 |@@@@ 7 2147483648 |@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 36 4% 86% 0.00 558055217 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 16777216 | 1 kern_execve+0x370 33554432 |@@ 3 sys_execve+0x80 67108864 | 1 trap+0xb80 134217728 |@ 2 powerpc_interrupt+0x1a4 268435456 |@@@@@@ 8 536870912 |@@@@ 5 1073741824 |@@@@@@@@ 10 2147483648 |@@@@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 20 2% 88% 0.00 575600 vm object 0xc000000003117de8 nsec ------ Time Distribution ------ count Stack 16384 |@ 1 vm_pager_allocate+0x5c 32768 |@@@ 2 vm_mmap_vnode+0x2c8 65536 |@@@ 2 0xc000000002e32684 131072 |@@@@@@ 4 kern_mmap+0x418 262144 |@@@@@@ 4 sys_mmap+0x3c 524288 |@@@@@@ 4 trap+0xb80 1048576 | 0 2097152 | 0 4194304 |@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 1% 90% 0.00 379882724 vm object (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@ 1 262144 | 0 524288 | 0 1048576 | 0 2097152 | 0 4194304 | 0 8388608 | 0 16777216 | 0 33554432 | 0 67108864 |@@ 1 134217728 | 0 268435456 |@@@@@@@@@ 4 536870912 |@@@@@@ 3 1073741824 |@@@@@@@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 1% 91% 0.00 605747106 vm object vm_imgact_map_page+0x3f0 nsec ------ Time Distribution ------ count Stack 33554432 |@@ 1 0xc000000002c4733c 67108864 |@@ 1 0xc000000002c4684c 134217728 |@@@@@ 2 kern_execve+0x678 268435456 |@@ 1 sys_execve+0x80 536870912 | 0 trap+0xb80 1073741824 |@@@@@@@@@@@@ 5 powerpc_interrupt+0x1a4 2147483648 |@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 1% 92% 0.00 366614766 vm object vm_object_deallocate+0x814 nsec ------ Time Distribution ------ count Stack 8388608 |@@ 1 0xc0000000030ea1fc 16777216 |@@ 1 _vm_map_unlock+0xf8 33554432 |@@ 1 vmspace_exit+0x194 67108864 | 0 exit1+0x954 134217728 |@@ 1 sys_sys_exit+0x28 268435456 |@@ 1 trap+0xb80 536870912 |@@@@@@@@@@ 4 1073741824 |@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 1% 93% 0.00 741622641 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@ 3 vm_map_fixed+0x120 536870912 |@@@ 1 0xc000000002c47a14 1073741824 |@@@@@@@@@@@@ 4 0xc000000002c472c0 2147483648 |@@@@@@ 2 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 1% 94% 0.00 415084 vm object vm_map_insert+0x758 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@ 2 vm_map_fixed+0x120 262144 | 0 vm_mmap_object+0x3ac 524288 |@@@@@@@@@@@@@@@@ 5 0xc000000002e326c4 1048576 |@@@ 1 kern_mmap+0x418 2097152 |@@@ 1 sys_mmap+0x3c trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 1% 95% 0.00 383557477 vm object exec_map_first_page+0x780 nsec ------ Time Distribution ------ count Stack 134217728 |@@@@ 1 kern_execve+0x37c 268435456 |@@@@ 1 sys_execve+0x80 536870912 |@@@@@@@@@@@@ 3 trap+0xb80 1073741824 |@@@@@@@@ 2 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 1% 95% 0.00 653312795 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@ 1 vm_map_lookup+0xb80 2097152 | 0 vm_fault_hold+0xf8 4194304 | 0 vm_fault+0x94 8388608 | 0 trap+0x10ac 16777216 | 0 powerpc_interrupt+0x1a4 33554432 | 0 67108864 | 0 134217728 | 0 268435456 |@@@@ 1 536870912 |@@@@ 1 1073741824 |@@@@@@@@@@@@ 3 2147483648 |@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 1% 96% 0.00 562376285 vm object (usermode) nsec ------ Time Distribution ------ count Stack 33554432 |@@@@ 1 0xc000000002c479cc 67108864 | 0 0xc000000002c472c0 134217728 | 0 0xc000000002c4684c 268435456 |@@@@ 1 kern_execve+0x678 536870912 |@@@@ 1 sys_execve+0x80 1073741824 |@@@@@@@@@@@@@@@@@ 4 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 1% 97% 0.00 826971 vm object (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@ 2 vm_pager_allocate+0x5c 262144 | 0 vm_mmap_vnode+0x2c8 524288 |@@@@@ 1 0xc000000002e32684 1048576 |@@@@@@@@@@ 2 kern_mmap+0x418 2097152 | 0 sys_mmap+0x3c 4194304 |@@@@@ 1 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 1% 97% 0.00 327334912 vm object (usermode) nsec ------ Time Distribution ------ count Stack 33554432 |@@@@@ 1 0xc0000000030ea1fc 67108864 | 0 _vm_map_unlock+0xf8 134217728 | 0 vmspace_exit+0x194 268435456 |@@@@@@@@@@@@@@@ 3 exit1+0x954 536870912 |@@@@@ 1 sys_sys_exit+0x28 1073741824 |@@@@@ 1 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 505105412 vm object (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@ 1 vm_map_lookup+0xb80 1048576 |@@@@@@@ 1 vm_fault_hold+0xf8 2097152 | 0 vm_fault+0x94 4194304 | 0 trap+0x10ac 8388608 | 0 powerpc_interrupt+0x1a4 16777216 | 0 33554432 | 0 67108864 | 0 134217728 | 0 268435456 | 0 536870912 |@@@@@@@ 1 1073741824 | 0 2147483648 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 500638866 vm object (usermode) nsec ------ Time Distribution ------ count Stack 33554432 |@@@@@@@ 1 vm_map_delete+0x1bc 67108864 | 0 vmspace_exit+0x184 134217728 | 0 exit1+0x954 268435456 |@@@@@@@ 1 sys_sys_exit+0x28 536870912 | 0 trap+0xb80 1073741824 |@@@@@@@ 1 powerpc_interrupt+0x1a4 2147483648 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 776167572 vm object vnode_create_vobject+0x18c nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@@ 1 0xc0000000030c5fe0 536870912 |@@@@@@@@@@ 1 VOP_OPEN_APV+0x98 1073741824 | 0 exec_check_permissions+0x164 2147483648 |@@@@@@@@@@ 1 kern_execve+0x348 sys_execve+0x80 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 284937592 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@@@@@@@ 1 kern_execve+0x9d0 536870912 |@@@@@@@@@@@@@@@ 1 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 455081 vm object (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@ 1 vm_map_find+0x4f0 1048576 |@@@@@@@@@@@@@@@ 1 vm_map_find_min+0x98 vm_mmap_object+0x46c 0xc000000002e326c4 kern_mmap+0x418 sys_mmap+0x3c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 864133224 vm object (usermode) nsec ------ Time Distribution ------ count Stack 536870912 |@@@@@@@@@@@@@@@ 1 kern_execve+0x37c 1073741824 | 0 sys_execve+0x80 2147483648 |@@@@@@@@@@@@@@@ 1 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 56430 Per-Filesystem Softdep Lock 0xc000000003087ebc nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 bufdone+0x3b8 65536 | 0 0xc000000002c2bfb0 131072 |@@@@@@@@@@@@@@@ 1 biodone+0x104 g_io_deliver+0x348 g_std_done+0x74 biodone+0x104 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 391883813 vm object (usermode) nsec ------ Time Distribution ------ count Stack 536870912 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_execve+0x370 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 660626 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c4733c 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 409595 vm object (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_map_delete+0x1bc kern_munmap+0xc4 sys_munmap+0x2c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 525605 kernel vm object 0xc0000000030e8b68 nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kmem_free+0x44 uma_large_free+0x30 free+0x90 zfs_kmem_free+0x28 zio_buf_free+0xac abd_free+0x1b4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 70810 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 bufdone+0x3b8 0xc000000002c2bfb0 biodone+0x104 g_io_deliver+0x348 g_std_done+0x74 biodone+0x104 ------------------------------------------------------------------------------- R/W writer blocked by readers: 1205 events in 21.178 seconds (57 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 483 40% 40% 0.00 427427176 vm object vm_fault_hold+0x1b28 nsec ------ Time Distribution ------ count Stack 524288 | 2 vm_fault+0x94 1048576 | 3 trap+0x10ac 2097152 | 0 powerpc_interrupt+0x1a4 4194304 | 4 8388608 | 5 16777216 |@ 20 33554432 |@ 29 67108864 |@ 24 134217728 |@ 25 268435456 |@@@@@@@ 113 536870912 |@@@@@@@ 115 1073741824 |@@@@@@ 106 2147483648 |@ 32 4294967296 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 106 9% 49% 0.00 412817100 vm object 0xc0000000030f0be8 nsec ------ Time Distribution ------ count Stack 131072 | 1 vm_map_delete+0x1bc 262144 | 0 vmspace_exit+0x184 524288 | 1 exit1+0x954 1048576 | 1 sys_sys_exit+0x28 2097152 | 3 trap+0xb80 4194304 | 0 powerpc_interrupt+0x1a4 8388608 |@ 4 16777216 |@ 4 33554432 |@ 5 67108864 | 2 134217728 |@@@ 12 268435456 |@@@@@ 19 536870912 |@@@@@@ 23 1073741824 |@@@@@@ 24 2147483648 |@ 6 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 105 9% 58% 0.00 577501456 vm object vnode_create_vobject+0x18c nsec ------ Time Distribution ------ count Stack 1048576 | 1 0xc0000000030c5fe0 2097152 | 1 VOP_OPEN_APV+0x98 4194304 | 0 exec_check_permissions+0x164 8388608 | 3 kern_execve+0x348 16777216 | 2 sys_execve+0x80 33554432 |@ 4 trap+0xb80 67108864 | 3 134217728 |@ 5 268435456 |@@@@ 16 536870912 |@@@@@@@@ 29 1073741824 |@@@@@@ 21 2147483648 |@@@@@ 19 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 92 8% 65% 0.00 483473964 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 16777216 |@ 4 0xc0000000030ea1fc 33554432 |@@ 7 _vm_map_unlock+0xf8 67108864 |@ 4 vmspace_exit+0x194 134217728 | 3 exit1+0x954 268435456 |@@@@@ 18 sys_sys_exit+0x28 536870912 |@@@@@@@ 22 trap+0xb80 1073741824 |@@@@@@@@ 25 2147483648 |@@ 9 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 88 7% 73% 0.00 458010816 vm object vm_object_deallocate+0x814 nsec ------ Time Distribution ------ count Stack 131072 | 1 0xc0000000030ea1fc 262144 | 1 _vm_map_unlock+0xf8 524288 | 0 vmspace_exit+0x194 1048576 | 0 exit1+0x954 2097152 | 0 sys_sys_exit+0x28 4194304 | 0 trap+0xb80 8388608 | 0 16777216 |@ 5 33554432 |@@ 6 67108864 |@@@ 9 134217728 | 0 268435456 |@@@@@@@ 22 536870912 |@@@@ 13 1073741824 |@@@@@@@ 22 2147483648 |@@ 8 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 57 5% 77% 0.00 370312390 vm object (usermode) nsec ------ Time Distribution ------ count Stack 524288 | 1 1048576 | 0 2097152 | 1 4194304 |@ 3 8388608 |@ 3 16777216 |@@@ 6 33554432 |@@@ 7 67108864 | 0 134217728 |@@ 4 268435456 |@@@@@ 11 536870912 |@@@ 7 1073741824 |@@@ 7 2147483648 |@@@ 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 46 4% 81% 0.00 601912537 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 16777216 |@@ 4 kern_execve+0x9d0 33554432 |@ 2 sys_execve+0x80 67108864 |@ 2 trap+0xb80 134217728 | 0 powerpc_interrupt+0x1a4 268435456 |@@@ 5 536870912 |@@@@@@@ 12 1073741824 |@@@@@@@@@ 14 2147483648 |@@@ 6 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 39 3% 84% 0.00 377991380 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 4194304 | 1 0xc000000002c479cc 8388608 | 0 0xc000000002c472c0 16777216 | 1 0xc000000002c4684c 33554432 |@ 2 kern_execve+0x678 67108864 |@@@ 4 sys_execve+0x80 134217728 |@@@@@ 7 trap+0xb80 268435456 |@@@@ 6 536870912 |@@@@@@ 9 1073741824 |@@@@@ 7 2147483648 | 1 4294967296 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 30 2% 87% 0.00 312192419 vm object vm_map_insert+0x758 nsec ------ Time Distribution ------ count Stack 1048576 |@ 1 vm_map_fixed+0x120 2097152 | 0 0xc000000002c47a14 4194304 | 0 0xc000000002c472c0 8388608 | 0 0xc000000002c4684c 16777216 |@ 1 kern_execve+0x678 33554432 |@@ 2 sys_execve+0x80 67108864 |@@@@ 4 134217728 | 0 268435456 |@@@@@@@@@@ 10 536870912 |@@@@@@ 6 1073741824 |@@@@@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 29 2% 89% 0.00 368684969 vm object vm_object_shadow+0x378 nsec ------ Time Distribution ------ count Stack 524288 |@ 1 vm_map_lookup+0xb80 1048576 | 0 vm_fault_hold+0xf8 2097152 | 0 vm_fault+0x94 4194304 | 0 trap+0x10ac 8388608 |@ 1 powerpc_interrupt+0x1a4 16777216 | 0 33554432 |@@@ 3 67108864 |@ 1 134217728 |@ 1 268435456 |@@@@@@ 6 536870912 |@@@@@@@@@@ 10 1073741824 |@@@@ 4 2147483648 |@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 23 2% 91% 0.00 582980474 vm object vm_imgact_map_page+0x3f0 nsec ------ Time Distribution ------ count Stack 4194304 |@ 1 0xc000000002c4733c 8388608 | 0 0xc000000002c4684c 16777216 | 0 kern_execve+0x678 33554432 | 0 sys_execve+0x80 67108864 | 0 trap+0xb80 134217728 |@ 1 powerpc_interrupt+0x1a4 268435456 |@@@@@@@ 6 536870912 |@@@@@@ 5 1073741824 |@@@@@@ 5 2147483648 |@@@@@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 1% 92% 0.00 465043735 vm object vm_object_shadow+0x390 nsec ------ Time Distribution ------ count Stack 33554432 |@@@@@@ 3 vm_map_lookup+0xb80 67108864 | 0 vm_fault_hold+0xf8 134217728 | 0 vm_fault+0x94 268435456 |@@ 1 trap+0x10ac 536870912 |@@@@@@@@ 4 powerpc_interrupt+0x1a4 1073741824 |@@@@@@@@@@ 5 2147483648 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 1% 93% 0.00 401380304 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@ 1 0xc0000000030ea1fc 33554432 | 0 _vm_map_unlock+0xf8 67108864 |@@@@@ 2 vmspace_exit+0x194 134217728 | 0 exit1+0x954 268435456 |@@@@@@@ 3 sys_sys_exit+0x28 536870912 |@@@@@@@@@@ 4 trap+0xb80 1073741824 |@@ 1 2147483648 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 1% 94% 0.00 643258266 vm object (usermode) nsec ------ Time Distribution ------ count Stack 8388608 |@@ 1 0xc0000000030c5fe0 16777216 | 0 VOP_OPEN_APV+0x98 33554432 | 0 exec_check_permissions+0x164 67108864 | 0 kern_execve+0x348 134217728 |@@ 1 sys_execve+0x80 268435456 |@@@@@ 2 trap+0xb80 536870912 | 0 1073741824 |@@@@@@@@@@@@@@@@ 6 2147483648 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 1% 95% 0.00 238327152 vm object (usermode) nsec ------ Time Distribution ------ count Stack 33554432 |@@@ 1 0xc0000000030ea1fc 67108864 |@@@ 1 _vm_map_unlock+0xf8 134217728 |@@@ 1 vmspace_exit+0x194 268435456 |@@@@@@@@@@@@@@@ 5 exit1+0x954 536870912 |@@@ 1 sys_sys_exit+0x28 1073741824 |@@@ 1 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 1% 96% 0.00 465304 vm object vnode_create_vobject+0x18c nsec ------ Time Distribution ------ count Stack 32768 |@@@ 1 0xc0000000030c5fe0 65536 |@@@ 1 VOP_OPEN_APV+0x98 131072 |@@@@@@ 2 vn_open_vnode+0x200 262144 |@@@@@@@@@@ 3 vn_open_cred+0x420 524288 | 0 vn_open+0x30 1048576 | 0 kern_openat+0x274 2097152 |@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 96% 0.00 879831 vm object (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@ 1 0xc0000000030c5fe0 131072 | 0 VOP_OPEN_APV+0x98 262144 |@@@@@ 1 vn_open_vnode+0x200 524288 |@@@@@@@@@@ 2 vn_open_cred+0x420 1048576 | 0 vn_open+0x30 2097152 |@@@@@ 1 kern_openat+0x274 4194304 |@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 97% 0.00 417911338 vm object exec_map_first_page+0x780 nsec ------ Time Distribution ------ count Stack 33554432 |@@@@@ 1 kern_execve+0x37c 67108864 | 0 sys_execve+0x80 134217728 |@@@@@ 1 trap+0xb80 268435456 | 0 powerpc_interrupt+0x1a4 536870912 |@@@@@@@@@@ 2 1073741824 |@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 97% 0.00 524028748 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@ 1 0xc000000002c479cc 536870912 |@@@@@@@@@@ 2 0xc000000002c472c0 1073741824 |@@@@@@@@@@@@@@@ 3 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 98% 0.00 180070342 vm object (usermode) nsec ------ Time Distribution ------ count Stack 2097152 |@@@@@ 1 vm_map_delete+0x1bc 4194304 | 0 vmspace_exit+0x184 8388608 |@@@@@ 1 exit1+0x954 16777216 | 0 sys_sys_exit+0x28 33554432 |@@@@@ 1 trap+0xb80 67108864 | 0 powerpc_interrupt+0x1a4 134217728 |@@@@@ 1 268435456 |@@@@@ 1 536870912 | 0 1073741824 |@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 409113224 vm object (usermode) nsec ------ Time Distribution ------ count Stack 134217728 |@@@@@@@ 1 kern_execve+0x9d0 268435456 |@@@@@@@ 1 sys_execve+0x80 536870912 |@@@@@@@ 1 trap+0xb80 1073741824 |@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 1746170 vm object (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@ 1 vm_map_delete+0x1bc 524288 |@@@@@@@ 1 kern_munmap+0xc4 1048576 | 0 sys_munmap+0x2c 2097152 | 0 trap+0xb80 4194304 |@@@@@@@@@@@@@@@ 2 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 388540097 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@ 1 0xc000000002c4733c 536870912 |@@@@@@@@@@@@@@@ 2 0xc000000002c4684c 1073741824 |@@@@@@@ 1 kern_execve+0x678 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 185534 vm object 0xc0000000030f0be8 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@ 2 vm_map_delete+0x1bc 131072 | 0 kern_munmap+0xc4 262144 | 0 sys_munmap+0x2c 524288 |@@@@@@@@@@ 1 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 423069279 vm object (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@ 1 vm_map_lookup+0xb80 131072 | 0 vm_fault_hold+0xf8 262144 | 0 vm_fault+0x94 524288 | 0 trap+0x10ac 1048576 | 0 powerpc_interrupt+0x1a4 2097152 | 0 4194304 | 0 8388608 | 0 16777216 | 0 33554432 | 0 67108864 | 0 134217728 | 0 268435456 | 0 536870912 | 0 1073741824 |@@@@@@@@@@@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 388268 vm object 0xc000000003117de8 nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@ 1 vm_pager_allocate+0x5c 524288 |@@@@@@@@@@@@@@@ 1 vm_mmap_vnode+0x2c8 0xc000000002e32684 kern_mmap+0x418 sys_mmap+0x3c trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 635286377 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@@@@@@@ 1 kern_execve+0x370 536870912 | 0 sys_execve+0x80 1073741824 |@@@@@@@@@@@@@@@ 1 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 411139023 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@@@@@@@ 1 vm_map_fixed+0x120 536870912 | 0 0xc000000002c47a14 1073741824 |@@@@@@@@@@@@@@@ 1 0xc000000002c472c0 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 713992 vm object vmspace_fork+0x2d4 nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 fork1+0x4bc sys_fork+0x5c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 217956861 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_map_lookup+0xb80 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 226618632 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_execve+0x37c sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- R/W reader blocked by writer: 323 events in 21.178 seconds (15 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 176 54% 54% 0.00 277735990 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 2097152 | 2 0xc0000000030ebe78 4194304 | 0 vm_map_insert+0x6a8 8388608 | 0 vm_map_fixed+0x120 16777216 |@ 8 0xc000000002c47a14 33554432 |@@@@ 27 0xc000000002c472c0 67108864 |@@ 14 0xc000000002c4684c 134217728 |@ 6 268435456 |@@@@@@@@ 51 536870912 |@@@@@@ 39 1073741824 |@@@@ 26 2147483648 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 107 33% 88% 0.00 262280246 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 2097152 | 1 vm_map_entry_set_vnode_text+0x4c 4194304 | 0 0xc0000000030ea1e4 8388608 | 0 _vm_map_unlock+0xf8 16777216 |@ 4 vmspace_exit+0x194 33554432 |@ 7 exit1+0x954 67108864 |@@ 10 sys_sys_exit+0x28 134217728 |@ 4 268435456 |@@@@@@@@@@@@@ 47 536870912 |@@@@@@ 23 1073741824 |@@ 8 2147483648 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 26 8% 96% 0.00 151298331 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@ 1 vm_map_entry_set_vnode_text+0x4c 33554432 |@@@@@@@@ 7 0xc0000000030ea1e4 67108864 |@@@@@ 5 _vm_map_unlock+0xf8 134217728 | 0 vmspace_exit+0x194 268435456 |@@@@@@@@@ 8 exit1+0x954 536870912 |@@@@ 4 sys_sys_exit+0x28 1073741824 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 4% 100% 0.00 239465392 vm object (usermode) nsec ------ Time Distribution ------ count Stack 67108864 |@@@@@@@@@@ 5 0xc0000000030ebe78 134217728 |@@ 1 vm_map_insert+0x6a8 268435456 |@@@@@@@@ 4 vm_map_fixed+0x120 536870912 |@@ 1 0xc000000002c47a14 1073741824 |@@@@@@ 3 0xc000000002c472c0 0xc000000002c4684c ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 48 events in 21.178 seconds (2 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 30 62% 62% 0.00 256483726 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 1048576 |@ 1 0xc0000000030ebe78 2097152 | 0 vm_map_insert+0x6a8 4194304 | 0 vm_map_fixed+0x120 8388608 | 0 0xc000000002c47a14 16777216 |@@@@ 4 0xc000000002c472c0 33554432 |@@@@@ 5 0xc000000002c4684c 67108864 |@ 1 134217728 | 0 268435456 |@@@@ 4 536870912 |@@@@@@@@@@@ 11 1073741824 |@@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 31% 94% 0.00 334614065 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 33554432 |@@@@ 2 vm_map_entry_set_vnode_text+0x4c 67108864 |@@ 1 0xc0000000030ea1e4 134217728 |@@ 1 _vm_map_unlock+0xf8 268435456 |@@@@@@@@@@ 5 vmspace_exit+0x194 536870912 |@@@@@@ 3 exit1+0x954 1073741824 |@@@@ 2 sys_sys_exit+0x28 2147483648 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 4% 98% 0.00 15289999 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 vm_map_entry_set_vnode_text+0x4c 0xc0000000030ea1e4 _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 2% 100% 0.00 145272595 vm object (usermode) nsec ------ Time Distribution ------ count Stack 268435456 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030ebe78 vm_map_insert+0x6a8 vm_map_fixed+0x120 0xc000000002c47a14 0xc000000002c472c0 0xc000000002c4684c ------------------------------------------------------------------------------- R/W writer spin on writer: 1833 events in 21.178 seconds (87 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 336 18% 18% 0.00 12721036 vm object vm_fault_hold+0x1b28 nsec ------ Time Distribution ------ count Stack 256 | 1 vm_fault+0x94 512 | 2 trap+0x10ac 1024 | 1 powerpc_interrupt+0x1a4 2048 | 8 4096 | 4 8192 |@ 13 16384 |@ 22 32768 |@ 20 65536 |@@ 25 131072 | 8 262144 | 9 524288 | 11 1048576 |@ 18 2097152 | 10 4194304 |@ 19 8388608 |@@ 31 16777216 |@@@ 38 33554432 |@@@@ 55 67108864 |@@ 33 134217728 | 8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 205 11% 30% 0.00 15620804 vm object vm_map_insert+0x758 nsec ------ Time Distribution ------ count Stack 4096 | 1 vm_map_fixed+0x120 8192 | 0 0xc000000002c47a14 16384 | 2 0xc000000002c472c0 32768 | 3 0xc000000002c4684c 65536 | 6 kern_execve+0x678 131072 |@ 8 sys_execve+0x80 262144 |@ 8 524288 |@ 10 1048576 |@ 10 2097152 |@ 13 4194304 |@@ 20 8388608 |@@@@@ 35 16777216 |@@@ 26 33554432 |@@@ 22 67108864 |@@@@@ 36 134217728 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 154 8% 38% 0.00 12491937 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 4096 | 3 0xc0000000030ea1fc 8192 | 2 _vm_map_unlock+0xf8 16384 | 1 vmspace_exit+0x194 32768 |@ 6 exit1+0x954 65536 | 1 sys_sys_exit+0x28 131072 | 1 trap+0xb80 262144 |@ 7 524288 |@@ 11 1048576 |@ 8 2097152 |@@@ 17 4194304 |@@@@ 21 8388608 |@@@ 19 16777216 |@@@@ 21 33554432 |@@ 13 67108864 |@@@ 19 134217728 | 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 106 6% 44% 0.00 19611868 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 1024 | 1 0xc000000002c479cc 2048 | 0 0xc000000002c472c0 4096 | 0 0xc000000002c4684c 8192 | 2 kern_execve+0x678 16384 | 1 sys_execve+0x80 32768 | 1 trap+0xb80 65536 | 3 131072 |@ 4 262144 |@ 4 524288 |@ 6 1048576 | 3 2097152 |@ 6 4194304 |@@ 9 8388608 |@@@ 12 16777216 |@@ 9 33554432 |@@@@@ 20 67108864 |@@@@ 17 134217728 |@@ 8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 99 5% 49% 0.00 16132171 vm object 0xc0000000030f0be8 nsec ------ Time Distribution ------ count Stack 4096 | 2 vm_map_delete+0x1bc 8192 | 0 vmspace_exit+0x184 16384 | 3 exit1+0x954 32768 | 1 sys_sys_exit+0x28 65536 |@ 6 trap+0xb80 131072 | 1 powerpc_interrupt+0x1a4 262144 |@ 4 524288 | 3 1048576 |@ 6 2097152 |@@ 7 4194304 |@@ 9 8388608 |@@@ 10 16777216 |@@@ 13 33554432 |@@@@@ 18 67108864 |@@@ 12 134217728 |@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 81 4% 54% 0.00 15737865 vm object vm_object_shadow+0x390 nsec ------ Time Distribution ------ count Stack 16384 | 2 vm_map_lookup+0xb80 32768 | 0 vm_fault_hold+0xf8 65536 | 2 vm_fault+0x94 131072 | 2 trap+0x10ac 262144 |@@ 8 powerpc_interrupt+0x1a4 524288 |@ 5 1048576 |@ 3 2097152 |@ 5 4194304 |@@@ 10 8388608 |@@@ 10 16777216 |@@ 7 33554432 |@@@ 10 67108864 |@@@@@ 16 134217728 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 79 4% 58% 0.00 13936217 vm object vm_object_shadow+0x378 nsec ------ Time Distribution ------ count Stack 512 | 1 vm_map_lookup+0xb80 1024 | 0 vm_fault_hold+0xf8 2048 | 0 vm_fault+0x94 4096 | 2 trap+0x10ac 8192 | 1 powerpc_interrupt+0x1a4 16384 |@ 3 32768 | 0 65536 | 2 131072 |@@ 6 262144 |@@@ 9 524288 |@ 3 1048576 | 1 2097152 | 2 4194304 | 1 8388608 |@@@@ 12 16777216 |@@@@ 11 33554432 |@@@@ 13 67108864 |@@@@ 12 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 66 4% 61% 0.00 111933 vm object 0xc000000003117de8 nsec ------ Time Distribution ------ count Stack 16384 | 1 vm_pager_allocate+0x5c 32768 |@@@ 7 vm_mmap_vnode+0x2c8 65536 |@@@ 8 0xc000000002e32684 131072 |@@@@@@@@@@@@ 28 kern_mmap+0x418 262144 |@@@@@@@@ 19 sys_mmap+0x3c 524288 |@ 3 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 62 3% 65% 0.00 11225869 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 1024 | 2 kern_execve+0x9d0 2048 | 1 sys_execve+0x80 4096 | 0 trap+0xb80 8192 | 2 powerpc_interrupt+0x1a4 16384 | 1 32768 |@ 3 65536 | 2 131072 | 1 262144 |@@ 5 524288 | 0 1048576 | 1 2097152 | 2 4194304 |@@@ 8 8388608 |@@@@ 9 16777216 |@@@@@ 12 33554432 |@@ 5 67108864 |@@@ 8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 48 3% 67% 0.00 64545 Per-Filesystem Softdep Lock 0xc000000002ce4a60 nsec ------ Time Distribution ------ count Stack 8192 | 1 _sleep+0x290 16384 |@@ 4 0xc000000003089d44 32768 |@ 3 fork_exit+0xdc 65536 |@@@@@@@@ 13 0xc000000003185c5c 131072 |@@@@@@@@@@@@@@@@ 26 262144 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 45 2% 70% 0.00 31372 vm object vm_fault_hold+0x2700 nsec ------ Time Distribution ------ count Stack 2048 |@@ 4 vm_fault+0x94 4096 |@ 2 trap+0x10ac 8192 |@@@ 5 powerpc_interrupt+0x1a4 16384 |@@@@@@@@@ 14 32768 |@@@@@@ 10 65536 |@@@@ 6 131072 | 1 262144 |@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 45 2% 72% 0.00 1600215 kernel vm object 0xc0000000030e8b68 nsec ------ Time Distribution ------ count Stack 32768 | 1 kmem_free+0x44 65536 | 0 uma_large_free+0x30 131072 | 1 free+0x90 262144 |@@@ 5 zfs_kmem_free+0x28 524288 |@@@@@@ 9 zio_buf_free+0xac 1048576 |@@@@@@@@@ 14 abd_free+0x1b4 2097152 |@@@@@@ 9 4194304 |@@ 4 8388608 | 1 16777216 | 0 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 42 2% 75% 0.00 19924185 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 131072 |@ 2 kern_execve+0x370 262144 | 0 sys_execve+0x80 524288 | 1 trap+0xb80 1048576 |@@ 3 powerpc_interrupt+0x1a4 2097152 |@@ 4 4194304 |@ 2 8388608 |@@@ 5 16777216 |@@@@@ 7 33554432 |@@@@@@ 9 67108864 |@@@@@ 7 134217728 |@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 34 2% 76% 0.00 2224344 kernel vm object kmem_back_domain+0x340 nsec ------ Time Distribution ------ count Stack 8192 | 1 kmem_malloc_domainset+0xd0 16384 | 0 uma_large_malloc_domain+0x148 32768 |@ 2 uma_large_malloc+0x28 65536 |@ 2 malloc+0x1a0 131072 |@ 2 zfs_kmem_alloc+0x2c 262144 |@ 2 0xc0000000026bb318 524288 |@@ 3 1048576 |@@@@@ 6 2097152 |@@@@@ 6 4194304 |@@@@@ 6 8388608 | 1 16777216 |@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 26 1% 78% 0.00 9090246 vm object (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@ 1 4096 | 0 8192 | 0 16384 |@@@ 3 32768 |@@@@ 4 65536 | 0 131072 |@ 1 262144 |@ 1 524288 | 0 1048576 |@@@@ 4 2097152 | 0 4194304 |@ 1 8388608 |@@@@ 4 16777216 |@ 1 33554432 |@@@@ 4 67108864 |@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 22 1% 79% 0.00 92278 vm object vm_map_insert+0x758 nsec ------ Time Distribution ------ count Stack 256 |@ 1 vm_map_fixed+0x120 512 | 0 vm_mmap_object+0x3ac 1024 | 0 0xc000000002e326c4 2048 | 0 kern_mmap+0x418 4096 | 0 sys_mmap+0x3c 8192 | 0 trap+0xb80 16384 |@ 1 32768 |@@@@@@@@ 6 65536 |@@@@@@ 5 131072 | 0 262144 |@@@@@@@@@@ 8 524288 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 20 1% 80% 0.00 4049976 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@ 1 vm_map_lookup+0xb80 2048 |@@@ 2 vm_fault_hold+0xf8 4096 | 0 vm_fault+0x94 8192 |@@@ 2 trap+0x10ac 16384 |@ 1 powerpc_interrupt+0x1a4 32768 | 0 65536 |@ 1 131072 |@@@@@@@ 5 262144 |@@@ 2 524288 |@@@@ 3 1048576 | 0 2097152 |@ 1 4194304 | 0 8388608 | 0 16777216 | 0 33554432 |@ 1 67108864 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19 1% 81% 0.00 79020 Per-Filesystem Softdep Lock softdep_freefile+0xbb4 nsec ------ Time Distribution ------ count Stack 16384 |@ 1 ffs_vfree+0x44 32768 |@@@@@@@@@@@@@@ 9 ufs_inactive+0x598 65536 |@@@@@@@@@ 6 VOP_INACTIVE_APV+0x98 131072 |@ 1 vinactive+0x1ac 262144 | 0 0xc000000002e1aa5c 524288 |@@@ 2 0xc000000002e338d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 18 1% 82% 0.00 137064 vm object (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@ 1 vm_pager_allocate+0x5c 131072 |@@@@@@@@@@@@@@@@@@ 11 vm_mmap_vnode+0x2c8 262144 |@@@@@@ 4 0xc000000002e32684 524288 |@@@ 2 kern_mmap+0x418 sys_mmap+0x3c trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 17 1% 83% 0.00 2294055 kernel vm object (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@ 1 kmem_free+0x44 524288 |@@@ 2 uma_large_free+0x30 1048576 |@@@@@@@ 4 free+0x90 2097152 |@@@@@@@@ 5 zfs_kmem_free+0x28 4194304 |@ 1 zio_buf_free+0xac 8388608 |@@@@@ 3 abd_free+0x1b4 16777216 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 1% 84% 0.00 22441224 vm object vm_imgact_map_page+0x3f0 nsec ------ Time Distribution ------ count Stack 8192 |@@ 1 0xc000000002c4733c 16384 | 0 0xc000000002c4684c 32768 | 0 kern_execve+0x678 65536 |@@ 1 sys_execve+0x80 131072 | 0 trap+0xb80 262144 | 0 powerpc_interrupt+0x1a4 524288 | 0 1048576 | 0 2097152 |@@ 1 4194304 | 0 8388608 |@@@@ 2 16777216 |@@@@@@@@ 4 33554432 |@@@@@@ 3 67108864 |@@ 1 134217728 |@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 1% 85% 0.00 25649 vm object (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@ 1 4096 |@@ 1 8192 | 0 16384 |@@@@@@@@@@@@ 6 32768 |@@@@ 2 65536 |@@@@@@@@ 4 131072 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 1% 86% 0.00 14098225 vm object vm_object_deallocate+0x814 nsec ------ Time Distribution ------ count Stack 16384 |@@ 1 0xc0000000030ea1fc 32768 |@@ 1 _vm_map_unlock+0xf8 65536 | 0 vmspace_exit+0x194 131072 |@@ 1 exit1+0x954 262144 | 0 sys_sys_exit+0x28 524288 | 0 trap+0xb80 1048576 | 0 2097152 | 0 4194304 |@@ 1 8388608 |@@@@ 2 16777216 |@@@@@@ 3 33554432 |@@@@@@ 3 67108864 |@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 1% 86% 0.00 95146 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 nsec ------ Time Distribution ------ count Stack 4096 |@@ 1 ffs_update+0x230 8192 | 0 ufs_inactive+0x3c0 16384 | 0 VOP_INACTIVE_APV+0x98 32768 | 0 vinactive+0x1ac 65536 |@@@@@@@@@@@@@@@ 7 0xc000000002e1aa5c 131072 |@@@@@@@@@@ 5 0xc000000002e338d0 262144 | 0 524288 | 0 1048576 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 1% 87% 0.00 20689557 vm object (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@ 1 vm_map_fixed+0x120 65536 | 0 0xc000000002c47a14 131072 | 0 0xc000000002c472c0 262144 | 0 0xc000000002c4684c 524288 | 0 kern_execve+0x678 1048576 |@@ 1 sys_execve+0x80 2097152 |@@ 1 4194304 | 0 8388608 |@@@@ 2 16777216 |@@@@ 2 33554432 |@@@@ 2 67108864 |@@@@@@@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 1% 88% 0.00 4334969 kernel vm object (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@ 3 kmem_malloc_domainset+0xd0 2097152 |@@@@@@ 3 uma_large_malloc_domain+0x148 4194304 |@@@@@@@@@@@ 5 uma_large_malloc+0x28 8388608 | 0 malloc+0x1a0 16777216 |@@ 1 zfs_kmem_alloc+0x2c 33554432 |@@ 1 0xc0000000026bb318 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 1% 88% 0.00 1632570 kernel vm object (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@ 2 kmem_malloc_domainset+0xd0 524288 |@@@@ 2 uma_large_malloc_domain+0x148 1048576 |@@@@@@@@@ 4 uma_large_malloc+0x28 2097152 |@@@@ 2 malloc+0x1a0 4194304 |@@ 1 zfs_kmem_alloc+0x2c 8388608 |@@@@ 2 zio_buf_alloc+0xc4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 1% 89% 0.00 17989116 vm object (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@ 1 vm_map_lookup+0xb80 16384 | 0 vm_fault_hold+0xf8 32768 |@@ 1 vm_fault+0x94 65536 | 0 trap+0x10ac 131072 | 0 powerpc_interrupt+0x1a4 262144 |@@@@@@@@ 3 524288 | 0 1048576 | 0 2097152 | 0 4194304 | 0 8388608 |@@ 1 16777216 |@@ 1 33554432 |@@@@@@@@ 3 67108864 | 0 134217728 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 1% 90% 0.00 5683497 kernel vm object kmem_back_domain+0x340 nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@ 3 kmem_malloc_domainset+0xd0 1048576 |@@@@@ 2 uma_large_malloc_domain+0x148 2097152 |@@ 1 uma_large_malloc+0x28 4194304 |@@ 1 malloc+0x1a0 8388608 |@@ 1 zfs_kmem_alloc+0x2c 16777216 |@@ 1 zio_buf_alloc+0xc4 33554432 |@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 1% 90% 0.00 18105211 vm object (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@ 1 0xc000000002c479cc 16384 | 0 0xc000000002c472c0 32768 | 0 0xc000000002c4684c 65536 | 0 kern_execve+0x678 131072 | 0 sys_execve+0x80 262144 |@@@ 1 trap+0xb80 524288 | 0 1048576 |@@@ 1 2097152 | 0 4194304 |@@@@@@ 2 8388608 | 0 16777216 | 0 33554432 |@@@@@@@@@ 3 67108864 |@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 1% 91% 0.00 42650 Per-Filesystem Softdep Lock softdep_setup_inofree+0x24c nsec ------ Time Distribution ------ count Stack 4096 |@@@ 1 ffs_freefile+0x2e4 8192 | 0 0xc0000000030972b8 16384 | 0 softdep_freefile+0x6f0 32768 |@@@ 1 ffs_vfree+0x44 65536 |@@@@@@@@@@@@@@@@@@@@@ 7 ufs_inactive+0x598 131072 |@@@ 1 VOP_INACTIVE_APV+0x98 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 91% 0.00 14008202 vm object exec_map_first_page+0x780 nsec ------ Time Distribution ------ count Stack 65536 |@@@ 1 kern_execve+0x37c 131072 | 0 sys_execve+0x80 262144 |@@@ 1 trap+0xb80 524288 | 0 powerpc_interrupt+0x1a4 1048576 | 0 2097152 | 0 4194304 | 0 8388608 |@@@ 1 16777216 |@@@@@@@@@@@@@ 4 33554432 |@@@ 1 67108864 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 92% 0.00 36376 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@ 3 0xc0000000030750b4 4096 | 0 0xc0000000030a5abc 8192 |@@@ 1 0xc000000003094220 16384 | 0 0xc00000000309f878 32768 |@@@ 1 0xc000000003085264 65536 |@@@@@@ 2 0xc000000003089c88 131072 |@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 92% 0.00 35813 Per-Filesystem Softdep Lock 0xc000000003087ebc nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@ 2 bufdone+0x3b8 2048 | 0 0xc000000002c2bfb0 4096 | 0 biodone+0x104 8192 |@@@ 1 g_io_deliver+0x348 16384 |@@@ 1 g_std_done+0x74 32768 |@@@ 1 biodone+0x104 65536 |@@@@@@@@@@ 3 131072 | 0 262144 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 93% 0.00 42169 Per-Filesystem Softdep Lock 0xc0000000030a1778 nsec ------ Time Distribution ------ count Stack 1024 |@@@ 1 0xc000000003085228 2048 |@@@ 1 0xc000000003089c88 4096 |@@@ 1 fork_exit+0xdc 8192 |@@@ 1 0xc000000003185c5c 16384 |@@@ 1 32768 | 0 65536 |@@@ 1 131072 |@@@ 1 262144 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 93% 0.00 10339840 vm object (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@ 1 0xc0000000030ea1fc 524288 |@@@@ 1 _vm_map_unlock+0xf8 1048576 | 0 vmspace_exit+0x194 2097152 |@@@@ 1 exit1+0x954 4194304 |@@@@ 1 sys_sys_exit+0x28 8388608 | 0 trap+0xb80 16777216 |@@@@ 1 33554432 |@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 93% 0.00 38659 vm object (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@ 1 vm_map_delete+0x1bc 16384 |@@@@@@@@@@ 2 kern_munmap+0xc4 32768 |@@@@@ 1 sys_munmap+0x2c 65536 |@@@@@ 1 trap+0xb80 131072 | 0 powerpc_interrupt+0x1a4 262144 |@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 94% 0.00 51993 Per-Filesystem Softdep Lock 0xc000000003097404 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@ 1 softdep_freefile+0x6f0 16384 | 0 ffs_vfree+0x44 32768 |@@@@@ 1 ufs_inactive+0x598 65536 |@@@@@@@@@@@@@@@ 3 VOP_INACTIVE_APV+0x98 131072 | 0 vinactive+0x1ac 262144 |@@@@@ 1 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 94% 0.00 41843 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@ 1 ffs_freefile+0x2e4 32768 |@@@@@ 1 0xc0000000030972b8 65536 |@@@@@@@@@@@@@@@ 3 softdep_freefile+0x6f0 131072 |@@@@@ 1 ffs_vfree+0x44 ufs_inactive+0x598 VOP_INACTIVE_APV+0x98 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 94% 0.00 87570 vm object (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@ 3 vm_map_find+0x4f0 131072 | 0 vm_map_find_min+0x98 262144 |@@@@@@@@@@@@ 2 vm_mmap_object+0x46c 0xc000000002e326c4 kern_mmap+0x418 sys_mmap+0x3c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 95% 0.00 25288 Per-Filesystem Softdep Lock 0xc0000000030a5ca8 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@ 1 0xc000000003094220 512 | 0 0xc00000000309f878 1024 | 0 0xc000000003085264 2048 |@@@@@@ 1 0xc000000003089c88 4096 | 0 fork_exit+0xdc 8192 | 0 0xc000000003185c5c 16384 |@@@@@@ 1 32768 | 0 65536 |@@@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 95% 0.00 78168 Per-Filesystem Softdep Lock 0xc000000003094844 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@ 1 0xc00000000309f878 65536 |@@@@@@@@@@@@ 2 0xc000000003085264 131072 |@@@@@@ 1 0xc000000003089c88 262144 |@@@@@@ 1 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 95% 0.00 57463 Per-Filesystem Softdep Lock 0xc00000000308da54 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@ 1 softdep_setup_freeblocks+0x180 1024 | 0 ffs_truncate+0x744 2048 | 0 ufs_inactive+0x29c 4096 | 0 VOP_INACTIVE_APV+0x98 8192 | 0 vinactive+0x1ac 16384 | 0 0xc000000002e1aa5c 32768 |@@@@@@ 1 65536 |@@@@@@ 1 131072 |@@@@@@ 1 262144 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 95% 0.00 71574 Per-Filesystem Softdep Lock 0xc00000000308705c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@ 1 0xc0000000030b1d58 65536 |@@@@@@ 1 bufwrite+0x2d0 131072 |@@@@@@@@@@@@@@@@@@ 3 0xc0000000030b1898 bawrite+0x40 0xc0000000030a1710 0xc000000003085228 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 96% 0.00 6922921 vm object (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@ 1 kern_execve+0x370 8192 |@@@@@@@ 1 sys_execve+0x80 16384 | 0 trap+0xb80 32768 | 0 powerpc_interrupt+0x1a4 65536 | 0 131072 | 0 262144 | 0 524288 | 0 1048576 | 0 2097152 |@@@@@@@ 1 4194304 | 0 8388608 | 0 16777216 | 0 33554432 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 96% 0.00 14802081 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@ 1 vm_map_delete+0x1bc 2097152 | 0 vmspace_exit+0x184 4194304 |@@@@@@@ 1 exit1+0x954 8388608 | 0 sys_sys_exit+0x28 16777216 |@@@@@@@ 1 trap+0xb80 33554432 | 0 powerpc_interrupt+0x1a4 67108864 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 96% 0.00 21677 Per-Filesystem Softdep Lock 0xc0000000030a17d0 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@ 1 0xc000000003085228 4096 | 0 0xc000000003089c88 8192 | 0 fork_exit+0xdc 16384 |@@@@@@@ 1 0xc000000003185c5c 32768 |@@@@@@@ 1 65536 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 96% 0.00 218899 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@ 1 softdep_freefile+0x6f0 131072 |@@@@@@@ 1 ffs_vfree+0x44 262144 | 0 ufs_inactive+0x598 524288 |@@@@@@@@@@@@@@@ 2 VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 96% 0.00 35375100 vm object vnode_create_vobject+0x18c nsec ------ Time Distribution ------ count Stack 8388608 |@@@@@@@@@@ 1 0xc0000000030c5fe0 16777216 |@@@@@@@@@@ 1 VOP_OPEN_APV+0x98 33554432 | 0 exec_check_permissions+0x164 67108864 | 0 kern_execve+0x348 134217728 |@@@@@@@@@@ 1 sys_execve+0x80 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 97% 0.00 56219 vm object 0xc0000000030f0be8 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@ 1 vm_map_delete+0x1bc 16384 | 0 kern_munmap+0xc4 32768 | 0 sys_munmap+0x2c 65536 |@@@@@@@@@@ 1 trap+0xb80 131072 |@@@@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 97% 0.00 11001791 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@ 1 kern_execve+0x9d0 2097152 | 0 sys_execve+0x80 4194304 | 0 trap+0xb80 8388608 | 0 powerpc_interrupt+0x1a4 16777216 |@@@@@@@@@@ 1 33554432 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 97% 0.00 15823351 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@ 1 kern_execve+0x37c 2097152 | 0 sys_execve+0x80 4194304 | 0 trap+0xb80 8388608 |@@@@@@@@@@ 1 powerpc_interrupt+0x1a4 16777216 | 0 33554432 | 0 67108864 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 97% 0.00 202843 Per-Filesystem Softdep Lock 0xc00000000309a1ac nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1 0xc00000000309f868 65536 | 0 0xc000000003085264 131072 |@@@@@@@@@@ 1 0xc000000003089c88 262144 | 0 fork_exit+0xdc 524288 |@@@@@@@@@@ 1 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 97% 0.00 45411 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@ 1 ffs_update+0x230 32768 | 0 ffs_truncate+0x760 65536 |@@@@@@@@@@ 1 ufs_inactive+0x29c 131072 |@@@@@@@@@@ 1 VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 97% 0.00 206648 Per-Filesystem Softdep Lock 0xc00000000308da54 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@ 1 0xc0000000030a10b8 65536 | 0 0xc000000003085228 131072 |@@@@@@@@@@ 1 0xc000000003089c88 262144 | 0 fork_exit+0xdc 524288 |@@@@@@@@@@ 1 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 98% 0.00 13017 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@ 1 0xc000000003085228 4096 | 0 0xc000000003089c88 8192 | 0 fork_exit+0xdc 16384 |@@@@@@@@@@ 1 0xc000000003185c5c 32768 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 98% 0.00 109948 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@ 2 0xc0000000030750b4 262144 |@@@@@@@@@@ 1 0xc0000000030a5abc 0xc000000003094220 0xc00000000309f878 0xc000000003085264 0xc000000003089c88 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 98% 0.00 86054 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@ 2 ffs_vfree+0x44 131072 | 0 ufs_inactive+0x598 262144 |@@@@@@@@@@ 1 VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c 0xc000000002e338d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 98% 0.00 40417 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 3 0xc0000000030a10b8 0xc000000003085228 0xc000000003089c88 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 98% 0.00 67685 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 0xc000000002c479cc 131072 |@@@@@@@@@@@@@@@ 1 0xc000000002c472c0 0xc000000002c47ed8 0xc000000002c46da0 kern_execve+0x678 sys_execve+0x80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 98% 0.00 41377 vm object vm_map_insert+0x758 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 vm_map_find+0x4f0 vm_map_find_min+0x98 vm_mmap_object+0x46c 0xc000000002e326c4 kern_mmap+0x418 sys_mmap+0x3c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 98% 0.00 14313 Per-Filesystem Softdep Lock softdep_slowdown+0x374 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@ 1 ffs_truncate+0x1f0 4096 | 0 ufs_inactive+0x29c 8192 | 0 VOP_INACTIVE_APV+0x98 16384 | 0 vinactive+0x1ac 32768 |@@@@@@@@@@@@@@@ 1 0xc000000002e1aa5c 0xc000000002e338d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 98% 0.00 284828 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc7c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 ffs_truncate+0x744 131072 | 0 ufs_inactive+0x29c 262144 | 0 VOP_INACTIVE_APV+0x98 524288 | 0 vinactive+0x1ac 1048576 |@@@@@@@@@@@@@@@ 1 0xc000000002e1aa5c 0xc000000002e338d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 93892 Per-Filesystem Softdep Lock softdep_setup_freeblocks+0xc4c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 ffs_truncate+0x744 131072 | 0 ufs_inactive+0x29c 262144 |@@@@@@@@@@@@@@@ 1 VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c 0xc000000002e338d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 44894 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 1 0xc00000000309220c 8192 | 0 softdep_setup_freeblocks+0x328 16384 | 0 ffs_truncate+0x744 32768 | 0 ufs_inactive+0x29c 65536 | 0 VOP_INACTIVE_APV+0x98 131072 |@@@@@@@@@@@@@@@ 1 vinactive+0x1ac ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 50969 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 _sleep+0x290 65536 | 0 0xc000000003089d44 131072 |@@@@@@@@@@@@@@@ 1 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 1849 vm object vm_object_collapse+0x710 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vmspace_fork+0xa50 fork1+0x4bc sys_fork+0x5c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 42312 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030ea1fc _vm_map_unlock+0xf8 kern_munmap+0xd8 sys_munmap+0x2c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 45309 vm object (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030ea1fc _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 sys_sys_exit+0x28 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 39894 tcpinp 0xc000000002f87f90 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sosend_generic+0x5cc sosend+0x68 0xc000000002d940c4 0xc000000002d87de0 sys_write+0xf4 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 23574506 vm object (usermode) nsec ------ Time Distribution ------ count Stack 33554432 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002c4733c 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 44540 Per-Filesystem Softdep Lock softdep_slowdown+0x374 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ffs_truncate+0x1f0 0xc0000000030c6e84 VOP_SETATTR_APV+0x98 vop_stdallocate+0x230 VOP_ALLOCATE_APV+0x98 kern_posix_fallocate+0x2c4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 44423 Per-Filesystem Softdep Lock softdep_setup_blkfree+0x248 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030750b4 0xc00000000309041c 0xc00000000309f858 0xc000000003085264 0xc000000003089c88 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 41255 bufobj interlock bqrelse+0x2a0 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 bdwrite+0x2a8 ffs_update+0x3c0 ufs_inactive+0x3c0 VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 346 Per-Filesystem Softdep Lock 0xc0000000030a17d0 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 softdep_update_inodeblock+0x1e4 ffs_update+0x230 ffs_truncate+0x760 0xc0000000030c6e84 VOP_SETATTR_APV+0x98 vop_stdallocate+0x1e8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 25567 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ffs_truncate+0x744 ufs_inactive+0x29c VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c 0xc000000002e338d0 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 62635 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 softdep_freefile+0xa4 ffs_vfree+0x44 ufs_inactive+0x598 VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 476 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ffs_balloc_ufs2+0xfe0 ffs_truncate+0x390 0xc0000000030c6e84 VOP_SETATTR_APV+0x98 vop_stdallocate+0x1e8 VOP_ALLOCATE_APV+0x98 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 41987 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ffs_update+0x230 ffs_truncate+0x760 ufs_inactive+0x29c VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3695 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 bufdone+0x3b8 0xc000000002c2bfb0 biodone+0x104 g_io_deliver+0x348 g_std_done+0x74 biodone+0x104 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 51238 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030b1d58 bufwrite+0x2d0 0xc0000000030b1898 softdep_freefile+0xa00 ffs_vfree+0x44 ufs_inactive+0x598 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 39730 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030b1d58 bufwrite+0x2d0 0xc0000000030b1898 bawrite+0x40 0xc0000000030a1710 0xc000000003085228 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 45138 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc00000000308df00 softdep_setup_blkfree+0xd0 0xc0000000030750b4 0xc0000000030a5abc 0xc000000003094220 0xc00000000309f878 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1718 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ffs_balloc_ufs2+0xe0 0xc0000000030b2be0 VOP_WRITE_APV+0x180 vop_stdallocate+0x370 VOP_ALLOCATE_APV+0x98 kern_posix_fallocate+0x2c4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 1744 Per-Filesystem Softdep Lock (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 softdep_setup_freeblocks+0x180 ffs_truncate+0x744 ufs_inactive+0x29c VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 3061 Per-Filesystem Softdep Lock 0xc00000000308705c nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030b1d58 bufwrite+0x2d0 0xc0000000030b1898 softdep_freefile+0xa00 ffs_vfree+0x44 ufs_inactive+0x598 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 43370 Per-Filesystem Softdep Lock 0xc000000003085470 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000003089c88 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 789 Per-Filesystem Softdep Lock softdep_update_inodeblock+0x798 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ffs_update+0x230 0xc0000000030ca178 ufs_vinit+0xe0 VOP_CREATE_APV+0x98 vn_open_cred+0x398 vn_open+0x30 ------------------------------------------------------------------------------- R/W writer spin on readers: 1277 events in 21.178 seconds (60 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 495 39% 39% 0.00 16762826 vm object vm_fault_hold+0x1b28 nsec ------ Time Distribution ------ count Stack 32768 | 2 vm_fault+0x94 65536 | 0 trap+0x10ac 131072 | 9 powerpc_interrupt+0x1a4 262144 |@ 20 524288 | 7 1048576 |@ 19 2097152 |@@ 40 4194304 |@@@@ 73 8388608 |@@@ 64 16777216 |@@@@ 75 33554432 |@@@@@@ 108 67108864 |@@@ 64 134217728 | 14 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 107 8% 47% 0.00 18482548 vm object 0xc0000000030f0be8 nsec ------ Time Distribution ------ count Stack 65536 | 1 vm_map_delete+0x1bc 131072 |@ 4 vmspace_exit+0x184 262144 |@@@ 12 exit1+0x954 524288 | 2 sys_sys_exit+0x28 1048576 | 3 trap+0xb80 2097152 |@ 5 powerpc_interrupt+0x1a4 4194304 |@@ 10 8388608 |@@@ 13 16777216 |@@@ 11 33554432 |@@@@@@@ 26 67108864 |@@@@ 17 134217728 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 105 8% 55% 0.00 22043211 vm object vnode_create_vobject+0x18c nsec ------ Time Distribution ------ count Stack 131072 | 1 0xc0000000030c5fe0 262144 | 3 VOP_OPEN_APV+0x98 524288 | 3 exec_check_permissions+0x164 1048576 | 2 kern_execve+0x348 2097152 |@ 5 sys_execve+0x80 4194304 |@@ 10 trap+0xb80 8388608 |@@@@ 16 16777216 |@@@ 13 33554432 |@@@@@ 20 67108864 |@@@@@@@@ 28 134217728 |@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 94 7% 63% 0.00 20166551 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 262144 | 2 0xc0000000030ea1fc 524288 | 0 _vm_map_unlock+0xf8 1048576 |@ 6 vmspace_exit+0x194 2097152 |@@ 8 exit1+0x954 4194304 |@@ 9 sys_sys_exit+0x28 8388608 |@@@@ 15 trap+0xb80 16777216 |@@@ 12 33554432 |@@@@@ 18 67108864 |@@@@@@@ 22 134217728 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 92 7% 70% 0.00 18199359 vm object vm_object_deallocate+0x814 nsec ------ Time Distribution ------ count Stack 65536 | 1 0xc0000000030ea1fc 131072 | 2 _vm_map_unlock+0xf8 262144 |@ 5 vmspace_exit+0x194 524288 | 1 exit1+0x954 1048576 |@@ 9 sys_sys_exit+0x28 2097152 |@ 5 trap+0xb80 4194304 |@@@ 10 8388608 |@@@ 11 16777216 |@@@@ 14 33554432 |@@@@ 13 67108864 |@@@@@ 18 134217728 | 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 59 5% 75% 0.00 11272250 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1024 | 1 2048 | 0 4096 | 0 8192 | 0 16384 | 0 32768 | 0 65536 | 0 131072 |@@@@ 9 262144 |@@@@@ 11 524288 |@ 2 1048576 | 1 2097152 |@@ 5 4194304 |@@@ 7 8388608 |@ 2 16777216 |@@ 5 33554432 |@@@ 7 67108864 |@@@@ 8 134217728 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 47 4% 78% 0.00 24314366 vm object vm_object_deallocate+0x800 nsec ------ Time Distribution ------ count Stack 32768 | 1 kern_execve+0x9d0 65536 | 0 sys_execve+0x80 131072 | 0 trap+0xb80 262144 | 0 powerpc_interrupt+0x1a4 524288 | 0 1048576 |@ 2 2097152 |@ 2 4194304 |@@ 4 8388608 |@@@ 6 16777216 |@@@@@ 9 33554432 |@@@@@ 9 67108864 |@@@@@@@ 11 134217728 |@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 40 3% 81% 0.00 16617138 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 262144 |@ 2 0xc000000002c479cc 524288 | 1 0xc000000002c472c0 1048576 | 1 0xc000000002c4684c 2097152 |@@@ 5 kern_execve+0x678 4194304 |@@ 3 sys_execve+0x80 8388608 |@@@@@@ 8 trap+0xb80 16777216 |@@@@ 6 33554432 |@@@@@@ 9 67108864 |@@@ 4 134217728 | 0 268435456 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 31 2% 84% 0.00 12009697 vm object vm_object_shadow+0x378 nsec ------ Time Distribution ------ count Stack 131072 | 1 vm_map_lookup+0xb80 262144 | 0 vm_fault_hold+0xf8 524288 | 0 vm_fault+0x94 1048576 |@@ 3 trap+0x10ac 2097152 | 1 powerpc_interrupt+0x1a4 4194304 |@@@@@@ 7 8388608 |@@@@ 5 16777216 |@@@@@@@ 8 33554432 |@@ 3 67108864 |@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 31 2% 86% 0.00 14669475 vm object vm_map_insert+0x758 nsec ------ Time Distribution ------ count Stack 1048576 |@@ 3 vm_map_fixed+0x120 2097152 |@ 2 0xc000000002c47a14 4194304 |@@@@ 5 0xc000000002c472c0 8388608 |@@@@ 5 0xc000000002c4684c 16777216 |@@@ 4 kern_execve+0x678 33554432 |@@@@@@ 7 sys_execve+0x80 67108864 |@@@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 23 2% 88% 0.00 23474727 vm object vm_imgact_map_page+0x3f0 nsec ------ Time Distribution ------ count Stack 262144 |@ 1 0xc000000002c4733c 524288 | 0 0xc000000002c4684c 1048576 |@ 1 kern_execve+0x678 2097152 | 0 sys_execve+0x80 4194304 |@ 1 trap+0xb80 8388608 |@@@ 3 powerpc_interrupt+0x1a4 16777216 |@@@@@@@ 6 33554432 |@@@@@ 4 67108864 |@@@@@@@ 6 134217728 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 1% 89% 0.00 22269 vm object vm_fault_hold+0x1298 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@ 4 vm_fault+0x94 16384 |@@@@@@@@@@@@@@ 7 trap+0x10ac 32768 |@@ 1 powerpc_interrupt+0x1a4 65536 |@@@@ 2 131072 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 1% 90% 0.00 85259 vm object vnode_create_vobject+0x18c nsec ------ Time Distribution ------ count Stack 256 |@@ 1 0xc0000000030c5fe0 512 | 0 VOP_OPEN_APV+0x98 1024 |@@ 1 vn_open_vnode+0x200 2048 |@@ 1 vn_open_cred+0x420 4096 | 0 vn_open+0x30 8192 | 0 kern_openat+0x274 16384 |@@ 1 32768 | 0 65536 | 0 131072 |@@@@@@@@@@@@@@@@@@@@@ 10 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 1% 91% 0.00 19748156 vm object vm_object_shadow+0x390 nsec ------ Time Distribution ------ count Stack 2097152 |@@ 1 vm_map_lookup+0xb80 4194304 |@@ 1 vm_fault_hold+0xf8 8388608 |@@@@ 2 vm_fault+0x94 16777216 |@@@@ 2 trap+0x10ac 33554432 |@@@@@@@@@@ 5 powerpc_interrupt+0x1a4 67108864 |@@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 1% 92% 0.00 47177 vm object vm_fault_hold+0x2700 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@ 3 vm_fault+0x94 32768 |@@@@@@@@@@@ 5 trap+0x10ac 65536 |@@@@ 2 powerpc_interrupt+0x1a4 131072 |@@ 1 262144 |@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 1% 93% 0.00 22617644 vm object (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@ 1 0xc0000000030c5fe0 524288 | 0 VOP_OPEN_APV+0x98 1048576 | 0 exec_check_permissions+0x164 2097152 |@@ 1 kern_execve+0x348 4194304 |@@ 1 sys_execve+0x80 8388608 |@@ 1 trap+0xb80 16777216 |@@ 1 33554432 |@@@@@ 2 67108864 |@@@@@@@@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 1% 94% 0.00 14782485 vm object (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@ 1 0xc0000000030ea1fc 524288 | 0 _vm_map_unlock+0xf8 1048576 | 0 vmspace_exit+0x194 2097152 |@@@@@@ 2 exit1+0x954 4194304 |@@@ 1 sys_sys_exit+0x28 8388608 |@@@@@@ 2 trap+0xb80 16777216 |@@@ 1 33554432 |@@@ 1 67108864 |@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 1% 95% 0.00 20501650 vm object (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@ 1 0xc0000000030ea1fc 524288 | 0 _vm_map_unlock+0xf8 1048576 | 0 vmspace_exit+0x194 2097152 | 0 exit1+0x954 4194304 | 0 sys_sys_exit+0x28 8388608 |@@@ 1 trap+0xb80 16777216 | 0 33554432 |@@@@@@@@@@@@@@@@@@@@@@@ 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 1% 95% 0.00 14855985 vm object exec_map_first_page+0x780 nsec ------ Time Distribution ------ count Stack 2097152 |@@@@ 1 kern_execve+0x37c 4194304 |@@@@ 1 sys_execve+0x80 8388608 | 0 trap+0xb80 16777216 |@@@@@@@@ 2 powerpc_interrupt+0x1a4 33554432 |@@@@@@@@ 2 67108864 |@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 96% 0.00 113659 vm object (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 6 0xc0000000030c5fe0 VOP_OPEN_APV+0x98 vn_open_vnode+0x200 vn_open_cred+0x420 vn_open+0x30 kern_openat+0x274 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 96% 0.00 18202521 vm object (usermode) nsec ------ Time Distribution ------ count Stack 4194304 |@@@@@ 1 0xc000000002c479cc 8388608 | 0 0xc000000002c472c0 16777216 |@@@@@@@@@@ 2 0xc000000002c4684c 33554432 |@@@@@@@@@@@@@@@ 3 kern_execve+0x678 sys_execve+0x80 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 97% 0.00 9650412 vm object (usermode) nsec ------ Time Distribution ------ count Stack 2097152 |@@@@@@@@@@ 2 vm_map_delete+0x1bc 4194304 |@@@@@@@@@@ 2 vmspace_exit+0x184 8388608 | 0 exit1+0x954 16777216 |@@@@@ 1 sys_sys_exit+0x28 33554432 | 0 trap+0xb80 67108864 |@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 97% 0.00 46707 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@ 1 32768 |@@@@@@@@@@ 2 65536 |@@@@@ 1 131072 |@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 98% 0.00 43733 vm object (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@ 1 16384 |@@@@@@ 1 32768 |@@@@@@ 1 65536 |@@@@@@ 1 131072 | 0 262144 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 15283626 vm object (usermode) nsec ------ Time Distribution ------ count Stack 4194304 |@@@@@@@ 1 kern_execve+0x9d0 8388608 |@@@@@@@ 1 sys_execve+0x80 16777216 |@@@@@@@ 1 trap+0xb80 33554432 | 0 powerpc_interrupt+0x1a4 67108864 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 117118 vm object (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 4 vm_map_delete+0x1bc kern_munmap+0xc4 sys_munmap+0x2c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 99% 0.00 17531833 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@ 1 0xc000000002c4733c 33554432 |@@@@@@@@@@@@@@@@@@@@@@ 3 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 104503 vm object 0xc000000003117de8 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@ 1 vm_pager_allocate+0x5c 131072 |@@@@@@@@@@ 1 vm_mmap_vnode+0x2c8 262144 |@@@@@@@@@@ 1 0xc000000002e32684 kern_mmap+0x418 sys_mmap+0x3c trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 116764 vm object 0xc0000000030f0be8 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 3 vm_map_delete+0x1bc kern_munmap+0xc4 sys_munmap+0x2c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 15411099 vm object (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@ 1 vm_map_lookup+0xb80 262144 | 0 vm_fault_hold+0xf8 524288 | 0 vm_fault+0x94 1048576 | 0 trap+0x10ac 2097152 | 0 powerpc_interrupt+0x1a4 4194304 | 0 8388608 | 0 16777216 |@@@@@@@@@@ 1 33554432 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 24973038 vm object vm_object_reference+0x134 nsec ------ Time Distribution ------ count Stack 8388608 |@@@@@@@@@@@@@@@ 1 kern_execve+0x370 16777216 | 0 sys_execve+0x80 33554432 | 0 trap+0xb80 67108864 |@@@@@@@@@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 17225619 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@ 1 vm_map_fixed+0x120 33554432 |@@@@@@@@@@@@@@@ 1 0xc000000002c47a14 0xc000000002c472c0 0xc000000002c4684c kern_execve+0x678 sys_execve+0x80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 332 bufobj interlock bqrelse+0x2a0 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 bdwrite+0x2a8 0xc0000000030750c0 0xc0000000030a5abc 0xc000000003094220 0xc00000000309f878 0xc000000003085264 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 120221 vm object vmspace_fork+0x2d4 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 fork1+0x4bc sys_fork+0x5c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 31914710 vm object (usermode) nsec ------ Time Distribution ------ count Stack 33554432 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_map_lookup+0xb80 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 223 vm object (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030f0a28 vm_map_delete+0x1bc vmspace_exit+0x184 exit1+0x954 sys_sys_exit+0x28 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 12002702 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16777216 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 kern_execve+0x37c sys_execve+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- R/W reader spin on writer: 494 events in 21.178 seconds (23 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 230 47% 47% 0.00 6111810 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 2048 | 1 0xc0000000030ebe78 4096 | 1 vm_map_insert+0x6a8 8192 | 1 vm_map_fixed+0x120 16384 | 4 0xc000000002c47a14 32768 | 3 0xc000000002c472c0 65536 | 6 0xc000000002c4684c 131072 | 4 262144 | 6 524288 |@ 12 1048576 |@@@ 25 2097152 |@@@@@ 42 4194304 |@@@@ 33 8388608 |@@@@ 32 16777216 |@@@ 28 33554432 |@@@@ 32 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 133 27% 73% 0.00 5547301 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 8192 | 2 vm_map_entry_set_vnode_text+0x4c 16384 | 2 0xc0000000030ea1e4 32768 | 3 _vm_map_unlock+0xf8 65536 | 3 vmspace_exit+0x194 131072 | 1 exit1+0x954 262144 |@ 7 sys_sys_exit+0x28 524288 |@ 5 1048576 | 4 2097152 |@@@@ 21 4194304 |@@@@@ 25 8388608 |@@@@@@@@ 37 16777216 |@@ 11 33554432 |@@ 12 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 43 9% 82% 0.00 2238942 vm object (usermode) nsec ------ Time Distribution ------ count Stack 16384 | 1 vm_map_entry_set_vnode_text+0x4c 32768 | 1 0xc0000000030ea1e4 65536 | 0 _vm_map_unlock+0xf8 131072 |@ 2 vmspace_exit+0x194 262144 |@@@@ 6 exit1+0x954 524288 |@@ 4 sys_sys_exit+0x28 1048576 |@@@@@@ 10 2097152 |@@@@@ 8 4194304 |@@@@ 7 8388608 |@ 2 16777216 | 1 33554432 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 28 6% 88% 0.00 425030 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 1024 |@ 1 vm_map_entry_set_vnode_text+0x64 2048 |@ 1 0xc0000000030ea1e4 4096 |@ 1 _vm_map_unlock+0xf8 8192 |@ 1 vmspace_exit+0x194 16384 |@@ 2 exit1+0x954 32768 |@ 1 sys_sys_exit+0x28 65536 | 0 131072 |@ 1 262144 |@@@@ 4 524288 |@@@@@@@ 7 1048576 |@@@@ 4 2097152 |@@@@@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 25 5% 93% 0.00 34335 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 512 |@ 1 vm_fault_hold+0x12a4 1024 |@@ 2 vm_fault+0x94 2048 |@@@ 3 trap+0x10ac 4096 |@@@@ 4 powerpc_interrupt+0x1a4 8192 |@ 1 16384 |@@@ 3 32768 | 0 65536 |@@@@@@@@ 7 131072 |@@@ 3 262144 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 17 3% 96% 0.00 8057104 vm object (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@ 1 0xc0000000030ebe78 131072 | 0 vm_map_insert+0x6a8 262144 | 0 vm_map_fixed+0x120 524288 | 0 0xc000000002c47a14 1048576 |@@@ 2 0xc000000002c472c0 2097152 |@@@ 2 0xc000000002c4684c 4194304 |@@@@@ 3 8388608 |@@@@@@@ 4 16777216 |@@@ 2 33554432 |@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 13 3% 99% 0.00 29424 vm object (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@@@ 2 vm_fault_hold+0x12a4 8192 |@@@@@@ 3 vm_fault+0x94 16384 |@@@@@@@@@ 4 trap+0x10ac 32768 | 0 powerpc_interrupt+0x1a4 65536 |@@@@@@ 3 131072 | 0 262144 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 41885 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 0xc0000000030ebe78 vm_map_insert+0x6a8 vm_map_find+0x4f0 vm_map_find_min+0x98 vm_mmap_object+0x46c 0xc000000002e326c4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 100% 0.00 44396 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 2 0xc0000000030ebe78 vm_map_insert+0x6a8 vm_map_fixed+0x120 vm_mmap_object+0x3ac 0xc000000002e326c4 kern_mmap+0x418 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 665829 vm object (usermode) nsec ------ Time Distribution ------ count Stack 1048576 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_map_entry_set_vnode_text+0x64 0xc0000000030ea1e4 _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- R/W reader spin on write wanted: 50 events in 21.178 seconds (2 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 33 66% 66% 0.00 5171190 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 262144 | 1 0xc0000000030ebe78 524288 |@@@@ 5 vm_map_insert+0x6a8 1048576 |@@@ 4 vm_map_fixed+0x120 2097152 |@@@ 4 0xc000000002c47a14 4194304 |@@@@ 5 0xc000000002c472c0 8388608 |@@@@@@@ 8 0xc000000002c4684c 16777216 |@@ 3 33554432 |@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 28% 94% 0.00 8066329 vm object __rw_rlock_int+0x9c nsec ------ Time Distribution ------ count Stack 2097152 |@@@@@@ 3 vm_map_entry_set_vnode_text+0x4c 4194304 |@@@@@@@@ 4 0xc0000000030ea1e4 8388608 |@@@@ 2 _vm_map_unlock+0xf8 16777216 |@@@@@@ 3 vmspace_exit+0x194 33554432 |@@@@ 2 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 4% 98% 0.00 931337 vm object (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@ 1 vm_map_entry_set_vnode_text+0x4c 1048576 | 0 0xc0000000030ea1e4 2097152 |@@@@@@@@@@@@@@@ 1 _vm_map_unlock+0xf8 vmspace_exit+0x194 exit1+0x954 sys_sys_exit+0x28 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 2% 100% 0.00 1225998 vm object (usermode) nsec ------ Time Distribution ------ count Stack 2097152 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000030ebe78 vm_map_insert+0x6a8 vm_map_fixed+0x120 0xc000000002c47a14 0xc000000002c472c0 0xc000000002c4684c ------------------------------------------------------------------------------- SX exclusive block: 67 events in 21.178 seconds (3 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19 28% 28% 0.00 67435 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@ 7 vm_map_find+0xf0 65536 |@@@@@@@@@ 6 vm_map_find_min+0x98 131072 |@@@ 2 vm_mmap_object+0x46c 262144 |@@@@@@ 4 kern_mmap+0x450 sys_mmap+0x3c trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 17 25% 54% 0.00 65029 vm map (user) _vm_map_lock+0x138 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@ 4 vm_map_find+0xf0 65536 |@@@@@@@@@@@@@@ 8 vm_map_find_min+0x98 131072 |@@@@@@@ 4 vm_mmap_object+0x46c 262144 |@ 1 kern_mmap+0x450 sys_mmap+0x3c trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 15% 69% 0.00 97392 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@ 3 vm_map_lookup+0xd68 65536 |@@@@@@@@@@@@ 4 vm_fault_hold+0xf8 131072 |@@@ 1 vm_fault+0x94 262144 |@@@ 1 trap+0x10ac 524288 |@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 13% 82% 0.00 45277 vq->vq_lock (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@ 1 0xc0000000026bdce0 32768 |@@@@@@ 2 zio_execute+0x19c 65536 |@@@@@@@@@@@@@@@@ 5 zio_nowait+0x118 131072 | 0 0xc000000002653d80 262144 |@@@ 1 0xc0000000026bddb0 zio_execute+0x19c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 6% 88% 0.00 41360 vm map (user) _vm_map_lock_upgrade+0x180 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 2 vm_map_lookup+0xd68 65536 |@@@@@@@ 1 vm_fault_hold+0xf8 131072 |@@@@@@@ 1 vm_fault+0x94 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 4% 93% 0.00 133075 vq->vq_lock vdev_queue_io+0x268 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@ 1 0xc0000000026bdce0 131072 | 0 zio_execute+0x19c 262144 |@@@@@@@@@@@@@@@@@@@@ 2 zio_nowait+0x118 0xc000000002653d80 0xc0000000026bddb0 zio_execute+0x19c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 3% 96% 0.00 145092 vm map (user) _vm_map_lock+0x138 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@ 1 vm_map_protect+0xc4 65536 | 0 sys_mprotect+0x80 131072 | 0 trap+0xb80 262144 | 0 powerpc_interrupt+0x1a4 524288 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 1% 97% 0.00 159847 vq->vq_lock (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026be130 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 1% 99% 0.00 174131 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_map_protect+0xc4 sys_mprotect+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 1% 100% 0.00 28179 ms->ms_lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 metaslab_alloc+0x110 0xc0000000026bd06c zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 ------------------------------------------------------------------------------- SX shared block: 6 events in 21.178 seconds (0 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 83% 83% 0.00 212095 vm map (user) _sx_slock_int+0xa0 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@ 2 vm_map_lookup+0x170 262144 |@@@@@@ 1 vm_fault_hold+0xf8 524288 |@@@@@@@@@@@@ 2 vm_fault+0x94 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 17% 100% 0.00 85576 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 vm_map_lookup+0x170 vm_fault_hold+0xf8 vm_fault+0x94 trap+0x10ac powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- SX exclusive spin: 4201 events in 21.178 seconds (198 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 642 15% 15% 0.00 626518 tm->tm_lock trim_map_free+0x19c nsec ------ Time Distribution ------ count Stack 256 | 3 0xc0000000026bdb00 512 | 10 zio_execute+0x19c 1024 | 8 zio_nowait+0x118 2048 | 19 0xc000000002653d80 4096 | 21 0xc0000000026bddb0 8192 | 14 zio_execute+0x19c 16384 |@ 33 32768 |@@ 61 65536 |@ 28 131072 |@@ 46 262144 |@@@ 79 524288 |@@@@ 90 1048576 |@@@@@ 110 2097152 |@@ 60 4194304 |@@ 54 8388608 | 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 282 7% 22% 0.00 196258 vq->vq_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 | 3 0xc0000000026bdce0 512 | 7 zio_execute+0x19c 1024 |@ 13 zio_nowait+0x118 2048 |@ 15 0xc000000002653d80 4096 |@ 10 0xc0000000026bddb0 8192 | 9 zio_execute+0x19c 16384 |@@ 25 32768 |@@@ 37 65536 |@@@@@ 47 131072 |@@@ 36 262144 |@@@ 34 524288 |@@ 23 1048576 |@ 14 2097152 | 6 4194304 | 1 8388608 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 211 5% 27% 0.00 96281 zio->io_lock 0xc0000000026bf498 nsec ------ Time Distribution ------ count Stack 256 | 2 zio_execute+0x19c 512 | 0 0xc000000002653d98 1024 | 2 0xc0000000026bddb0 2048 |@ 10 zio_execute+0x19c 4096 |@ 9 taskq_dispatch_ent+0xa4 8192 |@@ 20 0xc000000002d75088 16384 |@@@@@ 39 32768 |@@@@@@ 43 65536 |@ 12 131072 |@@ 21 262144 |@@@ 27 524288 |@@ 20 1048576 | 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 198 5% 32% 0.00 635133 tm->tm_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 | 2 0xc0000000026bdb00 512 | 3 zio_execute+0x19c 1024 | 2 zio_nowait+0x118 2048 | 2 0xc000000002653d80 4096 | 6 0xc0000000026bddb0 8192 | 2 zio_execute+0x19c 16384 | 5 32768 |@ 13 65536 |@ 12 131072 |@ 12 262144 |@@@ 22 524288 |@@@@@ 38 1048576 |@@@@@@ 41 2097152 |@@@ 25 4194304 |@ 13 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 192 5% 36% 0.00 139934 mg->mg_lock metaslab_alloc_dva+0x3a8 nsec ------ Time Distribution ------ count Stack 256 | 4 metaslab_alloc+0x110 512 | 3 0xc0000000026bd06c 1024 | 4 zio_execute+0x19c 2048 | 4 taskq_dispatch_ent+0xa4 4096 | 6 0xc000000002d75088 8192 |@@ 13 taskqueue_thread_loop+0x158 16384 |@@@ 24 32768 |@@@@@ 32 65536 |@@ 18 131072 |@@@ 22 262144 |@@@ 24 524288 |@@@@ 26 1048576 |@ 11 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 188 4% 41% 0.00 540583 os->os_userused_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 0xc00000000256ae50 512 | 1 0xc000000002d75088 1024 | 3 taskqueue_thread_loop+0x158 2048 | 4 fork_exit+0xdc 4096 | 4 0xc000000003185c5c 8192 |@ 7 16384 |@ 11 32768 |@ 12 65536 |@ 11 131072 |@ 12 262144 |@@ 18 524288 |@@@@ 29 1048576 |@@@@@@@ 45 2097152 |@@@ 25 4194304 | 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 186 4% 45% 0.00 182060 vq->vq_lock vdev_queue_io+0x268 nsec ------ Time Distribution ------ count Stack 256 | 3 0xc0000000026bdce0 512 | 3 zio_execute+0x19c 1024 | 3 zio_nowait+0x118 2048 |@ 10 0xc000000002653d80 4096 | 5 0xc0000000026bddb0 8192 |@ 9 zio_execute+0x19c 16384 | 6 32768 |@@@@@ 35 65536 |@@@@ 25 131072 |@@@@ 25 262144 |@@@ 22 524288 |@@@ 21 1048576 |@@ 15 2097152 | 3 4194304 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 167 4% 49% 0.00 116943 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@ 8 0xc0000000026bd95c 512 | 3 zio_execute+0x19c 1024 | 1 taskq_dispatch_ent+0xa4 2048 | 5 0xc000000002d75088 4096 |@ 9 taskqueue_thread_loop+0x158 8192 |@@ 15 fork_exit+0xdc 16384 |@@@ 18 32768 |@@@ 18 65536 |@@@ 17 131072 |@@@@ 23 262144 |@@@@ 25 524288 |@@@ 18 1048576 |@ 6 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 143 3% 53% 0.00 179129 mg->mg_lock metaslab_alloc_dva+0x7d4 nsec ------ Time Distribution ------ count Stack 256 |@ 5 metaslab_alloc+0x110 512 | 3 0xc0000000026bd06c 1024 | 2 zio_execute+0x19c 2048 | 2 taskq_dispatch_ent+0xa4 4096 | 4 0xc000000002d75088 8192 | 4 taskqueue_thread_loop+0x158 16384 |@@ 12 32768 |@@@@@ 24 65536 |@@@@ 21 131072 |@@@ 19 262144 |@@@ 16 524288 |@@@ 19 1048576 |@ 7 2097152 |@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 137 3% 56% 0.00 42306 vq->vq_lock vdev_queue_io_done+0x278 nsec ------ Time Distribution ------ count Stack 256 | 4 0xc0000000026be130 512 |@ 7 zio_execute+0x19c 1024 |@ 8 taskq_dispatch_ent+0xa4 2048 |@ 8 0xc000000002d75088 4096 |@ 9 taskqueue_thread_loop+0x158 8192 |@@@ 15 fork_exit+0xdc 16384 |@@@@ 19 32768 |@@@ 16 65536 |@@@@ 22 131072 |@@@@ 19 262144 |@ 7 524288 | 2 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 115 3% 59% 0.00 44841 vq->vq_lock vdev_queue_io_done+0x12c nsec ------ Time Distribution ------ count Stack 256 | 2 0xc0000000026be130 512 | 1 zio_execute+0x19c 1024 |@ 6 taskq_dispatch_ent+0xa4 2048 |@ 6 0xc000000002d75088 4096 |@ 4 taskqueue_thread_loop+0x158 8192 |@@@@ 18 fork_exit+0xdc 16384 |@@@ 14 32768 |@@@@@@@@ 34 65536 |@@@ 13 131072 |@ 5 262144 |@@ 8 524288 | 3 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 110 3% 61% 0.00 511478 os->os_userused_lock 0xc0000000025b7700 nsec ------ Time Distribution ------ count Stack 512 |@ 5 0xc00000000256ae50 1024 | 1 0xc000000002d75088 2048 | 3 taskqueue_thread_loop+0x158 4096 | 2 fork_exit+0xdc 8192 |@@ 8 0xc000000003185c5c 16384 |@ 5 32768 |@ 5 65536 |@ 6 131072 |@@ 10 262144 |@@@@ 16 524288 |@@@@ 16 1048576 |@@@ 12 2097152 |@@@@ 16 4194304 |@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 100 2% 64% 0.00 92704 vm map (user) _vm_map_lock+0x138 nsec ------ Time Distribution ------ count Stack 2048 | 2 vm_map_find+0xf0 4096 | 0 vm_map_find_min+0x98 8192 | 1 vm_mmap_object+0x46c 16384 | 1 kern_mmap+0x450 32768 |@@ 8 sys_mmap+0x3c 65536 |@@@@@@@@@ 31 trap+0xb80 131072 |@@@@@@@@@@@@ 40 262144 |@@@@ 14 524288 | 2 1048576 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 95 2% 66% 0.00 455290 os->os_userused_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 | 3 0xc00000000256ae50 512 | 0 0xc000000002d75088 1024 | 0 taskqueue_thread_loop+0x158 2048 | 0 fork_exit+0xdc 4096 | 0 0xc000000003185c5c 8192 | 0 16384 | 0 32768 |@@@ 10 65536 |@@ 7 131072 |@@@ 11 262144 |@@@ 12 524288 |@@@@@@ 22 1048576 |@@@@@@ 19 2097152 |@@ 9 4194304 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 87 2% 68% 0.00 119691 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 4096 | 1 vm_map_find+0xf0 8192 | 0 vm_map_find_min+0x98 16384 | 1 vm_mmap_object+0x46c 32768 |@ 5 kern_mmap+0x450 65536 |@@@@ 13 sys_mmap+0x3c 131072 |@@@@@@@@@@@@ 37 trap+0xb80 262144 |@@@@@@@@ 24 524288 |@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 86 2% 70% 0.00 62921 ms->ms_lock metaslab_alloc_dva+0x1138 nsec ------ Time Distribution ------ count Stack 512 |@ 4 metaslab_alloc+0x110 1024 | 2 0xc0000000026bd06c 2048 |@ 5 zio_execute+0x19c 4096 |@@@ 10 taskq_dispatch_ent+0xa4 8192 |@@@ 11 0xc000000002d75088 16384 |@@@ 11 taskqueue_thread_loop+0x158 32768 |@@@ 11 65536 |@ 4 131072 |@@@@ 14 262144 |@@@ 9 524288 |@ 5 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 73 2% 72% 0.00 83959 zio->io_lock zio_add_child+0x274 nsec ------ Time Distribution ------ count Stack 256 | 2 0xc0000000026b72d0 512 |@ 3 zio_free_sync+0x19c 1024 | 2 0xc000000002624e38 2048 |@ 3 bplist_iterate+0x194 4096 | 2 spa_sync+0xc48 8192 | 2 0xc000000002635b28 16384 |@@@@ 10 32768 |@@@ 9 65536 |@@@ 8 131072 |@@@@@@ 15 262144 |@@@@ 11 524288 |@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 64 2% 73% 0.00 642097 os->os_userused_lock 0xc0000000025b7898 nsec ------ Time Distribution ------ count Stack 32768 |@ 4 0xc00000000256ae50 65536 |@@@ 7 0xc000000002d75088 131072 |@@ 6 taskqueue_thread_loop+0x158 262144 |@@ 5 fork_exit+0xdc 524288 |@@@@@@ 13 0xc000000003185c5c 1048576 |@@@@@@@ 15 2097152 |@@@@@@ 13 4194304 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 63 1% 75% 0.00 16173 zp->z_acl_lock 0xc000000002678cac nsec ------ Time Distribution ------ count Stack 256 | 1 zfs_zaccess+0x128 512 |@@@ 8 zfs_zaccess_rwx+0x38 1024 |@@@@@@ 14 0xc0000000026a1488 2048 |@@@@ 10 VOP_ACCESS_APV+0x98 4096 |@@@ 8 vfs_cache_lookup+0xb0 8192 |@ 3 zfs_freebsd_aclcheck+0x74 16384 |@ 4 32768 |@@@ 8 65536 |@@ 6 131072 | 0 262144 | 0 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 61 1% 76% 0.00 152712 zio->io_lock 0xc0000000026c1474 nsec ------ Time Distribution ------ count Stack 256 | 1 0xc0000000026bf6ac 512 | 1 zio_execute+0x19c 1024 | 0 0xc000000002653d98 2048 | 0 0xc0000000026bddb0 4096 | 0 zio_execute+0x19c 8192 | 0 taskq_dispatch_ent+0xa4 16384 |@ 3 32768 |@@@@@@@@@@@@ 26 65536 | 2 131072 |@@@@@ 11 262144 |@@ 6 524288 |@@ 5 1048576 |@@ 5 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 55 1% 77% 0.00 241598 spa->spa_alloc_locks[i] (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 zio_execute+0x19c 512 | 1 taskq_dispatch_ent+0xa4 1024 |@ 3 0xc000000002d75088 2048 |@ 2 taskqueue_thread_loop+0x158 4096 |@ 2 fork_exit+0xdc 8192 |@ 2 0xc000000003185c5c 16384 |@ 2 32768 | 1 65536 |@ 2 131072 |@@@@@ 10 262144 |@@@ 6 524288 |@@@@@@@@@ 17 1048576 |@@@ 6 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 52 1% 79% 0.00 157796 spa->spa_alloc_locks[i] 0xc0000000026bcf70 nsec ------ Time Distribution ------ count Stack 512 |@ 2 zio_execute+0x19c 1024 |@ 2 taskq_dispatch_ent+0xa4 2048 |@@ 5 0xc000000002d75088 4096 |@ 3 taskqueue_thread_loop+0x158 8192 |@@ 5 fork_exit+0xdc 16384 |@@ 5 0xc000000003185c5c 32768 |@ 3 65536 |@ 3 131072 |@@@ 6 262144 |@@ 5 524288 |@@@ 6 1048576 |@@@@ 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 52 1% 80% 0.00 252727 mg->mg_lock (usermode) nsec ------ Time Distribution ------ count Stack 4096 |@@ 5 metaslab_alloc+0x110 8192 | 1 0xc0000000026bd06c 16384 |@@@ 6 zio_execute+0x19c 32768 |@@@@ 7 taskq_dispatch_ent+0xa4 65536 |@@@ 6 0xc000000002d75088 131072 |@@ 5 taskqueue_thread_loop+0x158 262144 |@@ 5 524288 |@@@@@ 10 1048576 |@ 3 2097152 |@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 51 1% 81% 0.00 180388 ms->ms_lock metaslab_free_concrete+0x2e8 nsec ------ Time Distribution ------ count Stack 256 | 1 0xc000000002604964 512 |@ 2 metaslab_free+0x190 1024 |@ 2 0xc0000000026bd5a4 2048 |@ 3 zio_execute+0x19c 4096 |@ 2 taskq_dispatch_ent+0xa4 8192 |@@ 5 0xc000000002d75088 16384 |@@ 4 32768 |@ 3 65536 |@ 2 131072 |@@@@ 7 262144 |@@ 5 524288 |@@@@ 8 1048576 |@@@@ 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 50 1% 82% 0.00 94763 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@ 2 vm_map_lookup+0xd68 32768 |@@ 4 vm_fault_hold+0xf8 65536 |@@@@ 7 vm_fault+0x94 131072 |@@@@@@@@@@@@@@@ 26 trap+0x10ac 262144 |@@@@@@ 11 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 48 1% 84% 0.00 48067 as->as_lock aggsum_add+0x360 nsec ------ Time Distribution ------ count Stack 256 |@ 3 0xc0000000025949b4 512 | 0 0xc00000000258c084 1024 | 1 0xc00000000258e910 2048 | 1 0xc0000000026bd790 4096 | 1 zio_execute+0x19c 8192 |@ 3 taskq_dispatch_ent+0xa4 16384 |@@@@@@ 10 32768 |@@@@@@ 11 65536 |@@@ 5 131072 |@@@@@@ 11 262144 | 1 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 44 1% 85% 0.00 82829 proctree kern_wait6+0x3e8 nsec ------ Time Distribution ------ count Stack 256 |@@ 3 sys_wait4+0xd8 512 | 0 trap+0xb80 1024 |@@@@ 6 powerpc_interrupt+0x1a4 2048 |@ 2 4096 |@@ 3 8192 |@@@@ 6 16384 |@@@@ 6 32768 |@@@@@@ 9 65536 |@@ 3 131072 |@ 2 262144 | 1 524288 | 0 1048576 |@ 2 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 39 1% 85% 0.00 24573 rrl->rr_lock 0xc00000000260a56c nsec ------ Time Distribution ------ count Stack 512 |@@ 3 0xc0000000026a1440 1024 | 1 VOP_ACCESS_APV+0x98 2048 | 0 vfs_cache_lookup+0xb0 4096 | 0 zfs_freebsd_aclcheck+0x74 8192 |@@@ 4 VOP_LOOKUP_APV+0x98 16384 |@@@@@@ 8 lookup+0x800 32768 |@@@@@@@@ 11 65536 |@@@@@@@@ 11 131072 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 39 1% 86% 0.00 242755 mg->mg_lock (usermode) nsec ------ Time Distribution ------ count Stack 512 | 1 metaslab_alloc+0x110 1024 | 0 0xc0000000026bd06c 2048 | 0 zio_execute+0x19c 4096 | 1 taskq_dispatch_ent+0xa4 8192 | 0 0xc000000002d75088 16384 | 1 taskqueue_thread_loop+0x158 32768 |@@@@ 6 65536 |@@@@ 6 131072 | 1 262144 |@@@@@@@@ 11 524288 |@@@ 5 1048576 |@@@@@ 7 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 38 1% 87% 0.00 145506 zio->io_lock 0xc0000000026c1474 nsec ------ Time Distribution ------ count Stack 256 |@ 2 0xc0000000026bd95c 512 |@@ 3 zio_execute+0x19c 1024 |@ 2 taskq_dispatch_ent+0xa4 2048 | 0 0xc000000002d75088 4096 | 0 taskqueue_thread_loop+0x158 8192 |@ 2 fork_exit+0xdc 16384 |@@@@ 6 32768 |@ 2 65536 |@@@ 5 131072 |@@@@@ 7 262144 |@@ 3 524288 |@@@ 4 1048576 | 1 2097152 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 37 1% 88% 0.00 129388 spa->spa_alloc_locks[i] zio_allocate_dispatch+0x264 nsec ------ Time Distribution ------ count Stack 512 | 1 0xc0000000026becfc 1024 | 1 zio_execute+0x19c 2048 |@ 2 taskq_dispatch_ent+0xa4 4096 | 1 0xc000000002d75088 8192 |@@ 3 taskqueue_thread_loop+0x158 16384 |@@@ 4 fork_exit+0xdc 32768 |@@@@ 6 65536 |@@@ 4 131072 |@@@@ 5 262144 |@ 2 524288 |@@@@ 5 1048576 |@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 33 1% 89% 0.00 91391 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 | 1 0xc0000000026b72d0 512 | 0 zio_free_sync+0x19c 1024 | 0 0xc000000002624e38 2048 | 1 bplist_iterate+0x194 4096 | 1 spa_sync+0xc48 8192 | 1 0xc000000002635b28 16384 |@@@@ 5 32768 | 1 65536 |@@@@ 5 131072 |@@@@@@@ 8 262144 |@@@@@@ 7 524288 |@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 33 1% 90% 0.00 41846 mc->mc_lock metaslab_class_throttle_unreserve+0x154 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@ 7 0xc0000000026becec 512 |@@@@ 5 zio_execute+0x19c 1024 | 0 taskq_dispatch_ent+0xa4 2048 | 0 0xc000000002d75088 4096 | 0 taskqueue_thread_loop+0x158 8192 | 1 fork_exit+0xdc 16384 |@@ 3 32768 |@@ 3 65536 |@@@@@ 6 131072 |@@@ 4 262144 |@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 26 1% 90% 0.00 83106 ms->ms_lock (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@ 1 metaslab_alloc+0x110 2048 |@@ 2 0xc0000000026bd06c 4096 |@@@@ 4 zio_execute+0x19c 8192 |@@@@ 4 taskq_dispatch_ent+0xa4 16384 |@@@@ 4 0xc000000002d75088 32768 | 0 taskqueue_thread_loop+0x158 65536 |@@@@ 4 131072 |@ 1 262144 |@@@ 3 524288 |@@ 2 1048576 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 25 1% 91% 0.00 51589 zio->io_lock zio_add_child+0x274 nsec ------ Time Distribution ------ count Stack 256 |@ 1 0xc0000000026b72d0 512 |@ 1 zio_free_sync+0x19c 1024 | 0 0xc000000002624e38 2048 | 0 0xc000000002598d38 4096 |@@ 2 spa_sync+0xef4 8192 |@@ 2 0xc000000002635b28 16384 |@@ 2 32768 |@@@@@@@ 6 65536 |@@ 2 131072 |@@@@@@@ 6 262144 |@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 20 0% 91% 0.00 96874 vm map (user) _vm_map_lock_upgrade+0x180 nsec ------ Time Distribution ------ count Stack 8192 |@ 1 vm_map_lookup+0xd68 16384 | 0 vm_fault_hold+0xf8 32768 | 0 vm_fault+0x94 65536 |@@@@@@@ 5 trap+0x10ac 131072 |@@@@@@@@@@@@@@@ 10 powerpc_interrupt+0x1a4 262144 |@@@@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 20 0% 92% 0.00 22410 rrl->rr_lock rrw_exit+0x310 nsec ------ Time Distribution ------ count Stack 256 |@@@ 2 rrm_exit+0x178 512 |@ 1 0xc0000000026a149c 1024 |@ 1 VOP_ACCESS_APV+0x98 2048 | 0 vfs_cache_lookup+0xb0 4096 |@ 1 zfs_freebsd_aclcheck+0x74 8192 | 0 VOP_LOOKUP_APV+0x98 16384 |@@@@@@ 4 32768 |@@@@@@@@@ 6 65536 |@@@@@@ 4 131072 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 19 0% 92% 0.00 22027 zp->z_acl_lock (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@ 2 zfs_zaccess+0x128 1024 |@@@@ 3 zfs_zaccess_rwx+0x38 2048 |@@@@@@@@@ 6 0xc0000000026a1488 4096 |@@@@@@ 4 VOP_ACCESS_APV+0x98 8192 |@ 1 vfs_cache_lookup+0xb0 16384 |@ 1 zfs_freebsd_aclcheck+0x74 32768 |@ 1 65536 | 0 131072 | 0 262144 | 0 524288 |@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 18 0% 93% 0.00 88758 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@ 1 0xc0000000026bf6ac 512 |@ 1 zio_execute+0x19c 1024 | 0 0xc000000002653d98 2048 | 0 0xc0000000026bddb0 4096 | 0 zio_execute+0x19c 8192 | 0 taskq_dispatch_ent+0xa4 16384 | 0 32768 |@@@@@@@@@@ 6 65536 |@@@ 2 131072 |@@@@@@ 4 262144 |@@@ 2 524288 |@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 15 0% 93% 0.00 45834 proctree (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@ 1 4096 |@@@@ 2 8192 |@@@@ 2 16384 | 0 32768 |@@@@@@ 3 65536 |@@@@@@ 3 131072 |@@@@ 2 262144 |@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 0% 94% 0.00 51276 db->db_mtx dmu_buf_will_dirty+0x2e8 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@ 3 dsl_dataset_block_kill+0x17c 32768 |@@@@@@@@ 4 0xc0000000025a616c 65536 |@@@@@@@@@@ 5 0xc00000000258ee9c 131072 |@@ 1 0xc0000000026bf390 262144 | 0 zio_execute+0x19c 524288 |@@ 1 taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 14 0% 94% 0.00 42315 db->db_mtx dmu_buf_will_dirty+0x2e8 nsec ------ Time Distribution ------ count Stack 256 |@@ 1 dsl_dataset_block_born+0x168 512 | 0 0xc0000000025a6180 1024 |@@ 1 0xc00000000258ee9c 2048 |@@@@ 2 0xc0000000026bf390 4096 | 0 zio_execute+0x19c 8192 | 0 taskq_dispatch_ent+0xa4 16384 | 0 32768 |@@@@@@ 3 65536 |@@@@@@@@@@ 5 131072 |@@ 1 262144 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 94% 0.00 87631 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@ 1 0xc0000000026b72d0 512 | 0 zio_free_sync+0x19c 1024 | 0 0xc000000002624e38 2048 | 0 0xc000000002598d38 4096 |@@ 1 spa_sync+0xef4 8192 |@@ 1 0xc000000002635b28 16384 | 0 32768 |@@ 1 65536 |@@@@@ 2 131072 |@@@@@ 2 262144 |@@@@@@@ 3 524288 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 94% 0.00 84006 vq->vq_lock (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@ 1 0xc0000000026be130 1024 | 0 zio_execute+0x19c 2048 |@@ 1 taskq_dispatch_ent+0xa4 4096 |@@ 1 0xc000000002d75088 8192 | 0 taskqueue_thread_loop+0x158 16384 |@@@@@ 2 fork_exit+0xdc 32768 |@@@@@@@@@@ 4 65536 |@@ 1 131072 | 0 262144 |@@ 1 524288 | 0 1048576 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 12 0% 95% 0.00 17375 dn->dn_struct_rwlock 0xc0000000025a6044 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@ 3 0xc00000000258e800 512 | 0 0xc0000000026bd790 1024 |@@@@@ 2 zio_execute+0x19c 2048 |@@ 1 taskq_dispatch_ent+0xa4 4096 |@@ 1 0xc000000002d75088 8192 | 0 taskqueue_thread_loop+0x158 16384 |@@@@@ 2 32768 |@@ 1 65536 |@@ 1 131072 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 95% 0.00 30144 scl->scl_lock spa_config_enter+0x140 nsec ------ Time Distribution ------ count Stack 256 |@@@@@ 2 metaslab_alloc+0xa0 512 |@@ 1 0xc0000000026bd06c 1024 | 0 zio_execute+0x19c 2048 |@@ 1 taskq_dispatch_ent+0xa4 4096 | 0 0xc000000002d75088 8192 |@@ 1 taskqueue_thread_loop+0x158 16384 |@@@@@@@@@@ 4 32768 | 0 65536 | 0 131072 |@@ 1 262144 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 95% 0.00 21687 rrl->rr_lock 0xc00000000260a56c nsec ------ Time Distribution ------ count Stack 256 |@@ 1 0xc0000000026a1638 512 | 0 VOP_GETATTR_APV+0x98 1024 | 0 vn_stat+0xb0 2048 |@@ 1 kern_statat+0xdc 4096 |@@@@@ 2 sys_fstatat+0x40 8192 |@@ 1 trap+0xb80 16384 | 0 32768 |@@@@@ 2 65536 |@@@@@@@@@@ 4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 11 0% 95% 0.00 84534 as->as_lock aggsum_add+0x360 nsec ------ Time Distribution ------ count Stack 1024 |@@ 1 0xc000000002594d88 2048 | 0 0xc0000000025887a4 4096 |@@ 1 arc_write+0x158 8192 |@@ 1 0xc0000000025a545c 16384 |@@ 1 0xc0000000025a4210 32768 |@@ 1 dbuf_sync_list+0xfc 65536 | 0 131072 |@@@@@@@@@@ 4 262144 |@@ 1 524288 |@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 96% 0.00 24846 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 512 |@@@@@@ 2 zio_execute+0x19c 1024 | 0 taskq_dispatch_ent+0xa4 2048 |@@@ 1 0xc000000002d75088 4096 | 0 taskqueue_thread_loop+0x158 8192 | 0 fork_exit+0xdc 16384 |@@@@@@@@@@@@ 4 0xc000000003185c5c 32768 |@@@@@@ 2 65536 | 0 131072 | 0 262144 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 96% 0.00 95785 vm map (user) _vm_map_lock+0x138 nsec ------ Time Distribution ------ count Stack 65536 |@@@ 1 vm_map_protect+0xc4 131072 |@@@@@@@@@@@@@@@@@@@@@ 7 sys_mprotect+0x80 262144 |@@@@@@ 2 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 10 0% 96% 0.00 15813 rrl->rr_lock (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@ 1 rrm_exit+0x178 2048 |@@@ 1 0xc0000000026a149c 4096 | 0 VOP_ACCESS_APV+0x98 8192 |@@@ 1 vfs_cache_lookup+0xb0 16384 |@@@@@@@@@ 3 zfs_freebsd_aclcheck+0x74 32768 |@@@@@@@@@ 3 VOP_LOOKUP_APV+0x98 65536 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 9 0% 96% 0.00 71740 as->as_lock aggsum_compare+0x3dc nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@ 2 0xc000000002594a28 16384 | 0 0xc00000000258c084 32768 | 0 0xc00000000258e910 65536 |@@@@@@ 2 0xc0000000026bd790 131072 |@@@@@@@@@@@@@ 4 zio_execute+0x19c 262144 |@@@ 1 taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 97% 0.00 48686 zio->io_lock 0xc0000000026c1474 nsec ------ Time Distribution ------ count Stack 256 |@@@ 1 0xc0000000026bf6ac 512 |@@@@@@@ 2 zio_execute+0x19c 1024 | 0 taskq_dispatch_ent+0xa4 2048 | 0 0xc000000002d75088 4096 | 0 taskqueue_thread_loop+0x158 8192 | 0 fork_exit+0xdc 16384 | 0 32768 |@@@ 1 65536 |@@@@@@@ 2 131072 |@@@ 1 262144 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 8 0% 97% 0.00 87865 scl->scl_lock spa_config_exit+0x84 nsec ------ Time Distribution ------ count Stack 8192 |@@@ 1 metaslab_alloc+0x184 16384 |@@@@@@@ 2 0xc0000000026bd06c 32768 | 0 zio_execute+0x19c 65536 |@@@@@@@ 2 taskq_dispatch_ent+0xa4 131072 |@@@ 1 0xc000000002d75088 262144 |@@@ 1 taskqueue_thread_loop+0x158 524288 |@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 97% 0.00 19424 zio->io_lock 0xc0000000026bf498 nsec ------ Time Distribution ------ count Stack 256 |@@@@ 1 zio_execute+0x19c 512 |@@@@@@@@ 2 taskq_dispatch_ent+0xa4 1024 | 0 0xc000000002d75088 2048 | 0 taskqueue_thread_loop+0x158 4096 | 0 fork_exit+0xdc 8192 | 0 0xc000000003185c5c 16384 | 0 32768 |@@@@@@@@ 2 65536 |@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 97% 0.00 566743 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 262144 |@@@@@@@@@@@@ 3 zio_execute+0x19c 524288 |@@@@ 1 0xc000000002653d98 1048576 |@@@@@@@@ 2 0xc0000000026bddb0 2097152 |@@@@ 1 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 97% 0.00 80616 vq->vq_lock 0xc000000002655fb4 nsec ------ Time Distribution ------ count Stack 32768 |@@@@ 1 vdev_queue_io_done+0x138 65536 |@@@@@@@@ 2 0xc0000000026be130 131072 |@@@@@@@@@@@@ 3 zio_execute+0x19c 262144 |@@@@ 1 taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 7 0% 97% 0.00 108688 vq->vq_lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@ 3 0xc0000000026be130 65536 | 0 zio_execute+0x19c 131072 |@@@@@@@@ 2 taskq_dispatch_ent+0xa4 262144 |@@@@@@@@ 2 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 6 0% 98% 0.00 24415 db->db_mtx dmu_buf_will_dirty+0x2e8 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@ 1 dsl_dir_diduse_space+0x94 2048 |@@@@@ 1 dsl_dataset_block_born+0x368 4096 | 0 0xc0000000025a6180 8192 | 0 0xc00000000258ee9c 16384 | 0 0xc0000000026bf390 32768 |@@@@@ 1 zio_execute+0x19c 65536 |@@@@@@@@@@@@@@@ 3 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 98% 0.00 21228 scl->scl_lock spa_config_exit+0x84 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@ 2 0xc0000000026be45c 512 | 0 zio_execute+0x19c 1024 | 0 taskq_dispatch_ent+0xa4 2048 | 0 0xc000000002d75088 4096 | 0 taskqueue_thread_loop+0x158 8192 |@@@@@@ 1 fork_exit+0xdc 16384 |@@@@@@ 1 32768 | 0 65536 | 0 131072 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 98% 0.00 23174 rrl->rr_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@ 1 0xc0000000026a1440 512 | 0 VOP_ACCESS_APV+0x98 1024 | 0 vfs_cache_lookup+0xb0 2048 | 0 zfs_freebsd_aclcheck+0x74 4096 | 0 VOP_LOOKUP_APV+0x98 8192 |@@@@@@ 1 lookup+0x800 16384 | 0 32768 |@@@@@@ 1 65536 |@@@@@@@@@@@@ 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 98% 0.00 20232 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@ 1 0xc0000000026bce7c 1024 | 0 zio_execute+0x19c 2048 | 0 taskq_dispatch_ent+0xa4 4096 | 0 0xc000000002d75088 8192 | 0 taskqueue_thread_loop+0x158 16384 |@@@@@@ 1 fork_exit+0xdc 32768 |@@@@@@@@@@@@ 2 65536 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 0% 98% 0.00 35396 as->as_lock (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@ 2 0xc000000002594d88 4096 |@@@@@@@@@@@@ 2 0xc0000000025887a4 8192 | 0 arc_write+0x158 16384 | 0 0xc0000000025a545c 32768 | 0 0xc0000000025a4210 65536 | 0 dbuf_sync_list+0xfc 131072 | 0 262144 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 23629 rrl->rr_lock rrw_exit+0x310 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@ 1 rrm_exit+0x178 2048 | 0 0xc0000000026a1f30 4096 | 0 VOP_GETATTR_APV+0x98 8192 | 0 vn_stat+0xb0 16384 | 0 kern_statat+0xdc 32768 |@@@@@@@@@@@@@@@ 2 sys_fstatat+0x40 65536 |@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 4 0% 98% 0.00 25474 db->db_mtx dmu_buf_will_dirty+0x2e8 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@ 1 dsl_dir_diduse_space+0x94 8192 |@@@@@@@ 1 dsl_dataset_block_kill+0x2e8 16384 |@@@@@@@ 1 0xc0000000025a616c 32768 | 0 0xc00000000258ee9c 65536 | 0 0xc0000000026bf390 131072 |@@@@@@@ 1 zio_execute+0x19c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 98% 0.00 121779 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@ 2 kern_munmap+0x9c 262144 |@@@@@@@@@@ 1 sys_munmap+0x2c trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 98% 0.00 190729 proctree fork1+0x28bc nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@ 1 sys_vfork+0x60 262144 |@@@@@@@@@@ 1 trap+0xb80 524288 |@@@@@@@@@@ 1 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 98% 0.00 20690 ds->ds_lock dsl_dataset_block_kill+0x5e4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@ 1 0xc0000000025a616c 512 | 0 0xc00000000258ee9c 1024 | 0 0xc0000000026bf390 2048 | 0 zio_execute+0x19c 4096 | 0 taskq_dispatch_ent+0xa4 8192 | 0 0xc000000002d75088 16384 | 0 32768 |@@@@@@@@@@ 1 65536 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 3 0% 99% 0.00 45849 db->db_mtx (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@ 1 dsl_dataset_block_born+0x168 4096 | 0 0xc0000000025a6180 8192 | 0 0xc00000000258ee9c 16384 |@@@@@@@@@@ 1 0xc0000000026bf390 32768 | 0 zio_execute+0x19c 65536 | 0 taskq_dispatch_ent+0xa4 131072 |@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 13060 zio->io_lock 0xc0000000026bda18 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 zio_execute+0x19c 16384 | 0 zio_wait+0x74 32768 |@@@@@@@@@@@@@@@ 1 spa_sync+0xf28 0xc000000002635b28 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 69095 vm map (user) _vm_map_lock+0x138 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 kern_munmap+0x9c 16384 | 0 sys_munmap+0x2c 32768 | 0 trap+0xb80 65536 | 0 powerpc_interrupt+0x1a4 131072 | 0 262144 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 515797 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@ 1 vm_map_protect+0xc4 1048576 |@@@@@@@@@@@@@@@ 1 sys_mprotect+0x80 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 2787 scl->scl_lock spa_config_enter+0x140 nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@ 1 0xc0000000026bdd5c 1024 | 0 zio_execute+0x19c 2048 | 0 taskq_dispatch_ent+0xa4 4096 | 0 0xc000000002d75088 8192 |@@@@@@@@@@@@@@@ 1 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 10279 scl->scl_lock (usermode) nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 0xc0000000026be45c 512 | 0 zio_execute+0x19c 1024 | 0 taskq_dispatch_ent+0xa4 2048 | 0 0xc000000002d75088 4096 | 0 taskqueue_thread_loop+0x158 8192 | 0 fork_exit+0xdc 16384 | 0 32768 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 103245 mc->mc_lock metaslab_class_throttle_reserve+0x1e8 nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@ 1 zio_allocate_dispatch+0xf0 131072 | 0 0xc0000000026becfc 262144 |@@@@@@@@@@@@@@@ 1 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 26376 ds->ds_lock dsl_dataset_block_kill+0x614 nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@ 1 0xc0000000025a616c 32768 | 0 0xc00000000258ee9c 65536 |@@@@@@@@@@@@@@@ 1 0xc0000000026bf390 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 82793 ds->ds_lock dsl_dataset_block_born+0x3d4 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 0xc0000000025a6180 512 | 0 0xc00000000258ee9c 1024 | 0 0xc0000000026bf390 2048 | 0 zio_execute+0x19c 4096 | 0 taskq_dispatch_ent+0xa4 8192 | 0 0xc000000002d75088 16384 | 0 32768 | 0 65536 | 0 131072 | 0 262144 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 22302 dn->dn_mtx 0xc0000000025a605c nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 0xc00000000258e800 512 | 0 0xc0000000026bd790 1024 | 0 zio_execute+0x19c 2048 | 0 taskq_dispatch_ent+0xa4 4096 | 0 0xc000000002d75088 8192 | 0 taskqueue_thread_loop+0x158 16384 | 0 32768 | 0 65536 |@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 511 db->db_mtx dmu_buf_will_dirty+0x2e8 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@ 1 dsl_dir_diduse_space+0x94 512 | 0 dsl_dir_diduse_space+0x1f4 1024 |@@@@@@@@@@@@@@@ 1 dsl_dataset_block_born+0x368 0xc0000000025a6180 0xc00000000258ee9c 0xc0000000026bf390 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 11563 db->db_mtx (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@ 1 dsl_dir_diduse_space+0x94 2048 | 0 dsl_dataset_block_born+0x368 4096 | 0 0xc0000000025a6180 8192 | 0 0xc00000000258ee9c 16384 | 0 0xc0000000026bf390 32768 |@@@@@@@@@@@@@@@ 1 zio_execute+0x19c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 13218 as->as_lock aggsum_add+0x360 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@ 1 0xc000000002594d88 16384 | 0 0xc000000002589300 32768 |@@@@@@@@@@@@@@@ 1 0xc000000002589110 dbuf_destroy+0x68 dbuf_rele_and_unlock+0x1d8 0xc0000000025a62b8 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 4872 as->as_lock aggsum_add+0x360 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@ 1 0xc000000002594d88 8192 |@@@@@@@@@@@@@@@ 1 0xc0000000025887a4 arc_write+0x158 0xc0000000025a545c 0xc0000000025a3bb0 dbuf_sync_list+0xec ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 2 0% 99% 0.00 20753 as->as_lock (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@ 1 0xc0000000025949b4 32768 |@@@@@@@@@@@@@@@ 1 0xc00000000258c084 0xc00000000258e910 0xc0000000026bd790 zio_execute+0x19c taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 88318 zio->io_lock zio_add_child+0x274 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026b72d0 zio_vdev_child_io+0x184 0xc000000002653d78 0xc0000000026bddb0 zio_execute+0x19c taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 1049 zio->io_lock zio_add_child+0x274 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026b72d0 zio_write+0xb8 arc_write+0x29c 0xc0000000025a545c 0xc0000000025a4210 dbuf_sync_list+0xfc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 41950 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026bf6ac zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 1725 zio->io_lock (usermode) nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc 0xc000000003185c5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 5824 vq->vq_lock vdev_queue_io+0x268 nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026bdce0 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 3148 tm->tm_lock trim_map_write_done+0x188 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026be124 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 287788 spa->spa_alloc_locks[i] (usermode) nsec ------ Time Distribution ------ count Stack 524288 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026becfc zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 45664 rrl->rr_lock 0xc00000000260a56c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026a117c VOP_OPEN_APV+0x98 vn_open_vnode+0x200 vn_open_cred+0x420 vn_open+0x30 kern_openat+0x274 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 832 rrl->rr_lock 0xc00000000260a56c nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026a0ba4 VOP_CREATE_APV+0x98 vn_open_cred+0x398 vn_open+0x30 kern_openat+0x274 sys_openat+0x58 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 20964 rrl->rr_lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026a1320 VOP_CLOSE_APV+0x98 0xc000000002e33890 0xc000000002e31db8 _fdrop+0x4c closef+0x39c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 99% 0.00 22585 rrl->rr_lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026a1440 VOP_ACCESS_APV+0x98 vn_open_vnode+0x140 vn_open_cred+0x420 vn_open+0x30 kern_openat+0x274 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 19952 rrl->rr_lock 0xc00000000260a56c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026a87e0 VOP_READLINK_APV+0x98 namei+0x750 kern_statat+0xb0 sys_fstatat+0x40 trap+0xb80 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 23285 rrl->rr_lock 0xc00000000260a56c nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026a1638 VOP_GETATTR_APV+0x98 vn_stat+0xb0 0xc000000002e31cac kern_fstat+0xec sys_fstat+0x30 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 39541 rrl->rr_lock 0xc00000000260a56c nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000026a1440 VOP_ACCESS_APV+0x98 vn_open_vnode+0x140 vn_open_cred+0x420 vn_open+0x30 kern_openat+0x274 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 800 rrl->rr_lock rrw_exit+0x310 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 rrm_exit+0x178 0xc0000000026a52f4 VOP_WRITE_APV+0x180 0xc000000002e37e00 0xc000000002e37790 0xc000000002e33f34 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21402 rrl->rr_lock rrw_exit+0x310 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 rrm_exit+0x178 0xc0000000026a889c VOP_READLINK_APV+0x98 namei+0x750 kern_statat+0xb0 sys_fstatat+0x40 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 2945 proctree exit1+0x1aec nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 sys_sys_exit+0x28 trap+0xb80 powerpc_interrupt+0x1a4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 15849 proctree (usermode) nsec ------ Time Distribution ------ count Stack 16384 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21076 dn->dn_mtx dbuf_dirty+0xd38 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 dnode_setdirty+0x100 dnode_set_blksz+0x4ec dmu_object_set_blocksize+0x74 zfs_grow_blocksize+0x6c 0xc0000000026a4b54 VOP_WRITE_APV+0x180 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 244 dd->dd_lock dsl_dir_diduse_space+0x270 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 dsl_dataset_block_born+0x368 0xc0000000025a6180 0xc00000000258ee9c 0xc0000000026bf390 zio_execute+0x19c taskq_dispatch_ent+0xa4 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 719 dn->dn_mtx (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 dnode_sync+0xc7c 0xc0000000025b70ec 0xc00000000256ae50 0xc000000002d75088 taskqueue_thread_loop+0x158 fork_exit+0xdc ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 240 dd->dd_lock dsl_dir_tempreserve_clear+0x88 nsec ------ Time Distribution ------ count Stack 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 dmu_tx_commit+0x1a4 zfs_inactive+0x22c 0xc0000000026a88fc VOP_INACTIVE_APV+0x98 vinactive+0x1ac 0xc000000002e1aa5c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 289 dp->dp_lock dsl_pool_undirty_space+0x18c nsec ------ Time Distribution ------ count Stack 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000025a60e0 0xc00000000258eaa8 0xc0000000026be790 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 24736 dp->dp_lock (usermode) nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000025a60e0 0xc00000000258eaa8 0xc0000000026be790 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 72939 dn->dn_struct_rwlock (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc00000000258e800 0xc0000000026bd790 zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 5916 mc->mc_lock (usermode) nsec ------ Time Distribution ------ count Stack 8192 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 zio_allocate_dispatch+0xf0 0xc0000000026becfc zio_execute+0x19c taskq_dispatch_ent+0xa4 0xc000000002d75088 taskqueue_thread_loop+0x158 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 42103 as->as_lock (usermode) nsec ------ Time Distribution ------ count Stack 65536 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000025949b4 0xc0000000025885b8 arc_read+0xb74 dbuf_read+0x820 dmu_buf_hold_array_by_dnode+0x31c dmu_read_uio_dnode+0x70 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21727 as->as_lock aggsum_add+0x360 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000025949b4 0xc00000000258c084 0xc0000000025882a4 arc_alloc_buf+0x48 dbuf_read+0x668 dnode_sync+0x90c ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 83322 as->as_lock (usermode) nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc000000002594d88 0xc000000002589300 arc_buf_destroy+0x190 0xc0000000025a6274 0xc00000000258ee9c 0xc0000000026bf390 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 21854 as->as_lock aggsum_add+0x360 nsec ------ Time Distribution ------ count Stack 32768 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 arc_space_consume+0xc8 0xc0000000025a2ca8 0xc0000000025a222c dbuf_hold+0x90 dmu_buf_hold_array_by_dnode+0x2f4 dmu_read_uio_dnode+0x70 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 627 bpl->bpl_lock (usermode) nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 zio_free+0x84 dsl_free+0x24 dsl_dataset_block_kill+0x1a8 0xc0000000025a616c 0xc00000000258ee9c 0xc0000000026bf390 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 85518 as->as_lock aggsum_add+0x360 nsec ------ Time Distribution ------ count Stack 131072 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 0xc0000000025966d8 0xc000000002569c40 uma_zalloc_arg+0xb4c kmem_cache_alloc+0x2c 0xc000000002588378 arc_alloc_buf+0x60 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 2969 bpl->bpl_lock bplist_append+0x158 nsec ------ Time Distribution ------ count Stack 4096 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 zio_free+0x84 dsl_free+0x24 dsl_dataset_block_kill+0x1a8 0xc0000000025a616c 0xc00000000258ee9c 0xc0000000026bf390 ------------------------------------------------------------------------------- SX shared spin: 305 events in 21.178 seconds (14 events/sec) ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 256 84% 84% 0.00 78676 vm map (user) _sx_slock_int+0xa0 nsec ------ Time Distribution ------ count Stack 2048 | 1 vm_map_lookup+0x170 4096 | 2 vm_fault_hold+0xf8 8192 | 1 vm_fault+0x94 16384 | 3 trap+0x10ac 32768 |@@@@ 38 powerpc_interrupt+0x1a4 65536 |@@@@@@@@@@ 87 131072 |@@@@@@@@@@ 87 262144 |@@@ 33 524288 | 2 1048576 | 2 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 43 14% 98% 0.00 59684 vm map (user) (usermode) nsec ------ Time Distribution ------ count Stack 512 | 1 vm_map_lookup+0x170 1024 | 1 vm_fault_hold+0xf8 2048 |@@ 3 vm_fault+0x94 4096 |@ 2 trap+0x10ac 8192 | 1 powerpc_interrupt+0x1a4 16384 |@@ 3 32768 |@@@@ 6 65536 |@@@@@@@@ 12 131072 |@@@@@@@ 11 262144 |@ 2 524288 | 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 5 2% 100% 0.00 21832 dn->dn_struct_rwlock _sx_slock_int+0xa0 nsec ------ Time Distribution ------ count Stack 2048 |@@@@@@ 1 0xc0000000025a4ca4 4096 |@@@@@@@@@@@@ 2 0xc0000000025a3f28 8192 | 0 dbuf_sync_list+0xfc 16384 | 0 0xc0000000025a3c24 32768 |@@@@@@ 1 dbuf_sync_list+0xec 65536 | 0 dnode_sync+0xc60 131072 |@@@@@@ 1 ------------------------------------------------------------------------------- Count indv cuml rcnt nsec Lock Caller 1 0% 100% 0.00 949 zf->zf_rwlock _sx_slock_int+0xa0 nsec ------ Time Distribution ------ count Stack 1024 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 1 dmu_zfetch+0xa8 dbuf_read+0xa7c dnode_hold_impl+0x264 dnode_hold+0x2c dmu_tx_hold_sa+0xa4 zfs_inactive+0x1c4 ------------------------------------------------------------------------------- root@fledgling:~ #