root@fledgling:~ # l cc -o stacktest stacktest.c Adaptive mutex spin: 4193 events in 0.245 seconds (17121 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 951 23% 23% 0.00 38304 pmap moea64_enter+0x880 689 16% 39% 0.00 40103 pmap moea64_is_prefaultable+0x13c 650 16% 55% 0.00 90773 page pv moea64_enter+0x86c 391 9% 64% 0.00 11651 page pv 0xc000000003142640 286 7% 71% 0.00 38002 page pv (usermode) 279 7% 77% 0.00 16878 umtxql (usermode) 274 7% 84% 0.00 29522 pmap (usermode) 141 3% 87% 0.00 587266 process lock ast+0x7e0 107 3% 90% 0.00 18412 umtxql 0xc000000002d228cc 63 2% 91% 0.00 24891 umtxql 0xc000000002d245d4 46 1% 92% 0.00 7141 umtxql 0xc000000002d22b78 40 1% 93% 0.00 259053 process lock (usermode) 36 1% 94% 0.00 194283 process lock kern_thr_exit+0x12c 35 1% 95% 0.00 188742 process lock kern_thr_exit+0x24c 28 1% 96% 0.00 17819 umtxql 0xc000000002d253f8 27 1% 96% 0.00 23094 umtxql 0xc000000002cc30c0 22 1% 97% 0.00 13549 vm page vm_page_change_lock+0x160 21 1% 97% 0.00 51732 page pv moea64_enter+0x910 18 0% 98% 0.00 4127 umtxql 0xc000000002d25428 12 0% 98% 0.00 54723 kernelpmap (usermode) 9 0% 98% 0.00 39267 kernelpmap moea64_enter+0x880 8 0% 99% 0.00 1500 umtxql 0xc000000002d24294 7 0% 99% 0.00 45877 umtxql 0xc000000002d2546c 7 0% 99% 0.00 18233 kernelpmap moea64_kextract+0x174 6 0% 99% 0.00 11044 vm page vm_page_sleep_if_busy+0x1c8 6 0% 99% 0.00 2682 vm page vm_fault_hold+0x2724 6 0% 99% 0.00 888 vm page (usermode) 6 0% 99% 0.00 10139 vm active pagequeue (usermode) 6 0% 100% 0.00 2416 vm active pagequeue 0xc00000000310e854 5 0% 100% 0.00 3233 vm page vm_fault_hold+0x27ec 5 0% 100% 0.00 14216 umtxql 0xc000000002d22e2c 3 0% 100% 0.00 24979 kernelpmap moea64_remove+0x200 2 0% 100% 0.00 2880 vm page vm_page_rename+0x224 1 0% 100% 0.00 3087 umtxql kern_umtx_wake+0x324 ------------------------------------------------------------------------------- Spin lock spin: 3546 events in 0.245 seconds (14479 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 2476 70% 70% 0.00 232041 turnstile chain turnstile_trywait+0x208 720 20% 90% 0.00 408178 turnstile chain (usermode) 224 6% 96% 0.00 55870 turnstile chain turnstile_chain_lock+0xdc 42 1% 98% 0.00 16502 sleepq chain sleepq_lock+0xe4 34 1% 99% 0.00 241 turnstile lock turnstile_trywait+0x150 25 1% 99% 0.00 1683 sleepq chain (usermode) 5 0% 99% 0.00 206 turnstile lock turnstile_lookup+0xd4 3 0% 100% 0.00 203 turnstile lock (usermode) 2 0% 100% 0.00 15858 sched lock 52 (usermode) 2 0% 100% 0.00 8627 sched lock 31 0xc000000002d3cccc 2 0% 100% 0.00 7354 sched lock 16 0xc000000002d3cccc 1 0% 100% 0.00 389 sched lock 65 sched_add+0x48c 1 0% 100% 0.00 2179 sched lock 59 (usermode) 1 0% 100% 0.00 279 sched lock 55 sched_add+0x48c 1 0% 100% 0.00 393 sched lock 6 sched_add+0x48c 1 0% 100% 0.00 1740 sched lock 63 sched_add+0x48c 1 0% 100% 0.00 461 sched lock 43 sched_add+0x48c 1 0% 100% 0.00 268 sched lock 35 (usermode) 1 0% 100% 0.00 157 sched lock 31 sched_add+0x48c 1 0% 100% 0.00 1746 sched lock 50 sched_add+0x48c 1 0% 100% 0.00 1078 sched lock 47 sched_add+0x48c 1 0% 100% 0.00 22379 sched lock 52 0xc000000002d3cce4 ------------------------------------------------------------------------------- Thread lock spin: 578 events in 0.245 seconds (2360 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 284 49% 49% 0.00 4974 sleepq chain _thread_lock+0xf4 218 38% 87% 0.00 4980 sleepq chain (usermode) 57 10% 97% 0.00 3899 turnstile lock _thread_lock+0xf4 7 1% 98% 0.00 4265 turnstile lock (usermode) 2 0% 98% 0.00 5836 sched lock 43 _thread_lock+0xf4 1 0% 98% 0.00 3329 sched lock 6 (usermode) 1 0% 99% 0.00 22835 sched lock 63 _thread_lock+0xf4 1 0% 99% 0.00 21230 sched lock 50 (usermode) 1 0% 99% 0.00 805 sched lock 51 (usermode) 1 0% 99% 0.00 1033 sched lock 52 _thread_lock+0xf4 1 0% 99% 0.00 11348 sched lock 23 _thread_lock+0xf4 1 0% 99% 0.00 342 sched lock 23 (usermode) 1 0% 100% 0.00 11479 sched lock 31 (usermode) 1 0% 100% 0.00 5989 sched lock 31 _thread_lock+0xf4 1 0% 100% 0.00 2964 sched lock 35 (usermode) ------------------------------------------------------------------------------- R/W writer blocked by writer: 236 events in 0.245 seconds (964 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 97 41% 41% 0.00 678238 vm object vm_page_sleep_if_busy+0x1f8 72 31% 72% 0.00 369333 vm object (usermode) 27 11% 83% 0.00 636606 vm object zfs_vmobject_wlock+0x9c 17 7% 90% 0.00 1223589 vm object 0xc00000000311fc9c 14 6% 96% 0.00 382936 vm object vm_fault_hold+0x2700 9 4% 100% 0.00 657001 vm object vm_object_deallocate+0x800 ------------------------------------------------------------------------------- R/W writer blocked by readers: 326 events in 0.245 seconds (1331 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 151 46% 46% 0.00 558352 vm object vm_fault_hold+0x1298 48 15% 61% 0.00 360071 vm object (usermode) 44 13% 75% 0.00 436553 vm object zfs_vmobject_wlock+0x9c 29 9% 83% 0.00 560377 vm object vm_object_deallocate+0x800 21 6% 90% 0.00 774658 vm object vm_fault_hold+0x2700 18 6% 95% 0.00 895554 vm object 0xc00000000311fc9c 13 4% 99% 0.00 663599 vm object vm_page_sleep_if_busy+0x1f8 1 0% 100% 0.00 6175521 vm object 0xc0000000030f8558 1 0% 100% 0.00 35801 vm object vm_fault_hold+0x1b28 ------------------------------------------------------------------------------- R/W reader blocked by writer: 69 events in 0.245 seconds (282 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 61 88% 88% 0.00 785584 vm object __rw_rlock_int+0x9c 8 12% 100% 0.00 112726 vm object (usermode) ------------------------------------------------------------------------------- R/W reader blocked by write wanted: 321 events in 0.245 seconds (1311 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 295 92% 92% 0.00 773112 vm object __rw_rlock_int+0x9c 26 8% 100% 0.00 798672 vm object (usermode) ------------------------------------------------------------------------------- R/W writer spin on writer: 1068 events in 0.245 seconds (4361 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 399 37% 37% 0.00 881916 vm object vm_page_sleep_if_busy+0x1f8 252 24% 61% 0.00 863103 vm object (usermode) 134 13% 74% 0.00 238071 vm object vm_fault_hold+0x2700 121 11% 85% 0.00 976950 vm object zfs_vmobject_wlock+0x9c 69 6% 91% 0.00 1335274 vm object 0xc00000000311fc9c 56 5% 97% 0.00 610321 vm object vm_object_deallocate+0x800 12 1% 98% 0.00 1060881 kernel vm object (usermode) 9 1% 99% 0.00 185632 kernel vm object 0xc0000000030f09c0 7 1% 99% 0.00 192870 kernel vm object kmem_back_domain+0x340 3 0% 99% 0.00 367780 vm object vm_fault_hold+0x1298 1 0% 100% 0.00 22562 vm object 0xc00000000311f65c 1 0% 100% 0.00 11419 vm object vm_object_shadow+0x378 1 0% 100% 0.00 12336 vm object vm_map_insert+0x598 1 0% 100% 0.00 260 vm object vm_fault_hold+0x1b28 1 0% 100% 0.00 801 Per-Filesystem Softdep Lock 0xc000000002cec288 1 0% 100% 0.00 295 tidhash (usermode) ------------------------------------------------------------------------------- R/W writer spin on readers: 928 events in 0.245 seconds (3789 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 362 39% 39% 0.00 850334 vm object vm_fault_hold+0x1298 178 19% 58% 0.00 718693 vm object (usermode) 130 14% 72% 0.00 748513 vm object zfs_vmobject_wlock+0x9c 104 11% 83% 0.00 622964 vm object vm_fault_hold+0x2700 65 7% 90% 0.00 967613 vm object 0xc00000000311fc9c 54 6% 96% 0.00 880699 vm object vm_object_deallocate+0x800 31 3% 100% 0.00 1242054 vm object vm_page_sleep_if_busy+0x1f8 3 0% 100% 0.00 47229 vm object vm_fault_hold+0x1b28 1 0% 100% 0.00 117287 vm object 0xc0000000030f8558 ------------------------------------------------------------------------------- R/W reader spin on writer: 774 events in 0.245 seconds (3160 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 689 89% 89% 0.00 279765 vm object __rw_rlock_int+0x9c 85 11% 100% 0.00 358773 vm object (usermode) ------------------------------------------------------------------------------- R/W reader spin on write wanted: 953 events in 0.245 seconds (3891 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 858 90% 90% 0.00 428311 vm object __rw_rlock_int+0x9c 95 10% 100% 0.00 547573 vm object (usermode) ------------------------------------------------------------------------------- SX exclusive block: 8 events in 0.245 seconds (33 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 8 100% 100% 0.00 1747132 vm map (user) (usermode) ------------------------------------------------------------------------------- SX shared block: 179 events in 0.245 seconds (731 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 174 97% 97% 0.00 1395402 vm map (user) _sx_slock_int+0xa0 5 3% 100% 0.00 1506760 vm map (user) (usermode) ------------------------------------------------------------------------------- SX exclusive spin: 427 events in 0.245 seconds (1744 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 190 44% 44% 0.00 33510 vm map (user) (usermode) 62 15% 59% 0.00 14373 vm map (user) _vm_map_lock+0x138 34 8% 67% 0.00 263296 h->hash_mutexes[i] dbuf_find+0x114 28 7% 74% 0.00 36065663 dn->dn_dbufs_mtx (usermode) 23 5% 79% 0.00 216696 db->db_mtx dbuf_find+0x12c 18 4% 83% 0.00 288616 db->db_mtx (usermode) 17 4% 87% 0.00 20167 vm map (user) _vm_map_lock_upgrade+0x180 17 4% 91% 0.00 201242 h->hash_mutexes[i] (usermode) 10 2% 93% 0.00 174092 h->hash_mutexes[i] dbuf_destroy+0x6dc 9 2% 96% 0.00 13740 db->db_mtx dbuf_read+0xa44 6 1% 97% 0.00 1625625 dn->dn_dbufs_mtx dbuf_destroy+0x724 6 1% 98% 0.00 6760 db->db_mtx dmu_buf_hold_array_by_dnode+0x6d0 2 0% 99% 0.00 44819 h->hash_mutexes[i] 0xc0000000025a2f24 2 0% 99% 0.00 587 db->db_mtx dbuf_rele+0xbc 1 0% 100% 0.00 293 filedesc structure (usermode) 1 0% 100% 0.00 287 dn->dn_mtx dbuf_destroy+0x6c0 1 0% 100% 0.00 79399 dn->dn_dbufs_mtx 0xc0000000025a2f0c ------------------------------------------------------------------------------- SX shared spin: 493 events in 0.245 seconds (2013 events/sec) Count indv cuml rcnt nsec Lock Caller ------------------------------------------------------------------------------- 448 91% 91% 0.00 89829 vm map (user) _sx_slock_int+0xa0 45 9% 100% 0.00 32095 vm map (user) (usermode) ------------------------------------------------------------------------------- root@fledgling:~ #