1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html><head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="generator" content="Roller Weblogger 4.0.0.14u1 (BSC)">
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="http://blogs.sun.com/ahl/rsd">
<link rel="alternate" type="application/atom+xml" title="Recent Entries (Atom)" href="http://blogs.sun.com/ahl/feed/entries/atom">
<link rel="alternate" type="application/rss+xml" title="Recent Entries (RSS)" href="http://blogs.sun.com/ahl/feed/entries/rss">
<link rel="alternate" type="application/atom+xml" title="Recent Comments (Atom)" href="http://blogs.sun.com/ahl/feed/comments/atom">
<link rel="alternate" type="application/rss+xml" title="Recent Comments (RSS)" href="http://blogs.sun.com/ahl/feed/comments/rss"><title>DTrace for Linux : Adam Leventhal's Weblog</title>
<link rel="stylesheet" type="text/css" media="all" href="dtrace_for_linux_files/pacifica-custom.css"></head><body bgcolor="#3c464d">
<div class="pagewrap">
<div class="innerpagewrap">
<!--
Main table layout
-->
<table class="layout" cellpadding="0" cellspacing="0">
<tbody>
<!--Masthead - spans all table columns -->
<tr><td colspan="3"><div class="masthead">
<span class="blogdesc">inside the sausage factory</span>
<span class="blogname">Adam Leventhal's Weblog</span>
</div></td></tr>
<!-- Subnav row -->
<tr><td class="subnav_left"></td>
<td class="subnav">
<ul class="categories">
<li class="selected">
<a href="http://blogs.sun.com/ahl/feed/entries/atom">
<img src="dtrace_for_linux_files/feed-12x.gif">
</a>
All
</li>
<li><a href="http://blogs.sun.com/ahl/category/DTrace">DTrace</a></li>
<li><a href="http://blogs.sun.com/ahl/category/Fishworks">Fishworks</a></li>
<li><a href="http://blogs.sun.com/ahl/category/OpenSolaris">OpenSolaris</a></li>
<li><a href="http://blogs.sun.com/ahl/category/Other">Other</a></li>
<li><a href="http://blogs.sun.com/ahl/category/Software">Software</a></li>
<li><a href="http://blogs.sun.com/ahl/category/ZFS">ZFS</a></li>
</ul>
</td>
<td class="subnav_right"></td></tr>
<!-- Main content -->
<tr>
<td class="content_left"></td>
<td class="content">
<div class="container aboutheader">
<div class="blogabout">
Adam Leventhal, Fishworks engineer
</div>
<div class="next-previous">
<p>
« <a href="http://blogs.sun.com/ahl/entry/opensolaris_on_lugradio">OpenSolaris on LugRa...</a> |
<a href="http://blogs.sun.com/ahl/">Main</a>
| <a href="http://blogs.sun.com/ahl/entry/dtrace_in_acm_queue">DTrace in ACM Queue</a> »
</p>
</div>
</div>
<div class="container">
<div class="entries">
<div class="day">
<div class="dayheader">
<h2>Tuesday Dec 13, 2005</h2>
</div>
<div class="entry">
<div class="entryheader">
<h3 id="dtrace_for_linux"><a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux">DTrace for Linux</a></h3>
</div>
<div class="entrybody">
<p>
With <a href="http://opensolaris.org/os/community/brandz">BrandZ</a>, it's now possible to use <a href="http://opensolaris.org/os/community/dtrace">DTrace</a> on
Linux applications. For the uninitiated, DTrace is the dynamic tracing facility
in <a href="http://opensolaris.org/">OpenSolaris</a>; it allows for <b>systemic</b> analysis of a scope and precision unequalled in the industry. With DTrace, administrators and
developers can trace low
level services like I/O and scheduling, up the system stack through
kernel functions calls, system calls, and system library calls, and into
applications written in C and C++ or any of a host of dynamic languages like
Java, Ruby, Perl or php. One of my contributions to BrandZ was to extend
DTrace support for Linux binaries executed in a branded Zone.
</p>
<p>
DTrace has several different <b>instrumentation providers</b> that
know how to instrument a particular part of the system and provide relevant
probes for that component. The <a href="http://docs.sun.com/app/docs/doc/817-6223/6mlkidllc?a=view">io provider</a> lets you trace disk
I/O, the <a href="http://docs.sun.com/app/docs/doc/817-6223/6mlkidljt?a=view">fbt (function boundary tracing) provider</a>
lets you trace
any kernel function call, etc. A typical system will start with more
than 30,000 probes but providers can create probes dynamically to trace
new kernel modules
or user-land processes. When strictly focused on a user-land
application, the
most useful providers are typically the <a href="http://docs.sun.com/app/docs/doc/817-6223/6mlkidlk8?a=view">syscall provider</a> to
examine <a href="http://en.wikipedia.org/wiki/System_call">system calls</a> and
the <a href="http://docs.sun.com/app/docs/doc/817-6223/6mlkidlls?a=view">pid provider</a> that can trace any instruction in a any process
executing on the system.
</p>
<p>
For Linux processes, the pid provider just worked (well, once <a href="http://blogs.sun.com/rab">Russ</a> built a library to understand the Linux run-time linker),
and we introduced a new
provider -- the <b>lx-syscall provider</b>
-- to trace entry and return for
emulated Linux system calls. With these providers it's possible to
understand
every facet of a Linux application's behavior and with the other DTrace
probes
it's possible to reason about an application's use of system resources.
In other words, you can take that sluggish Linux application, stick it
in a branded
Zone, dissect it using Solaris tools, and then bring it back to a
native Linux
system with the fruits of your DTrace investigation<a name="ymmv_ref" href="#ymmv">[1]</a>.
</p>
<p>
To give an example of using DTrace on Linux applications, I needed an
application to examine. I wanted a well known program that either didn't run
on Solaris or operated sufficiently differently such examining the Linux
version rather than the Solaris port made sense. I decided on
<tt>/usr/bin/top</tt> partly because of the dramatic differences between how it
operates on Linux vs. Solaris (due to the differences in /proc), but mostly
because of what I've heard my colleague, <a href="http://blogs.sun.com/bmc">Bryan</a>, refer to as the
"top problem": your system is slow, so you run <tt>top</tt>. What's the top
process? Top!
</p>
<p>
Running <tt>top</tt> in the Linux branded zone, I opened a shell in the
global (Solaris) zone to use DTrace.
I started as I do on Solaris applications: I looked at system calls. I was
interested to see which system calls were being executed most frequently
which is easily expressed in DTrace:
</p>
<p>
</p><pre>bash-3.00# dtrace -n lx-syscall:::entry'/execname == "top"/{ @[probefunc] = count(); }'
dtrace: description 'lx-syscall:::entry' matched 272 probes
^C
fstat64 322
access 323
gettimeofday 323
gtime 323
llseek 323
mmap2 323
munmap 323
select 323
getdents64 1289
lseek 1291
stat64 3545
rt_sigaction 5805
write 6459
fcntl64 6772
alarm 8708
close 11282
open 14827
read 14830
</pre>
<p>
Note the use of the <a href="http://docs.sun.com/app/docs/doc/817-6223/6mlkidlh7?a=view">aggregation</a> denoted with the '@'.
Aggregations are the mechanism by which DTrace allows users to examine
<b>patterns</b> of system behavior rather than examining each individual datum
-- each system call for example.
(In case you also noticed the strange discrepancy between the number of open
and close system calls, many of those opens are failing so it makes sense that
they would have no corresponding close. I used the lx-syscall provider to suss
this out, but I omitted that investigation in a vain appeal for brevity.)
</p>
<p>
There may well be something fishy about this output, but nothing struck
me as so compellingly fishy to explore immediately. Instead, I fired up
vi and wrote a
short D script to see which system calls were taking the most time:
</p>
<p>
<b>lx-sys.d</b>
</p><pre>#!/usr/sbin/dtrace -s
lx-syscall:::entry
/execname == "top"/
{
self->ts = vtimestamp;
}
lx-syscall:::return
/self->ts/
{
@[probefunc] = sum(vtimestamp - self->ts);
self->ts = 0;
}
</pre>
<p>
This script creates a table of system calls and the time spent in them (in
nanoseconds). The results were fairly interesting.
</p>
<p>
</p><pre>bash-3.00# ./lx-sys.d
dtrace: script './lx-sys.d' matched 550 probes
^C
llseek 4940978
gtime 5993454
gettimeofday 6603844
fstat64 14217312
select 26594875
lseek 30956093
mmap2 43463946
access 49033498
alarm 72216971
fcntl64 188281402
rt_sigaction 197646176
stat64 268188885
close 417574118
getdents64 781844851
open 1314209040
read 1862007391
write 2030173630
munmap 2195846497
</pre>
<p>
That seems like a lot of time spent in <b>munmap</b> for top. In fact, I'm
rather surprised that there's any mapping and unmapping going on at all (I
guess that should have raised an eyebrow after my initial system call count).
Unmapping memory is a pretty expensive operation that gets more expensive
on bigger systems as it requires the kernel to do some work on <b>every</b> CPU
to completely wipe out the mapping.
</p>
<p>
I then modified lx-sys.d to record the total amount of time top spent on the
CPU and the total amount of time spent in system calls to see how large a chunk
of time these seemingly expensive unmap operations were taking:
</p>
<p>
<b>lx-sys2.d</b>
</p><pre>#!/usr/sbin/dtrace -s
lx-syscall:::entry
/execname == "top"/
{
self->ts = vtimestamp;
}
lx-syscall:::return
/self->ts/
{
@[probefunc] = sum(vtimestamp - self->ts);
@["- all syscalls -"] = sum(vtimestamp - self->ts);
self->ts = 0;
}
sched:::on-cpu
/execname == "top"/
{
self->on = timestamp;
}
sched:::off-cpu
/self->on/
{
@["- total -"] = sum(timestamp - self->on);
self->on = 0;
}
</pre>
<p>
I used the <a href="http://docs.sun.com/app/docs/doc/817-6223/6mlkidll6?a=view">sched provider</a> to see when top was going on and off
of the CPU, and I added a row to record the <b>total</b> time spent in all
system call. Here were the results (keep in mind I was just hitting ^C to
end the experiment after a few seconds so it's expected that these numbers
would be different from those above; there are ways to have more accurately
timed experiments):
</p><p>
</p><pre>bash-3.00# ./lx-sys2.d
dtrace: script './lx-sys2.d' matched 550 probes
^C
llseek 939771
gtime 1088745
gettimeofday 1090020
fstat64 2494614
select 4566569
lseek 5186943
mmap2 7300830
access 8587484
alarm 11671436
fcntl64 31147636
rt_sigaction 33207341
stat64 45223200
close 69338595
getdents64 131196732
open 220188139
read 309764996
write 340413183
munmap 365830103
- all syscalls - 1589236337
- total - 3258101690
</pre>
<p>
So system calls are consuming roughly half of top's time on the CPU and
the munmap syscall is consuming roughly a quarter of that. This was enough
to convince me that there was probably room for improvement and further
investigation might bear fruit.
</p>
<p>
Next, I wanted to understand what this mapped memory was being used for so
I wrote a little script that traces all the functions called in the process
between when memory is mapped using the mmap2(2) system call and when it's
unmapped and returned to the system through the munmap(2) system call:
</p>
<p>
<b>map.d</b>
</p><pre>#!/usr/sbin/dtrace -s
#pragma D option quiet
lx-syscall::mmap2:return
/pid == $target/
{
self->ptr = arg1;
self->depth = 10;
printf("%*.s <- %s syscall\n", self->depth, "", probefunc);
}
pid$target:::entry
/self->ptr/
{
self->depth++;
printf("%*.s -> %s`%s\n", self->depth, "", probemod, probefunc);
}
pid$target:::return
/self->ptr/
{
printf("%*.s <- %s`%s\n", self->depth, "", probemod, probefunc);
self->depth--;
}
lx-syscall::munmap:entry
/arg0 == self->ptr/
{
self->depth++;
printf("%*.s -> %s syscall\n", self->depth, "", probefunc);
self->ptr = 0;
self->depth = 0;
exit(0);
}
</pre>
<p>
This script uses the $target variable which means that we need to run it with
the -p <pid> option where <pid> is the process ID of top. When mmap2 returns,
we set a thread local variable, 'ptr', which stores the address at the base
of the mapped region; for every function entry and return in the process we
call printf() if <tt>self->ptr</tt> is set; finally, we exit DTrace when
munmap is called with that same address. Here are the results:
</pid></pid></p>
<p>
</p><pre>bash-3.00# ./map.d -p `pgrep top`
<- mmap2 syscall
<- LM2`libc.so.1`syscall
<- LM2`lx_brand.so.1`lx_emulate
<- LM2`lx_brand.so.1`lx_handler
<- libc.so.6`mmap
<- libc.so.6`malloc
-> libc.so.6`memset
<- libc.so.6`memset
-> libc.so.6`cfree
<- libc.so.6`cfree
-> libc.so.6`munmap
<- LM2`lx_brand.so.1`lx_handler_table
-> LM2`lx_brand.so.1`lx_handler
-> LM2`lx_brand.so.1`lx_emulate
-> LM2`libc.so.1`syscall
-> munmap syscall
</pre>
<p>
I traced the probemod (shared object name) in addition to probefunc (function
name) so that we'd be able to differentiate proper Linux functions (in this
case all in libc.so.6) from functions in the emulation library
(LM2`lx_brand.so.1). What we can glean from this is that the mmap call is a
result of a call to malloc() and the unmap is due to a call to free(). What's
unfortunate is that we're not seeing any function calls in top itself. For some
reason, the top developer chose to strip this binary (presumably to save
precious 2k the symbol table would have used on disk) so we're stuck with no symbolic information for top's
functions and no ability to trace the individual function calls<a name="dash_ref" href="#dash">[2]</a>, but we can
still reason about this a bit more.
</p>
<p>
A little analysis in <a href="http://opensolaris.org/os/community/mdb/">mdb</a> revealed that cfree (an alias for free)
makes a tail-call to a function that calls munmap. It seems strange to me that
freeing memory would immediately results in an unmap operation (since it would
cause exactly the high overhead we're seeing here. To explore this, I wrote
another script which looks at what proportion of calls to malloc result in a
call to mmap():
</p>
<p>
<b>malloc.d</b>
</p><pre>#!/usr/sbin/dtrace -s
pid$target::malloc:entry
{
self->follow = arg0;
}
lx-syscall::mmap2:entry
/self->follow/
{
@["mapped"] = count();
self->follow = 0;
}
pid$target::malloc:return
/self->follow/
{
@["no map"] = count();
self->follow = 0;
}
</pre>
<p>
Here are the results:
</p>
<p>
</p><pre>bash-3.00# ./malloc.d -p `pgrep top`
dtrace: script './malloc.d' matched 11 probes
^C
mapped 275
no map 3024
</pre>
<p>
So a bunch of allocations result in a mmap, but not a huge number. Next I
decided to explore if there might be a correlation between the size of the
allocation and whether or not it resulted in a call to mmap using the following
script:
</p>
<p>
<b>malloc2.d</b>
</p><pre>#!/usr/sbin/dtrace -s
pid$target::malloc:entry
{
self->size = arg0;
}
lx-syscall::mmap2:entry
/self->size/
{
@["mapped"] = quantize(self->size);
self->size = 0;
}
pid$target::malloc:return
/self->size/
{
@["no map"] = quantize(self->size);
self->size = 0;
}
</pre>
<p>
Rather than just counting the frequency, I used the <a href="">quantize
aggregating action</a> to built a power-of-two histogram on the number of bytes
being allocated (<tt>self->size</tt>). The output was quite illustrative:
</p>
<p>
</p><pre>bash-3.00# ./malloc2.d -p `pgrep top`
dtrace: script './malloc2.d' matched 11 probes
^C
no map
value ------------- Distribution ------------- count
2 | 0
4 |@@@@@@@ 426
8 |@@@@@@@@@@@@@@@ 852
16 |@@@@@@@@@@@ 639
32 |@@@@ 213
64 | 0
128 | 0
256 | 0
512 |@@@@ 213
1024 | 0
mapped
value ------------- Distribution ------------- count
131072 | 0
262144 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 213
524288 | 0
</pre>
<p>
All the allocations that required a mmap were <b>huge</b> -- between 256k and
512k. Now it makes sense why the Linux libc allocator would treat these
allocations a little differently than reasonably sized allocations. And this is
clearly a smoking gun for top performance: it would do much better to
preallocate a huge buffer and grow it as needed (assuming it actually needs it
at all) than to malloc it each time. Tracking down the offending line of code
would just require a non-stripped binary and a little DTrace invocation like
this:
</p>
<p>
</p><pre># dtrace -n pid`pgrep top`::malloc:entry'/arg0 >= 262144/{@[ustack()] = count()}'
</pre>
<p>
From symptoms to root cause on a Linux application in a few DTrace scripts --
and it took me approximately 1000 times longer to cobble together some vaguely
coherent prose describing the scripts than it did for me to actually perform
the investigation. BrandZ opens up some pretty interesting new vistas for
DTrace. I look forward to seeing Linux applications being brought in for
tune-ups on BrandZ and then reaping those benefits either back on their
mother Linux or sticking around to enjoy the <a href="http://opensolaris.org/os/community/fm/">fault management</a>, <a href="http://opensolaris.org/os/community/zfs/">ZFS</a>,
scalability, and, of course, continued access to DTrace in BrandZ.
</p>
<hr>
<p>
<a name="ymmv" href="#ymmv_ref">[1]</a> Of course, results may vary since the guts of the Linux kernel differ
significantly from those of the Solaris kernel, but they're often fairly
similar. I/O or scheduling problems will be slightly different, but often
not <i>so</i> different that the conclusions lack applicability.
<br>
<a name="dash" href="#dash_ref">[2]</a> Actually, we <b>can</b> can
still trace function calls -- in fact, we can trace any instruction --
but it takes something of a heroic effort. We could disassemble parts
of top to identify calls sites and then use esoteric <tt>pid123::-:<i>address</i></tt> probe format to trace the stripped function. I said we could do it; I never said it would be pretty.
</p>
<hr>
<p>
Technorati Tags:
<a href="http://technorati.com/tag/BrandZ" rel="tag">BrandZ</a>
<a href="http://technorati.com/tag/DTrace" rel="tag">DTrace</a>
<a href="http://technorati.com/tag/Solaris" rel="tag">Solaris</a>
<a href="http://technorati.com/tag/OpenSolaris" rel="tag">OpenSolaris</a>
</p>
</div>
<div class="entryfooter">
<p>
Posted at 01:00PM Dec 13, 2005
by ahl in <span class="category">DTrace</span>
| <a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comments" class="commentsLink">Comments[9]</a>
| <a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux">Permalink</a>
</p>
</div>
</div>
</div>
<!-- Begin SiteCatalyst code version: G.5. -->
<script type="text/javascript" language="JavaScript">
<!--
if(typeof s_channel=='undefined'){var s_channel="ahl";}
//--></script>
<script type="text/javascript" language="JavaScript" src="dtrace_for_linux_files/s_code_remote.js"></script><script type="text/javascript" src="dtrace_for_linux_files/metrics_group1.js"></script>
<!-- End SiteCatalyst code version: G.5. -->
<div class="comments">
<a name="comments"></a>
<div class="comments" id="comments">
<div class="comments-head">Comments:</div>
<br>
<a name="comment-1134576732000" id="comment-1134576732000"></a>
<div class="comment odd" id="comment1">
Awsome! I've been waiting to try out dtrace for a long time now. Thanks!
<p class="comment-details">
Posted by
<b>Andreas Bruse</b>
on December 14, 2005 at 08:12 AM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134576732000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1134622184000" id="comment-1134622184000"></a>
<div class="comment even" id="comment2">
Rather than jump to profiling an application I'd
much perfer to see the results of a three
way comparison of the libMicro benchmark running
on the same machine, Solaris/Linux/Linux_translated.
At least this will give us an idea of how much
overhead the translator has on system calls
in a controlled condition. If there is a large
anomaly a dtrace session would be interesting.
---Bob
palowoda@fiver.net
<p class="comment-details">
Posted by
<b>Bob Palowoda</b>
on December 14, 2005 at 08:49 PM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134622184000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1134631985000" id="comment-1134631985000"></a>
<div class="comment odd" id="comment3">
Bob,<br>
That would be a different investigation entirely -- one I've started to
do and will continue to do, no doubt -- but the point was to
demonstrate how performance of a Linux application could be improved
not that of the emulation environment. <p class="comment-details">
Posted by
<a rel="nofollow" href="http://blogs.sun.com/ahl"><b>Adam Leventhal</b></a>
on December 14, 2005 at 11:33 PM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134631985000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1134653721000" id="comment-1134653721000"></a>
<div class="comment even" id="comment4">
Agreed :) Its nice to see dtrace coming to linux. My only problem
currently, is that the documentation for installing Brandz on a linux
box is not very clear and i am somewhat confused on what to do with the
required 2 packages on a Fedora Core server. <p class="comment-details">
Posted by
<a rel="nofollow" href="http://hunter.userfriendly.net/"><b>Michael Weiner</b></a>
on December 15, 2005 at 05:35 AM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134653721000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1134760917000" id="comment-1134760917000"></a>
<div class="comment odd" id="comment5">
Adam, This is great work, thanks.
Couple of observations: Looks like Linux is going to have a tool to do
the similar analysis natively. Here is the link to their version of the
script you tried.
http://sourceware.org/ml/systemtap/2005-q4/msg00437.html
Looking at the Linux native analysis, the problem you have experienced
doesn't seem to be there when ran natively. That brings to my mind a
question, is it really true that Solaris virtually environment and
native Linux are so close that one can do performance problem analysis
in one environment and apply in the other. Your thoughts on this and if
possibly explanation of the difference in results would be appreciated.
(I understand that the details of the machine configuration and
software versions are missing hence that might be the difference in the
results.) <p class="comment-details">
Posted by
<b>Johnd</b>
on December 16, 2005 at 11:21 AM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134760917000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1134978306000" id="comment-1134978306000"></a>
<div class="comment even" id="comment6">
Michael,<br>
If there's something in particular that needs clarification, could you
post a question to brandz-discuss@opensolaris.org? We're looking for
feedback from the community.
<br>
<br>
Johnd,<br>As you point out, differences in the hardware and
software configurations could account for the difference in the
results. It's possible that the performance problem could be exagerated
in the BrandZ environment, but this result is still valid in that
mapping and unmapping memory is surely not the most efficient way for
the allocator to behave in this situation. We don't have much data to
support this claim, but I do think analysis under BrandZ will yeild
results which are applicable on a native Linux system. The results will
tend to be more accurate for investigations that deal strictly with
application code rather than interactions with the kernel. What I
didn't see in that email thread you referenced was any sort of analysis
of the user-land flow control (map.d and malloc.d above). Being able to
follow your investigation accross the user/kernel boundary and into
applications (in a bunch of different languages) is one of the key
differences between DTrace and every other tracing framework. Without
user-land tracing even if their analysis of the system call timing had
matched mine, it's not clear that they would have been able to drive to
the root cause. <p class="comment-details">
Posted by
<a rel="nofollow" href="http://blogs.sun.com/ahl"><b>Adam Leventhal</b></a>
on December 18, 2005 at 11:45 PM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134978306000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1134982710000" id="comment-1134982710000"></a>
<div class="comment odd" id="comment7">
I havnt seen anything here that wouldnt have been possible with a
combination of strace and ltrace. And I wouldnt have had to write any
scripts to use them either. Congratulations on writing a cool utility,
but this stuff is nothing new, and certainly not only possible on
solaris. <p class="comment-details">
Posted by
<b>John</b>
on December 19, 2005 at 12:58 AM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134982710000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1134997198000" id="comment-1134997198000"></a>
<div class="comment even" id="comment8">
John,<br>
Perhaps you're right about these specific examples, but DTrace lets you
do far more than strace and ltrace. For example, you can mix data from
the kernel, system libraries and Java in a single stream as I describe <a href="http://blogs.sun.com/roller/page/ahl?entry=open_sourcing_the_javaone_keynote" rel="nofollow">here</a>.
You can't do that in Linux or on any other operating system for that
matter. And that's just one example. You may want to understand a bit
more about what DTrace is capable of before you discount it. <p class="comment-details">
Posted by
<a rel="nofollow" href="http://blogs.sun.com/"><b>Adam Leventhal</b></a>
on December 19, 2005 at 04:59 AM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1134997198000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
<a name="comment-1135140694000" id="comment-1135140694000"></a>
<div class="comment odd" id="comment9">
To avoid a stripped binary, build and install
procps like this:
<p>
<tt>make CFLAGS=-g3 install</tt>
</p><p>
I think I have a fairly good handle on problems
actually seen on Linux. The munmap() problem does
not exist; the name "Slowaris" exists for a reason.
(be thankful, because performance problems give
Sun a reason to pay your salary)
</p><p>
Every system has it's cruddy part though, and /proc is surely where Linux collects cruft.
</p><p>
<b>What top is required to do is quite ugly:</b>
</p><p>
First, it must support up to 4 million tasks.
Probably top can only do this at a reduced
refresh rate, but anyway, this means that top
can't leave the file descriptors open.
</p><p>
The per-task info is about 400 to 600 bytes.
This is not nice to the cache or TLB. At only
1000 tasks, that's half a meg of RAM to cache
and a full 128 TLB entries. Ouch. That's not
anywhere near 4 million tasks! Just playing around with the order of fields in a struct can make a huge difference for top.
</p><p>
Then there is the problem of doing lots of
64-bit division on a register-starved 32-bit
processor. This happens on both sides of the
user/kernel boundry, creating and parsing
numbers as ASCII decimal. Timestamps also
require 64-bit division.
</p><p>
A custom stack-like allocator would help.
The glibc obstacks stuff would work great.
(hey, add that to Solaris and send it off
to POSIX to be set in stone as a standard)
</p><p>
Smaller data structures would sure help,
in the short run at least, but top may be
getting support for more data columns in
the future. This also reduces code sharing
with the ps program, making maintenence hard.
</p><p class="comment-details">
Posted by
<a rel="nofollow" href="http://procps.sf.net/"><b>Albert Cahalan</b></a>
on December 20, 2005 at 08:51 PM PST
<a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux#comment-1135140694000" class="entrypermalink" title="comment permalink">#</a>
</p>
</div>
</div>
<div class="comments-form">
<div class="comments-head">Post a Comment:</div>
<a name="comment-form"></a>
<span class="status">Comments are closed for this entry.</span>
</div>
</div>
</div>
<div class="sidebar">
<!-- ARCHIVES -->
<div class="navsect">
<div class="navhead">
<h3>Archives</h3>
</div>
<div class="navbody">
<div class="hCalendarTable"><table summary="Blog Archive Calendar" class="hCalendarTable" border="0" cellspacing="0"><tbody><tr><td colspan="7" class="hCalendarMonthYearRow" align="center"><a href="http://blogs.sun.com/ahl/date/200902" title="Prev" class="hCalendarNavBar">«</a> March 2009</td></tr><tr><th class="hCalendarDayNameRow" align="center">Sun</th><th class="hCalendarDayNameRow" align="center">Mon</th><th class="hCalendarDayNameRow" align="center">Tue</th><th class="hCalendarDayNameRow" align="center">Wed</th><th class="hCalendarDayNameRow" align="center">Thu</th><th class="hCalendarDayNameRow" align="center">Fri</th><th class="hCalendarDayNameRow" align="center">Sat</th></tr><tr><td class="hCalendarDay"><div class="hCalendarDayTitle">1</div></td><td class="hCalendarDayLinked"><div class="hCalendarDayTitle"><a href="http://blogs.sun.com/ahl/date/20090302">2</a></div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">3</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">4</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">5</div></td><td class="hCalendarDayLinked"><div class="hCalendarDayTitle"><a href="http://blogs.sun.com/ahl/date/20090306">6</a></div></td><td class="hCalendarDayLinked"><div class="hCalendarDayTitle"><a href="http://blogs.sun.com/ahl/date/20090307">7</a></div></td></tr><tr><td class="hCalendarDay"><div class="hCalendarDayTitle">8</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">9</div></td><td class="hCalendarDayLinked"><div class="hCalendarDayTitle"><a href="http://blogs.sun.com/ahl/date/20090310">10</a></div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">11</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">12</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">13</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">14</div></td></tr><tr><td class="hCalendarDay"><div class="hCalendarDayTitle">15</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">16</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">17</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">18</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">19</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">20</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">21</div></td></tr><tr><td class="hCalendarDay"><div class="hCalendarDayTitle">22</div></td><td class="hCalendarDayCurrent"><div class="hCalendarDayTitle">23</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">24</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">25</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">26</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">27</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">28</div></td></tr><tr><td class="hCalendarDay"><div class="hCalendarDayTitle">29</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">30</div></td><td class="hCalendarDay"><div class="hCalendarDayTitle">31</div></td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td></tr><tr><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td><td class="hCalendarDayNotInMonth"> </td></tr><tr class="hCalendarNextPrev"><td colspan="7" align="center"><a href="http://blogs.sun.com/ahl/" class="hCalendarNavBar">Today</a></td></tr></tbody></table>
</div>
</div>
</div>
<!-- SEARCH -->
<div class="navsect">
<div class="navbody">
<form id="searchForm" method="get" action="http://search.sun.com/blogs/index.jsp" style="margin: 0pt; padding: 0pt;">
<input name="weblog" value="ahl" type="hidden">
<p>
<input name="qt" size="10" maxlength="255" type="text">
<input value="Search" type="submit">
</p>
</form>
</div>
</div>
<!-- BOOKMARKS -->
<div class="navsect">
<div class="navhead">
<h3>The DTrace Three</h3>
</div>
<div class="navbody">
<ul class="rFolder">
<li class="rFolderItem">
<a href="http://blogs.sun.com/ahl" title="Adam Leventhal" class="rBookmark0">Adam Leventhal</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/bmc" title="Bryan Cantrill" class="rBookmark0">Bryan Cantrill</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/mws" title="Mike Shapiro" class="rBookmark0">Mike Shapiro</a>
</li>
</ul>
</div>
</div>
<div class="navsect">
<div class="navhead">
<h3>DTrace Links</h3>
</div>
<div class="navbody">
<ul class="rFolder">
<li class="rFolderItem">
<a href="http://www.infoworld.com/article/05/08/01/31FEinnovator10_1.html" title="InfoWorld Innovators 2005" class="rBookmark0">InfoWorld Innovators 2005</a>
</li>
<li class="rFolderItem">
<a href="http://www.opensolaris.org/os/community/dtrace/" title="DTrace Home" class="rBookmark0">DTrace Home</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/ahl/dtracesched" title="DTrace Schedule" class="rBookmark0">DTrace Schedule</a>
</li>
<li class="rFolderItem">
<a href="http://www.opensolaris.org/jive/forum.jspa?forumID=7" title="Discussion Forum" class="rBookmark0">Discussion Forum</a>
</li>
<li class="rFolderItem">
<a href="http://see.sun.com/Apps/DCS/mcp?r=703SsI4297t0006304e4m32f403gpD04299i0" title="Expert Exchange" class="rBookmark0">Expert Exchange</a>
</li>
<li class="rFolderItem">
<a href="http://www.linuxinsider.com/story/The-Importance-of-Solaris-10-37993.html" title="LinuxInsider story" class="rBookmark0">LinuxInsider story</a>
</li>
<li class="rFolderItem">
<a href="http://news.com.com/2100-7344_3-5374412.html" title="NY Times Solaris story" class="rBookmark0">NY Times Solaris story</a>
</li>
<li class="rFolderItem">
<a href="http://www.theregister.co.uk/2004/07/08/dtrace_user_take/" title="Register story" class="rBookmark0">Register story</a>
</li>
<li class="rFolderItem">
<a href="http://www.samag.com/documents/s=9171/sam0406h/0406h.htm" title="SysAdmin story" class="rBookmark0">SysAdmin story</a>
</li>
<li class="rFolderItem">
<a href="http://www.eweek.com/article2/0,1759,1626474,00.asp?kc=EWRSS03129TX1K0000612" title="eWEEK story" class="rBookmark0">eWEEK story</a>
</li>
<li class="rFolderItem">
<a href="http://www.sun.com/2004-0518/feature/index.html" title="sun.com story" class="rBookmark0">sun.com story</a>
</li>
</ul>
</div>
</div>
<div class="navsect">
<div class="navhead">
<h3>Fishworks Engineering</h3>
</div>
<div class="navbody">
<ul class="rFolder">
<li class="rFolderItem">
<a href="http://blogs.sun.com/wdp" title="" class="rBookmark0">Bill Pijewski</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/brendan" title="" class="rBookmark0">Brendan Gregg</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/bmc" title="" class="rBookmark0">Bryan Cantrill</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/cindi" title="" class="rBookmark0">Cindi McGuire</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/dap" title="" class="rBookmark0">Dave Pacheco</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/eschrock" title="" class="rBookmark0">Eric Schrock</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/greg" title="" class="rBookmark0">Greg Price</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/wesolows" title="" class="rBookmark0">Keith Wesolowski</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/mws" title="" class="rBookmark0">Mike Shapiro</a>
</li>
<li class="rFolderItem">
<a href="http://blogs.sun.com/tmp" title="" class="rBookmark0">Todd Patrick</a>
</li>
</ul>
</div>
</div>
<!-- LINKS -->
<div class="navsect">
<div class="navhead">
<h3>Recent Posts</h3>
</div>
<div class="navbody">
<ul class="rEntriesList">
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/ssd_announcement">SSDs for HSPs</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/hsp_wish_2009">Presentation: Hybrid Storage Pools and SSDs</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/fishworks_vm">Fishworks VM: the 7000 series on your laptop</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/more_anarchy">More from the storage anarchist</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dancing_with_the_anarchist">Dancing with the Anarchist</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/hsp_talk_at_the_opensolaris">HSP talk at the OpenSolaris Storage Summit</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/flash_workshop_at_asplos">Flash workshop at ASPLOS</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/shadow_of_hsp">Casting the shadow of the Hybrid Storage Pool</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/sun_storage_7410_space_calculator">Sun Storage 7410 space calculator</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/fishworks_launch">Hybrid Storage Pools in the 7410</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/hsp_goes_glossy">Hybrid Storage Pool goes glossy</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/apple_updates_dtrace_again">Apple updates DTrace... again</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/glimpse_into_netapp_s_flash">A glimpse into Netapp's flash future</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/hybrid_storage_pools_the_l2arc">Hybrid Storage Pools: The L2ARC</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/hybrid_storage_pools_in_cacm">Hybrid Storage Pools in CACM</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/flash_hybrid_pools_and_future">Flash, Hybrid Pools, and Future Storage</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/apple_updates_dtrace">Apple updates DTrace</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_conf_post_post_mortem">dtrace.conf post-post-mortem</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_at_javaone_no_more">DTrace and JavaOne: The End of the Beginning</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/expand_o_matic_raid_z">Expand-O-Matic RAID-Z</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/pid2proc_for_dtrace">pid2proc for DTrace</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/mac_os_x_and_the">Mac OS X and the missing probes</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_firefox_leopard">DTrace/Firefox/Leopard</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/what_if_machine_dtrace_port">What-If Machine: DTrace Port</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_knockoffs">DTrace Knockoffs</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_for_ruby_at_oscon">DTrace for Ruby at OSCON 2007</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_scobleized">DTrace "Scobleized"</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/iscsi_dtrace_provider_and_other">iSCSI DTrace provider and more to come</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_javaone_2007">DTrace @ JavaOne 2007</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/java_dtrace_article">Java/DTrace article</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/linux_defection">Linux Defection</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/gzip_for_zfs_update">gzip for ZFS update</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/a_little_zfs_hack">a small ZFS hack</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/on_testing">It's tested or it's broken</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace%3A_a_history">DTrace: a history</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_is_a_web_developer">DTrace is a web developer's best friend</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_user_number_one">DTrace user number one</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_on_mac_os_x">Apple's DTrace team</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/double_parity_raid_z">Double-Parity RAID-Z</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_on_geek_muse">DTrace on Geek Muse</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/javaone_dtrace_session_2006">JavaOne DTrace Session 2006</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_caption_contest">DTrace caption contest</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_at_javaone_2006">DTrace at JavaOne 2006</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/user_land_tracing_gets_better">User-land tracing gets better and better</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_at_scale_4x">DTrace at SCALE 4x</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_in_acm_queue">DTrace in ACM Queue</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/dtrace_for_linux">DTrace for Linux</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/opensolaris_on_lugradio">OpenSolaris on LugRadio</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/fall_of_code">Fall of Code?</a></li>
<li class="recentposts"><a href="http://blogs.sun.com/ahl/entry/opensolaris_and_svk">OpenSolaris and svk</a></li>
</ul>
</div>
</div>
<div class="navsect">
<div class="navhead">
<h3>Other Blog Links</h3>
</div>
<div class="navbody">
<ul class="rNavigationBar">
<li class="rNavItem">
<a href="http://blogs.sun.com/"><span>blogs.sun.com</span></a>
</li>
<li class="rNavItem">
<a href="http://blogs.sun.com/ahl/"><span>Weblog</span></a>
</li>
<li class="rNavItem"><a href="http://blogs.sun.com/ahl/page/dtracesched"><span>DTrace Schedule</span></a></li>
<li class="rNavItem">
<a href="http://blogs.sun.com/roller-ui/login-redirect.rol"><span>Login</span></a>
</li>
</ul>
</div>
</div>
<!-- AFFINITY -->
<div class="affinity"></div>
<!-- REFERRERS -->
<div class="navsect">
<div class="navbody">
<div class="rFolderitem">
<p>Today's Page Hits: 1571</p>
</div>
</div>
</div>
<script type="text/javascript">
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script><script src="dtrace_for_linux_files/ga.js" type="text/javascript"></script>
<script type="text/javascript">
var pageTracker = _gat._getTracker("UA-1092717-1");
pageTracker._trackPageview();
</script>
</div>
</div>
</td>
<td class="content_right"></td></tr>
<!-- Footer - spans all columns -->
<tr><td colspan="3" class="footer"></td></tr>
</tbody>
</table>
</div>
</div>
</body></html>
|