1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
6 Package trace implements tracing of requests and long-lived objects.
7 It exports HTTP interfaces on /debug/requests and /debug/events.
9 A trace.Trace provides tracing for short-lived objects, usually requests.
10 A request handler might be implemented like this:
12 func fooHandler(w http.ResponseWriter, req *http.Request) {
13 tr := trace.New("mypkg.Foo", req.URL.Path)
16 tr.LazyPrintf("some event %q happened", str)
18 if err := somethingImportant(); err != nil {
19 tr.LazyPrintf("somethingImportant failed: %v", err)
24 The /debug/requests HTTP endpoint organizes the traces by family,
25 errors, and duration. It also provides histogram of request duration
28 A trace.EventLog provides tracing for long-lived objects, such as RPC
31 // A Fetcher fetches URL paths for a single domain.
37 func NewFetcher(domain string) *Fetcher {
40 trace.NewEventLog("mypkg.Fetcher", domain),
44 func (f *Fetcher) Fetch(path string) (string, error) {
45 resp, err := http.Get("http://" + f.domain + "/" + path)
47 f.events.Errorf("Get(%q) = %v", path, err)
50 f.events.Printf("Get(%q) = %s", path, resp.Status)
54 func (f *Fetcher) Close() error {
59 The /debug/events HTTP endpoint organizes the event logs by family and
60 by time since the last error. The expanded view displays recent log
61 entries and the log's call stack.
63 package trace // import "golang.org/x/net/trace"
82 "golang.org/x/net/internal/timeseries"
85 // DebugUseAfterFinish controls whether to debug uses of Trace values after finishing.
86 // FOR DEBUGGING ONLY. This will slow down the program.
87 var DebugUseAfterFinish = false
89 // HTTP ServeMux paths.
91 debugRequestsPath = "/debug/requests"
92 debugEventsPath = "/debug/events"
95 // AuthRequest determines whether a specific request is permitted to load the
96 // /debug/requests or /debug/events pages.
98 // It returns two bools; the first indicates whether the page may be viewed at all,
99 // and the second indicates whether sensitive events will be shown.
101 // AuthRequest may be replaced by a program to customize its authorization requirements.
103 // The default AuthRequest function returns (true, true) if and only if the request
104 // comes from localhost/127.0.0.1/[::1].
105 var AuthRequest = func(req *http.Request) (any, sensitive bool) {
106 // RemoteAddr is commonly in the form "IP" or "IP:port".
107 // If it is in the form "IP:port", split off the port.
108 host, _, err := net.SplitHostPort(req.RemoteAddr)
110 host = req.RemoteAddr
113 case "localhost", "127.0.0.1", "::1":
121 _, pat := http.DefaultServeMux.Handler(&http.Request{URL: &url.URL{Path: debugRequestsPath}})
122 if pat == debugRequestsPath {
123 panic("/debug/requests is already registered. You may have two independent copies of " +
124 "golang.org/x/net/trace in your binary, trying to maintain separate state. This may " +
125 "involve a vendored copy of golang.org/x/net/trace.")
128 // TODO(jbd): Serve Traces from /debug/traces in the future?
129 // There is no requirement for a request to be present to have traces.
130 http.HandleFunc(debugRequestsPath, Traces)
131 http.HandleFunc(debugEventsPath, Events)
134 // NewContext returns a copy of the parent context
135 // and associates it with a Trace.
136 func NewContext(ctx context.Context, tr Trace) context.Context {
137 return context.WithValue(ctx, contextKey, tr)
140 // FromContext returns the Trace bound to the context, if any.
141 func FromContext(ctx context.Context) (tr Trace, ok bool) {
142 tr, ok = ctx.Value(contextKey).(Trace)
146 // Traces responds with traces from the program.
147 // The package initialization registers it in http.DefaultServeMux
148 // at /debug/requests.
150 // It performs authorization by running AuthRequest.
151 func Traces(w http.ResponseWriter, req *http.Request) {
152 any, sensitive := AuthRequest(req)
154 http.Error(w, "not allowed", http.StatusUnauthorized)
157 w.Header().Set("Content-Type", "text/html; charset=utf-8")
158 Render(w, req, sensitive)
161 // Events responds with a page of events collected by EventLogs.
162 // The package initialization registers it in http.DefaultServeMux
165 // It performs authorization by running AuthRequest.
166 func Events(w http.ResponseWriter, req *http.Request) {
167 any, sensitive := AuthRequest(req)
169 http.Error(w, "not allowed", http.StatusUnauthorized)
172 w.Header().Set("Content-Type", "text/html; charset=utf-8")
173 RenderEvents(w, req, sensitive)
176 // Render renders the HTML page typically served at /debug/requests.
177 // It does not do any auth checking. The request may be nil.
179 // Most users will use the Traces handler.
180 func Render(w io.Writer, req *http.Request, sensitive bool) {
183 ActiveTraceCount map[string]int
184 CompletedTraces map[string]*family
186 // Set when a bucket has been selected.
193 ShowSensitive bool // whether to show sensitive events
195 Histogram template.HTML
196 HistogramWindow string // e.g. "last minute", "last hour", "all time"
198 // If non-zero, the set of traces is a partial set,
199 // and this is the total number.
202 CompletedTraces: completedTraces,
205 data.ShowSensitive = sensitive
207 // Allow show_sensitive=0 to force hiding of sensitive data for testing.
208 // This only goes one way; you can't use show_sensitive=1 to see things.
209 if req.FormValue("show_sensitive") == "0" {
210 data.ShowSensitive = false
213 if exp, err := strconv.ParseBool(req.FormValue("exp")); err == nil {
216 if exp, err := strconv.ParseBool(req.FormValue("rtraced")); err == nil {
222 data.Families = make([]string, 0, len(completedTraces))
223 for fam := range completedTraces {
224 data.Families = append(data.Families, fam)
226 completedMu.RUnlock()
227 sort.Strings(data.Families)
229 // We are careful here to minimize the time spent locking activeMu,
230 // since that lock is required every time an RPC starts and finishes.
231 data.ActiveTraceCount = make(map[string]int, len(data.Families))
233 for fam, s := range activeTraces {
234 data.ActiveTraceCount[fam] = s.Len()
239 data.Family, data.Bucket, ok = parseArgs(req)
243 case data.Bucket == -1:
245 n := data.ActiveTraceCount[data.Family]
246 data.Traces = getActiveTraces(data.Family)
247 if len(data.Traces) < n {
250 case data.Bucket < bucketsPerFamily:
251 if b := lookupBucket(data.Family, data.Bucket); b != nil {
252 data.Traces = b.Copy(data.Traced)
255 if f := getFamily(data.Family, false); f != nil {
256 var obs timeseries.Observable
258 switch o := data.Bucket - bucketsPerFamily; o {
260 obs = f.Latency.Minute()
261 data.HistogramWindow = "last minute"
263 obs = f.Latency.Hour()
264 data.HistogramWindow = "last hour"
266 obs = f.Latency.Total()
267 data.HistogramWindow = "all time"
269 f.LatencyMu.RUnlock()
271 data.Histogram = obs.(*histogram).html()
276 if data.Traces != nil {
277 defer data.Traces.Free()
278 sort.Sort(data.Traces)
282 defer completedMu.RUnlock()
283 if err := pageTmpl().ExecuteTemplate(w, "Page", data); err != nil {
284 log.Printf("net/trace: Failed executing template: %v", err)
288 func parseArgs(req *http.Request) (fam string, b int, ok bool) {
292 fam, bStr := req.FormValue("fam"), req.FormValue("b")
293 if fam == "" || bStr == "" {
296 b, err := strconv.Atoi(bStr)
297 if err != nil || b < -1 {
304 func lookupBucket(fam string, b int) *traceBucket {
305 f := getFamily(fam, false)
306 if f == nil || b < 0 || b >= len(f.Buckets) {
312 type contextKeyT string
314 var contextKey = contextKeyT("golang.org/x/net/trace.Trace")
316 // Trace represents an active request.
317 type Trace interface {
318 // LazyLog adds x to the event log. It will be evaluated each time the
319 // /debug/requests page is rendered. Any memory referenced by x will be
320 // pinned until the trace is finished and later discarded.
321 LazyLog(x fmt.Stringer, sensitive bool)
323 // LazyPrintf evaluates its arguments with fmt.Sprintf each time the
324 // /debug/requests page is rendered. Any memory referenced by a will be
325 // pinned until the trace is finished and later discarded.
326 LazyPrintf(format string, a ...interface{})
328 // SetError declares that this trace resulted in an error.
331 // SetRecycler sets a recycler for the trace.
332 // f will be called for each event passed to LazyLog at a time when
333 // it is no longer required, whether while the trace is still active
334 // and the event is discarded, or when a completed trace is discarded.
335 SetRecycler(f func(interface{}))
337 // SetTraceInfo sets the trace info for the trace.
338 // This is currently unused.
339 SetTraceInfo(traceID, spanID uint64)
341 // SetMaxEvents sets the maximum number of events that will be stored
342 // in the trace. This has no effect if any events have already been
343 // added to the trace.
346 // Finish declares that this trace is complete.
347 // The trace should not be used after calling this method.
351 type lazySprintf struct {
356 func (l *lazySprintf) String() string {
357 return fmt.Sprintf(l.format, l.a...)
360 // New returns a new Trace with the specified family and title.
361 func New(family, title string) Trace {
364 tr.Family, tr.Title = family, title
365 tr.Start = time.Now()
366 tr.maxEvents = maxEventsPerTrace
367 tr.events = tr.eventsBuf[:0]
370 s := activeTraces[tr.Family]
374 s = activeTraces[tr.Family] // check again
377 activeTraces[tr.Family] = s
383 // Trigger allocation of the completed trace structure for this family.
384 // This will cause the family to be present in the request page during
385 // the first trace of this family. We don't care about the return value,
386 // nor is there any need for this to run inline, so we execute it in its
387 // own goroutine, but only if the family isn't allocated yet.
389 if _, ok := completedTraces[tr.Family]; !ok {
390 go allocFamily(tr.Family)
392 completedMu.RUnlock()
397 func (tr *trace) Finish() {
398 elapsed := time.Now().Sub(tr.Start)
403 if DebugUseAfterFinish {
404 buf := make([]byte, 4<<10) // 4 KB should be enough
405 n := runtime.Stack(buf, false)
406 tr.finishStack = buf[:n]
410 m := activeTraces[tr.Family]
414 f := getFamily(tr.Family, true)
415 tr.mu.RLock() // protects tr fields in Cond.match calls
416 for _, b := range f.Buckets {
417 if b.Cond.match(tr) {
423 // Add a sample of elapsed time as microseconds to the family's timeseries
425 h.addMeasurement(elapsed.Nanoseconds() / 1e3)
430 tr.unref() // matches ref in New
436 maxActiveTraces = 20 // Maximum number of active traces to show.
437 maxEventsPerTrace = 10
438 numHistogramBuckets = 38
442 // The active traces.
443 activeMu sync.RWMutex
444 activeTraces = make(map[string]*traceSet) // family -> traces
446 // Families of completed traces.
447 completedMu sync.RWMutex
448 completedTraces = make(map[string]*family) // family -> traces
451 type traceSet struct {
455 // We could avoid the entire map scan in FirstN by having a slice of all the traces
456 // ordered by start time, and an index into that from the trace struct, with a periodic
457 // repack of the slice after enough traces finish; we could also use a skip list or similar.
458 // However, that would shift some of the expense from /debug/requests time to RPC time,
459 // which is probably the wrong trade-off.
462 func (ts *traceSet) Len() int {
464 defer ts.mu.RUnlock()
468 func (ts *traceSet) Add(tr *trace) {
471 ts.m = make(map[*trace]bool)
477 func (ts *traceSet) Remove(tr *trace) {
483 // FirstN returns the first n traces ordered by time.
484 func (ts *traceSet) FirstN(n int) traceList {
486 defer ts.mu.RUnlock()
491 trl := make(traceList, 0, n)
493 // Fast path for when no selectivity is needed.
495 for tr := range ts.m {
497 trl = append(trl, tr)
503 // Pick the oldest n traces.
504 // This is inefficient. See the comment in the traceSet struct.
505 for tr := range ts.m {
506 // Put the first n traces into trl in the order they occur.
507 // When we have n, sort trl, and thereafter maintain its order.
510 trl = append(trl, tr)
512 // This is guaranteed to happen exactly once during this loop.
517 if tr.Start.After(trl[n-1].Start) {
521 // Find where to insert this one.
523 i := sort.Search(n, func(i int) bool { return trl[i].Start.After(tr.Start) })
525 copy(trl[i+1:], trl[i:])
532 func getActiveTraces(fam string) traceList {
534 s := activeTraces[fam]
539 return s.FirstN(maxActiveTraces)
542 func getFamily(fam string, allocNew bool) *family {
544 f := completedTraces[fam]
545 completedMu.RUnlock()
546 if f == nil && allocNew {
552 func allocFamily(fam string) *family {
554 defer completedMu.Unlock()
555 f := completedTraces[fam]
558 completedTraces[fam] = f
563 // family represents a set of trace buckets and associated latency information.
565 // traces may occur in multiple buckets.
566 Buckets [bucketsPerFamily]*traceBucket
568 // latency time series
569 LatencyMu sync.RWMutex
570 Latency *timeseries.MinuteHourSeries
573 func newFamily() *family {
575 Buckets: [bucketsPerFamily]*traceBucket{
577 {Cond: minCond(50 * time.Millisecond)},
578 {Cond: minCond(100 * time.Millisecond)},
579 {Cond: minCond(200 * time.Millisecond)},
580 {Cond: minCond(500 * time.Millisecond)},
581 {Cond: minCond(1 * time.Second)},
582 {Cond: minCond(10 * time.Second)},
583 {Cond: minCond(100 * time.Second)},
586 Latency: timeseries.NewMinuteHourSeries(func() timeseries.Observable { return new(histogram) }),
590 // traceBucket represents a size-capped bucket of historic traces,
591 // along with a condition for a trace to belong to the bucket.
592 type traceBucket struct {
595 // Ring buffer implementation of a fixed-size FIFO queue.
597 buf [tracesPerBucket]*trace
598 start int // < tracesPerBucket
599 length int // <= tracesPerBucket
602 func (b *traceBucket) Add(tr *trace) {
606 i := b.start + b.length
607 if i >= tracesPerBucket {
610 if b.length == tracesPerBucket {
611 // "Remove" an element from the bucket.
614 if b.start == tracesPerBucket {
619 if b.length < tracesPerBucket {
625 // Copy returns a copy of the traces in the bucket.
626 // If tracedOnly is true, only the traces with trace information will be returned.
627 // The logs will be ref'd before returning; the caller should call
628 // the Free method when it is done with them.
629 // TODO(dsymonds): keep track of traced requests in separate buckets.
630 func (b *traceBucket) Copy(tracedOnly bool) traceList {
634 trl := make(traceList, 0, b.length)
635 for i, x := 0, b.start; i < b.length; i++ {
637 if !tracedOnly || tr.spanID != 0 {
639 trl = append(trl, tr)
649 func (b *traceBucket) Empty() bool {
655 // cond represents a condition on a trace.
656 type cond interface {
661 type minCond time.Duration
663 func (m minCond) match(t *trace) bool { return t.Elapsed >= time.Duration(m) }
664 func (m minCond) String() string { return fmt.Sprintf("≥%gs", time.Duration(m).Seconds()) }
666 type errorCond struct{}
668 func (e errorCond) match(t *trace) bool { return t.IsError }
669 func (e errorCond) String() string { return "errors" }
671 type traceList []*trace
673 // Free calls unref on each element of the list.
674 func (trl traceList) Free() {
675 for _, t := range trl {
680 // traceList may be sorted in reverse chronological order.
681 func (trl traceList) Len() int { return len(trl) }
682 func (trl traceList) Less(i, j int) bool { return trl[i].Start.After(trl[j].Start) }
683 func (trl traceList) Swap(i, j int) { trl[i], trl[j] = trl[j], trl[i] }
685 // An event is a timestamped log entry in a trace.
688 Elapsed time.Duration // since previous event in trace
689 NewDay bool // whether this event is on a different day to the previous event
690 Recyclable bool // whether this event was passed via LazyLog
691 Sensitive bool // whether this event contains sensitive information
692 What interface{} // string or fmt.Stringer
695 // WhenString returns a string representation of the elapsed time of the event.
696 // It will include the date if midnight was crossed.
697 func (e event) WhenString() string {
699 return e.When.Format("2006/01/02 15:04:05.000000")
701 return e.When.Format("15:04:05.000000")
704 // discarded represents a number of discarded events.
705 // It is stored as *discarded to make it easier to update in-place.
708 func (d *discarded) String() string {
709 return fmt.Sprintf("(%d events discarded)", int(*d))
712 // trace represents an active or complete request,
713 // either sent or received by this program.
715 // Family is the top-level grouping of traces to which this belongs.
718 // Title is the title of this trace.
721 // Start time of the this trace.
725 events []event // Append-only sequence of events (modulo discards).
727 recycler func(interface{})
728 IsError bool // Whether this trace resulted in an error.
729 Elapsed time.Duration // Elapsed time for this trace, zero while active.
730 traceID uint64 // Trace information if non-zero.
733 refs int32 // how many buckets this is in
734 disc discarded // scratch space to avoid allocation
736 finishStack []byte // where finish was called, if DebugUseAfterFinish is set
738 eventsBuf [4]event // preallocated buffer in case we only log a few events
741 func (tr *trace) reset() {
742 // Clear all but the mutex. Mutexes may not be copied, even when unlocked.
745 tr.Start = time.Time{}
760 for i := range tr.eventsBuf {
761 tr.eventsBuf[i] = event{}
765 // delta returns the elapsed time since the last event or the trace start,
766 // and whether it spans midnight.
768 func (tr *trace) delta(t time.Time) (time.Duration, bool) {
769 if len(tr.events) == 0 {
770 return t.Sub(tr.Start), false
772 prev := tr.events[len(tr.events)-1].When
773 return t.Sub(prev), prev.Day() != t.Day()
776 func (tr *trace) addEvent(x interface{}, recyclable, sensitive bool) {
777 if DebugUseAfterFinish && tr.finishStack != nil {
778 buf := make([]byte, 4<<10) // 4 KB should be enough
779 n := runtime.Stack(buf, false)
780 log.Printf("net/trace: trace used after finish:\nFinished at:\n%s\nUsed at:\n%s", tr.finishStack, buf[:n])
786 If you are here because your program panicked in this code,
787 it is almost definitely the fault of code using this package,
788 and very unlikely to be the fault of this code.
790 The most likely scenario is that some code elsewhere is using
791 a trace.Trace after its Finish method is called.
792 You can temporarily set the DebugUseAfterFinish var
793 to help discover where that is; do not leave that var set,
794 since it makes this package much less efficient.
797 e := event{When: time.Now(), What: x, Recyclable: recyclable, Sensitive: sensitive}
799 e.Elapsed, e.NewDay = tr.delta(e.When)
800 if len(tr.events) < tr.maxEvents {
801 tr.events = append(tr.events, e)
803 // Discard the middle events.
804 di := int((tr.maxEvents - 1) / 2)
805 if d, ok := tr.events[di].What.(*discarded); ok {
808 // disc starts at two to count for the event it is replacing,
809 // plus the next one that we are about to drop.
811 if tr.recycler != nil && tr.events[di].Recyclable {
812 go tr.recycler(tr.events[di].What)
814 tr.events[di].What = &tr.disc
816 // The timestamp of the discarded meta-event should be
817 // the time of the last event it is representing.
818 tr.events[di].When = tr.events[di+1].When
820 if tr.recycler != nil && tr.events[di+1].Recyclable {
821 go tr.recycler(tr.events[di+1].What)
823 copy(tr.events[di+1:], tr.events[di+2:])
824 tr.events[tr.maxEvents-1] = e
829 func (tr *trace) LazyLog(x fmt.Stringer, sensitive bool) {
830 tr.addEvent(x, true, sensitive)
833 func (tr *trace) LazyPrintf(format string, a ...interface{}) {
834 tr.addEvent(&lazySprintf{format, a}, false, false)
837 func (tr *trace) SetError() {
843 func (tr *trace) SetRecycler(f func(interface{})) {
849 func (tr *trace) SetTraceInfo(traceID, spanID uint64) {
851 tr.traceID, tr.spanID = traceID, spanID
855 func (tr *trace) SetMaxEvents(m int) {
857 // Always keep at least three events: first, discarded count, last.
858 if len(tr.events) == 0 && m > 3 {
864 func (tr *trace) ref() {
865 atomic.AddInt32(&tr.refs, 1)
868 func (tr *trace) unref() {
869 if atomic.AddInt32(&tr.refs, -1) == 0 {
871 if tr.recycler != nil {
872 // freeTrace clears tr, so we hold tr.recycler and tr.events here.
873 go func(f func(interface{}), es []event) {
874 for _, e := range es {
879 }(tr.recycler, tr.events)
887 func (tr *trace) When() string {
888 return tr.Start.Format("2006/01/02 15:04:05.000000")
891 func (tr *trace) ElapsedTime() string {
898 t = time.Since(tr.Start)
900 return fmt.Sprintf("%.6f", t.Seconds())
903 func (tr *trace) Events() []event {
905 defer tr.mu.RUnlock()
909 var traceFreeList = make(chan *trace, 1000) // TODO(dsymonds): Use sync.Pool?
911 // newTrace returns a trace ready to use.
912 func newTrace() *trace {
914 case tr := <-traceFreeList:
921 // freeTrace adds tr to traceFreeList if there's room.
922 // This is non-blocking.
923 func freeTrace(tr *trace) {
924 if DebugUseAfterFinish {
925 return // never reuse
929 case traceFreeList <- tr:
934 func elapsed(d time.Duration) string {
935 b := []byte(fmt.Sprintf("%.6f", d.Seconds()))
937 // For subsecond durations, blank all zeros before decimal point,
938 // and all zeros between the decimal point and the first non-zero digit.
940 dot := bytes.IndexByte(b, '.')
941 for i := 0; i < dot; i++ {
944 for i := dot + 1; i < len(b); i++ {
956 var pageTmplCache *template.Template
957 var pageTmplOnce sync.Once
959 func pageTmpl() *template.Template {
960 pageTmplOnce.Do(func() {
961 pageTmplCache = template.Must(template.New("Page").Funcs(template.FuncMap{
963 "add": func(a, b int) int { return a + b },
970 {{template "Prolog" .}}
971 {{template "StatusTable" .}}
972 {{template "Epilog" .}}
977 <title>/debug/requests</title>
978 <style type="text/css">
980 font-family: sans-serif;
982 table#tr-status td.family {
985 table#tr-status td.active {
988 table#tr-status td.latency-first {
991 table#tr-status td.empty {
997 table#reqs tr.first {
998 {{if $.Expanded}}font-weight: bold;{{end}}
1001 font-family: monospace;
1003 table#reqs td.when {
1005 white-space: nowrap;
1007 table#reqs td.elapsed {
1021 <h1>/debug/requests</h1>
1022 {{end}} {{/* end of Prolog */}}
1024 {{define "StatusTable"}}
1025 <table id="tr-status">
1026 {{range $fam := .Families}}
1028 <td class="family">{{$fam}}</td>
1030 {{$n := index $.ActiveTraceCount $fam}}
1031 <td class="active {{if not $n}}empty{{end}}">
1032 {{if $n}}<a href="?fam={{$fam}}&b=-1{{if $.Expanded}}&exp=1{{end}}">{{end}}
1034 {{if $n}}</a>{{end}}
1037 {{$f := index $.CompletedTraces $fam}}
1038 {{range $i, $b := $f.Buckets}}
1039 {{$empty := $b.Empty}}
1040 <td {{if $empty}}class="empty"{{end}}>
1041 {{if not $empty}}<a href="?fam={{$fam}}&b={{$i}}{{if $.Expanded}}&exp=1{{end}}">{{end}}
1043 {{if not $empty}}</a>{{end}}
1047 {{$nb := len $f.Buckets}}
1048 <td class="latency-first">
1049 <a href="?fam={{$fam}}&b={{$nb}}">[minute]</a>
1052 <a href="?fam={{$fam}}&b={{add $nb 1}}">[hour]</a>
1055 <a href="?fam={{$fam}}&b={{add $nb 2}}">[total]</a>
1061 {{end}} {{/* end of StatusTable */}}
1066 <h3>Family: {{$.Family}}</h3>
1068 {{if or $.Expanded $.Traced}}
1069 <a href="?fam={{$.Family}}&b={{$.Bucket}}">[Normal/Summary]</a>
1074 {{if or (not $.Expanded) $.Traced}}
1075 <a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1">[Normal/Expanded]</a>
1081 {{if or $.Expanded (not $.Traced)}}
1082 <a href="?fam={{$.Family}}&b={{$.Bucket}}&rtraced=1">[Traced/Summary]</a>
1086 {{if or (not $.Expanded) (not $.Traced)}}
1087 <a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1&rtraced=1">[Traced/Expanded]</a>
1094 <p><em>Showing <b>{{len $.Traces}}</b> of <b>{{$.Total}}</b> traces.</em></p>
1099 {{if $.Active}}Active{{else}}Completed{{end}} Requests
1101 <tr><th>When</th><th>Elapsed (s)</th></tr>
1102 {{range $tr := $.Traces}}
1104 <td class="when">{{$tr.When}}</td>
1105 <td class="elapsed">{{$tr.ElapsedTime}}</td>
1106 <td>{{$tr.Title}}</td>
1107 {{/* TODO: include traceID/spanID */}}
1110 {{range $tr.Events}}
1112 <td class="when">{{.WhenString}}</td>
1113 <td class="elapsed">{{elapsed .Elapsed}}</td>
1114 <td>{{if or $.ShowSensitive (not .Sensitive)}}... {{.What}}{{else}}<em>[redacted]</em>{{end}}</td>
1120 {{end}} {{/* if $.Traces */}}
1123 <h4>Latency (µs) of {{$.Family}} over {{$.HistogramWindow}}</h4>
1125 {{end}} {{/* if $.Histogram */}}
1129 {{end}} {{/* end of Epilog */}}