1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
6 Package trace implements tracing of requests and long-lived objects.
7 It exports HTTP interfaces on /debug/requests and /debug/events.
9 A trace.Trace provides tracing for short-lived objects, usually requests.
10 A request handler might be implemented like this:
12 func fooHandler(w http.ResponseWriter, req *http.Request) {
13 tr := trace.New("mypkg.Foo", req.URL.Path)
16 tr.LazyPrintf("some event %q happened", str)
18 if err := somethingImportant(); err != nil {
19 tr.LazyPrintf("somethingImportant failed: %v", err)
24 The /debug/requests HTTP endpoint organizes the traces by family,
25 errors, and duration. It also provides histogram of request duration
28 A trace.EventLog provides tracing for long-lived objects, such as RPC
31 // A Fetcher fetches URL paths for a single domain.
37 func NewFetcher(domain string) *Fetcher {
40 trace.NewEventLog("mypkg.Fetcher", domain),
44 func (f *Fetcher) Fetch(path string) (string, error) {
45 resp, err := http.Get("http://" + f.domain + "/" + path)
47 f.events.Errorf("Get(%q) = %v", path, err)
50 f.events.Printf("Get(%q) = %s", path, resp.Status)
54 func (f *Fetcher) Close() error {
59 The /debug/events HTTP endpoint organizes the event logs by family and
60 by time since the last error. The expanded view displays recent log
61 entries and the log's call stack.
63 package trace // import "golang.org/x/net/trace"
80 "golang.org/x/net/internal/timeseries"
83 // DebugUseAfterFinish controls whether to debug uses of Trace values after finishing.
84 // FOR DEBUGGING ONLY. This will slow down the program.
85 var DebugUseAfterFinish = false
87 // AuthRequest determines whether a specific request is permitted to load the
88 // /debug/requests or /debug/events pages.
90 // It returns two bools; the first indicates whether the page may be viewed at all,
91 // and the second indicates whether sensitive events will be shown.
93 // AuthRequest may be replaced by a program to customize its authorization requirements.
95 // The default AuthRequest function returns (true, true) if and only if the request
96 // comes from localhost/127.0.0.1/[::1].
97 var AuthRequest = func(req *http.Request) (any, sensitive bool) {
98 // RemoteAddr is commonly in the form "IP" or "IP:port".
99 // If it is in the form "IP:port", split off the port.
100 host, _, err := net.SplitHostPort(req.RemoteAddr)
102 host = req.RemoteAddr
105 case "localhost", "127.0.0.1", "::1":
113 // TODO(jbd): Serve Traces from /debug/traces in the future?
114 // There is no requirement for a request to be present to have traces.
115 http.HandleFunc("/debug/requests", Traces)
116 http.HandleFunc("/debug/events", Events)
119 // Traces responds with traces from the program.
120 // The package initialization registers it in http.DefaultServeMux
121 // at /debug/requests.
123 // It performs authorization by running AuthRequest.
124 func Traces(w http.ResponseWriter, req *http.Request) {
125 any, sensitive := AuthRequest(req)
127 http.Error(w, "not allowed", http.StatusUnauthorized)
130 w.Header().Set("Content-Type", "text/html; charset=utf-8")
131 Render(w, req, sensitive)
134 // Events responds with a page of events collected by EventLogs.
135 // The package initialization registers it in http.DefaultServeMux
138 // It performs authorization by running AuthRequest.
139 func Events(w http.ResponseWriter, req *http.Request) {
140 any, sensitive := AuthRequest(req)
142 http.Error(w, "not allowed", http.StatusUnauthorized)
145 w.Header().Set("Content-Type", "text/html; charset=utf-8")
146 RenderEvents(w, req, sensitive)
149 // Render renders the HTML page typically served at /debug/requests.
150 // It does not do any auth checking. The request may be nil.
152 // Most users will use the Traces handler.
153 func Render(w io.Writer, req *http.Request, sensitive bool) {
156 ActiveTraceCount map[string]int
157 CompletedTraces map[string]*family
159 // Set when a bucket has been selected.
166 ShowSensitive bool // whether to show sensitive events
168 Histogram template.HTML
169 HistogramWindow string // e.g. "last minute", "last hour", "all time"
171 // If non-zero, the set of traces is a partial set,
172 // and this is the total number.
175 CompletedTraces: completedTraces,
178 data.ShowSensitive = sensitive
180 // Allow show_sensitive=0 to force hiding of sensitive data for testing.
181 // This only goes one way; you can't use show_sensitive=1 to see things.
182 if req.FormValue("show_sensitive") == "0" {
183 data.ShowSensitive = false
186 if exp, err := strconv.ParseBool(req.FormValue("exp")); err == nil {
189 if exp, err := strconv.ParseBool(req.FormValue("rtraced")); err == nil {
195 data.Families = make([]string, 0, len(completedTraces))
196 for fam := range completedTraces {
197 data.Families = append(data.Families, fam)
199 completedMu.RUnlock()
200 sort.Strings(data.Families)
202 // We are careful here to minimize the time spent locking activeMu,
203 // since that lock is required every time an RPC starts and finishes.
204 data.ActiveTraceCount = make(map[string]int, len(data.Families))
206 for fam, s := range activeTraces {
207 data.ActiveTraceCount[fam] = s.Len()
212 data.Family, data.Bucket, ok = parseArgs(req)
216 case data.Bucket == -1:
218 n := data.ActiveTraceCount[data.Family]
219 data.Traces = getActiveTraces(data.Family)
220 if len(data.Traces) < n {
223 case data.Bucket < bucketsPerFamily:
224 if b := lookupBucket(data.Family, data.Bucket); b != nil {
225 data.Traces = b.Copy(data.Traced)
228 if f := getFamily(data.Family, false); f != nil {
229 var obs timeseries.Observable
231 switch o := data.Bucket - bucketsPerFamily; o {
233 obs = f.Latency.Minute()
234 data.HistogramWindow = "last minute"
236 obs = f.Latency.Hour()
237 data.HistogramWindow = "last hour"
239 obs = f.Latency.Total()
240 data.HistogramWindow = "all time"
242 f.LatencyMu.RUnlock()
244 data.Histogram = obs.(*histogram).html()
249 if data.Traces != nil {
250 defer data.Traces.Free()
251 sort.Sort(data.Traces)
255 defer completedMu.RUnlock()
256 if err := pageTmpl().ExecuteTemplate(w, "Page", data); err != nil {
257 log.Printf("net/trace: Failed executing template: %v", err)
261 func parseArgs(req *http.Request) (fam string, b int, ok bool) {
265 fam, bStr := req.FormValue("fam"), req.FormValue("b")
266 if fam == "" || bStr == "" {
269 b, err := strconv.Atoi(bStr)
270 if err != nil || b < -1 {
277 func lookupBucket(fam string, b int) *traceBucket {
278 f := getFamily(fam, false)
279 if f == nil || b < 0 || b >= len(f.Buckets) {
285 type contextKeyT string
287 var contextKey = contextKeyT("golang.org/x/net/trace.Trace")
289 // Trace represents an active request.
290 type Trace interface {
291 // LazyLog adds x to the event log. It will be evaluated each time the
292 // /debug/requests page is rendered. Any memory referenced by x will be
293 // pinned until the trace is finished and later discarded.
294 LazyLog(x fmt.Stringer, sensitive bool)
296 // LazyPrintf evaluates its arguments with fmt.Sprintf each time the
297 // /debug/requests page is rendered. Any memory referenced by a will be
298 // pinned until the trace is finished and later discarded.
299 LazyPrintf(format string, a ...interface{})
301 // SetError declares that this trace resulted in an error.
304 // SetRecycler sets a recycler for the trace.
305 // f will be called for each event passed to LazyLog at a time when
306 // it is no longer required, whether while the trace is still active
307 // and the event is discarded, or when a completed trace is discarded.
308 SetRecycler(f func(interface{}))
310 // SetTraceInfo sets the trace info for the trace.
311 // This is currently unused.
312 SetTraceInfo(traceID, spanID uint64)
314 // SetMaxEvents sets the maximum number of events that will be stored
315 // in the trace. This has no effect if any events have already been
316 // added to the trace.
319 // Finish declares that this trace is complete.
320 // The trace should not be used after calling this method.
324 type lazySprintf struct {
329 func (l *lazySprintf) String() string {
330 return fmt.Sprintf(l.format, l.a...)
333 // New returns a new Trace with the specified family and title.
334 func New(family, title string) Trace {
337 tr.Family, tr.Title = family, title
338 tr.Start = time.Now()
339 tr.maxEvents = maxEventsPerTrace
340 tr.events = tr.eventsBuf[:0]
343 s := activeTraces[tr.Family]
347 s = activeTraces[tr.Family] // check again
350 activeTraces[tr.Family] = s
356 // Trigger allocation of the completed trace structure for this family.
357 // This will cause the family to be present in the request page during
358 // the first trace of this family. We don't care about the return value,
359 // nor is there any need for this to run inline, so we execute it in its
360 // own goroutine, but only if the family isn't allocated yet.
362 if _, ok := completedTraces[tr.Family]; !ok {
363 go allocFamily(tr.Family)
365 completedMu.RUnlock()
370 func (tr *trace) Finish() {
371 tr.Elapsed = time.Now().Sub(tr.Start)
372 if DebugUseAfterFinish {
373 buf := make([]byte, 4<<10) // 4 KB should be enough
374 n := runtime.Stack(buf, false)
375 tr.finishStack = buf[:n]
379 m := activeTraces[tr.Family]
383 f := getFamily(tr.Family, true)
384 for _, b := range f.Buckets {
385 if b.Cond.match(tr) {
389 // Add a sample of elapsed time as microseconds to the family's timeseries
391 h.addMeasurement(tr.Elapsed.Nanoseconds() / 1e3)
396 tr.unref() // matches ref in New
402 maxActiveTraces = 20 // Maximum number of active traces to show.
403 maxEventsPerTrace = 10
404 numHistogramBuckets = 38
408 // The active traces.
409 activeMu sync.RWMutex
410 activeTraces = make(map[string]*traceSet) // family -> traces
412 // Families of completed traces.
413 completedMu sync.RWMutex
414 completedTraces = make(map[string]*family) // family -> traces
417 type traceSet struct {
421 // We could avoid the entire map scan in FirstN by having a slice of all the traces
422 // ordered by start time, and an index into that from the trace struct, with a periodic
423 // repack of the slice after enough traces finish; we could also use a skip list or similar.
424 // However, that would shift some of the expense from /debug/requests time to RPC time,
425 // which is probably the wrong trade-off.
428 func (ts *traceSet) Len() int {
430 defer ts.mu.RUnlock()
434 func (ts *traceSet) Add(tr *trace) {
437 ts.m = make(map[*trace]bool)
443 func (ts *traceSet) Remove(tr *trace) {
449 // FirstN returns the first n traces ordered by time.
450 func (ts *traceSet) FirstN(n int) traceList {
452 defer ts.mu.RUnlock()
457 trl := make(traceList, 0, n)
459 // Fast path for when no selectivity is needed.
461 for tr := range ts.m {
463 trl = append(trl, tr)
469 // Pick the oldest n traces.
470 // This is inefficient. See the comment in the traceSet struct.
471 for tr := range ts.m {
472 // Put the first n traces into trl in the order they occur.
473 // When we have n, sort trl, and thereafter maintain its order.
476 trl = append(trl, tr)
478 // This is guaranteed to happen exactly once during this loop.
483 if tr.Start.After(trl[n-1].Start) {
487 // Find where to insert this one.
489 i := sort.Search(n, func(i int) bool { return trl[i].Start.After(tr.Start) })
491 copy(trl[i+1:], trl[i:])
498 func getActiveTraces(fam string) traceList {
500 s := activeTraces[fam]
505 return s.FirstN(maxActiveTraces)
508 func getFamily(fam string, allocNew bool) *family {
510 f := completedTraces[fam]
511 completedMu.RUnlock()
512 if f == nil && allocNew {
518 func allocFamily(fam string) *family {
520 defer completedMu.Unlock()
521 f := completedTraces[fam]
524 completedTraces[fam] = f
529 // family represents a set of trace buckets and associated latency information.
531 // traces may occur in multiple buckets.
532 Buckets [bucketsPerFamily]*traceBucket
534 // latency time series
535 LatencyMu sync.RWMutex
536 Latency *timeseries.MinuteHourSeries
539 func newFamily() *family {
541 Buckets: [bucketsPerFamily]*traceBucket{
543 {Cond: minCond(50 * time.Millisecond)},
544 {Cond: minCond(100 * time.Millisecond)},
545 {Cond: minCond(200 * time.Millisecond)},
546 {Cond: minCond(500 * time.Millisecond)},
547 {Cond: minCond(1 * time.Second)},
548 {Cond: minCond(10 * time.Second)},
549 {Cond: minCond(100 * time.Second)},
552 Latency: timeseries.NewMinuteHourSeries(func() timeseries.Observable { return new(histogram) }),
556 // traceBucket represents a size-capped bucket of historic traces,
557 // along with a condition for a trace to belong to the bucket.
558 type traceBucket struct {
561 // Ring buffer implementation of a fixed-size FIFO queue.
563 buf [tracesPerBucket]*trace
564 start int // < tracesPerBucket
565 length int // <= tracesPerBucket
568 func (b *traceBucket) Add(tr *trace) {
572 i := b.start + b.length
573 if i >= tracesPerBucket {
576 if b.length == tracesPerBucket {
577 // "Remove" an element from the bucket.
580 if b.start == tracesPerBucket {
585 if b.length < tracesPerBucket {
591 // Copy returns a copy of the traces in the bucket.
592 // If tracedOnly is true, only the traces with trace information will be returned.
593 // The logs will be ref'd before returning; the caller should call
594 // the Free method when it is done with them.
595 // TODO(dsymonds): keep track of traced requests in separate buckets.
596 func (b *traceBucket) Copy(tracedOnly bool) traceList {
600 trl := make(traceList, 0, b.length)
601 for i, x := 0, b.start; i < b.length; i++ {
603 if !tracedOnly || tr.spanID != 0 {
605 trl = append(trl, tr)
615 func (b *traceBucket) Empty() bool {
621 // cond represents a condition on a trace.
622 type cond interface {
627 type minCond time.Duration
629 func (m minCond) match(t *trace) bool { return t.Elapsed >= time.Duration(m) }
630 func (m minCond) String() string { return fmt.Sprintf("≥%gs", time.Duration(m).Seconds()) }
632 type errorCond struct{}
634 func (e errorCond) match(t *trace) bool { return t.IsError }
635 func (e errorCond) String() string { return "errors" }
637 type traceList []*trace
639 // Free calls unref on each element of the list.
640 func (trl traceList) Free() {
641 for _, t := range trl {
646 // traceList may be sorted in reverse chronological order.
647 func (trl traceList) Len() int { return len(trl) }
648 func (trl traceList) Less(i, j int) bool { return trl[i].Start.After(trl[j].Start) }
649 func (trl traceList) Swap(i, j int) { trl[i], trl[j] = trl[j], trl[i] }
651 // An event is a timestamped log entry in a trace.
654 Elapsed time.Duration // since previous event in trace
655 NewDay bool // whether this event is on a different day to the previous event
656 Recyclable bool // whether this event was passed via LazyLog
657 Sensitive bool // whether this event contains sensitive information
658 What interface{} // string or fmt.Stringer
661 // WhenString returns a string representation of the elapsed time of the event.
662 // It will include the date if midnight was crossed.
663 func (e event) WhenString() string {
665 return e.When.Format("2006/01/02 15:04:05.000000")
667 return e.When.Format("15:04:05.000000")
670 // discarded represents a number of discarded events.
671 // It is stored as *discarded to make it easier to update in-place.
674 func (d *discarded) String() string {
675 return fmt.Sprintf("(%d events discarded)", int(*d))
678 // trace represents an active or complete request,
679 // either sent or received by this program.
681 // Family is the top-level grouping of traces to which this belongs.
684 // Title is the title of this trace.
687 // Timing information.
689 Elapsed time.Duration // zero while active
691 // Trace information if non-zero.
695 // Whether this trace resulted in an error.
698 // Append-only sequence of events (modulo discards).
703 refs int32 // how many buckets this is in
704 recycler func(interface{})
705 disc discarded // scratch space to avoid allocation
707 finishStack []byte // where finish was called, if DebugUseAfterFinish is set
709 eventsBuf [4]event // preallocated buffer in case we only log a few events
712 func (tr *trace) reset() {
713 // Clear all but the mutex. Mutexes may not be copied, even when unlocked.
716 tr.Start = time.Time{}
727 for i := range tr.eventsBuf {
728 tr.eventsBuf[i] = event{}
732 // delta returns the elapsed time since the last event or the trace start,
733 // and whether it spans midnight.
735 func (tr *trace) delta(t time.Time) (time.Duration, bool) {
736 if len(tr.events) == 0 {
737 return t.Sub(tr.Start), false
739 prev := tr.events[len(tr.events)-1].When
740 return t.Sub(prev), prev.Day() != t.Day()
743 func (tr *trace) addEvent(x interface{}, recyclable, sensitive bool) {
744 if DebugUseAfterFinish && tr.finishStack != nil {
745 buf := make([]byte, 4<<10) // 4 KB should be enough
746 n := runtime.Stack(buf, false)
747 log.Printf("net/trace: trace used after finish:\nFinished at:\n%s\nUsed at:\n%s", tr.finishStack, buf[:n])
753 If you are here because your program panicked in this code,
754 it is almost definitely the fault of code using this package,
755 and very unlikely to be the fault of this code.
757 The most likely scenario is that some code elsewhere is using
758 a trace.Trace after its Finish method is called.
759 You can temporarily set the DebugUseAfterFinish var
760 to help discover where that is; do not leave that var set,
761 since it makes this package much less efficient.
764 e := event{When: time.Now(), What: x, Recyclable: recyclable, Sensitive: sensitive}
766 e.Elapsed, e.NewDay = tr.delta(e.When)
767 if len(tr.events) < tr.maxEvents {
768 tr.events = append(tr.events, e)
770 // Discard the middle events.
771 di := int((tr.maxEvents - 1) / 2)
772 if d, ok := tr.events[di].What.(*discarded); ok {
775 // disc starts at two to count for the event it is replacing,
776 // plus the next one that we are about to drop.
778 if tr.recycler != nil && tr.events[di].Recyclable {
779 go tr.recycler(tr.events[di].What)
781 tr.events[di].What = &tr.disc
783 // The timestamp of the discarded meta-event should be
784 // the time of the last event it is representing.
785 tr.events[di].When = tr.events[di+1].When
787 if tr.recycler != nil && tr.events[di+1].Recyclable {
788 go tr.recycler(tr.events[di+1].What)
790 copy(tr.events[di+1:], tr.events[di+2:])
791 tr.events[tr.maxEvents-1] = e
796 func (tr *trace) LazyLog(x fmt.Stringer, sensitive bool) {
797 tr.addEvent(x, true, sensitive)
800 func (tr *trace) LazyPrintf(format string, a ...interface{}) {
801 tr.addEvent(&lazySprintf{format, a}, false, false)
804 func (tr *trace) SetError() { tr.IsError = true }
806 func (tr *trace) SetRecycler(f func(interface{})) {
810 func (tr *trace) SetTraceInfo(traceID, spanID uint64) {
811 tr.traceID, tr.spanID = traceID, spanID
814 func (tr *trace) SetMaxEvents(m int) {
815 // Always keep at least three events: first, discarded count, last.
816 if len(tr.events) == 0 && m > 3 {
821 func (tr *trace) ref() {
822 atomic.AddInt32(&tr.refs, 1)
825 func (tr *trace) unref() {
826 if atomic.AddInt32(&tr.refs, -1) == 0 {
827 if tr.recycler != nil {
828 // freeTrace clears tr, so we hold tr.recycler and tr.events here.
829 go func(f func(interface{}), es []event) {
830 for _, e := range es {
835 }(tr.recycler, tr.events)
842 func (tr *trace) When() string {
843 return tr.Start.Format("2006/01/02 15:04:05.000000")
846 func (tr *trace) ElapsedTime() string {
850 t = time.Since(tr.Start)
852 return fmt.Sprintf("%.6f", t.Seconds())
855 func (tr *trace) Events() []event {
857 defer tr.mu.RUnlock()
861 var traceFreeList = make(chan *trace, 1000) // TODO(dsymonds): Use sync.Pool?
863 // newTrace returns a trace ready to use.
864 func newTrace() *trace {
866 case tr := <-traceFreeList:
873 // freeTrace adds tr to traceFreeList if there's room.
874 // This is non-blocking.
875 func freeTrace(tr *trace) {
876 if DebugUseAfterFinish {
877 return // never reuse
881 case traceFreeList <- tr:
886 func elapsed(d time.Duration) string {
887 b := []byte(fmt.Sprintf("%.6f", d.Seconds()))
889 // For subsecond durations, blank all zeros before decimal point,
890 // and all zeros between the decimal point and the first non-zero digit.
892 dot := bytes.IndexByte(b, '.')
893 for i := 0; i < dot; i++ {
896 for i := dot + 1; i < len(b); i++ {
908 var pageTmplCache *template.Template
909 var pageTmplOnce sync.Once
911 func pageTmpl() *template.Template {
912 pageTmplOnce.Do(func() {
913 pageTmplCache = template.Must(template.New("Page").Funcs(template.FuncMap{
915 "add": func(a, b int) int { return a + b },
922 {{template "Prolog" .}}
923 {{template "StatusTable" .}}
924 {{template "Epilog" .}}
929 <title>/debug/requests</title>
930 <style type="text/css">
932 font-family: sans-serif;
934 table#tr-status td.family {
937 table#tr-status td.active {
940 table#tr-status td.latency-first {
943 table#tr-status td.empty {
949 table#reqs tr.first {
950 {{if $.Expanded}}font-weight: bold;{{end}}
953 font-family: monospace;
959 table#reqs td.elapsed {
973 <h1>/debug/requests</h1>
974 {{end}} {{/* end of Prolog */}}
976 {{define "StatusTable"}}
977 <table id="tr-status">
978 {{range $fam := .Families}}
980 <td class="family">{{$fam}}</td>
982 {{$n := index $.ActiveTraceCount $fam}}
983 <td class="active {{if not $n}}empty{{end}}">
984 {{if $n}}<a href="?fam={{$fam}}&b=-1{{if $.Expanded}}&exp=1{{end}}">{{end}}
989 {{$f := index $.CompletedTraces $fam}}
990 {{range $i, $b := $f.Buckets}}
991 {{$empty := $b.Empty}}
992 <td {{if $empty}}class="empty"{{end}}>
993 {{if not $empty}}<a href="?fam={{$fam}}&b={{$i}}{{if $.Expanded}}&exp=1{{end}}">{{end}}
995 {{if not $empty}}</a>{{end}}
999 {{$nb := len $f.Buckets}}
1000 <td class="latency-first">
1001 <a href="?fam={{$fam}}&b={{$nb}}">[minute]</a>
1004 <a href="?fam={{$fam}}&b={{add $nb 1}}">[hour]</a>
1007 <a href="?fam={{$fam}}&b={{add $nb 2}}">[total]</a>
1013 {{end}} {{/* end of StatusTable */}}
1018 <h3>Family: {{$.Family}}</h3>
1020 {{if or $.Expanded $.Traced}}
1021 <a href="?fam={{$.Family}}&b={{$.Bucket}}">[Normal/Summary]</a>
1026 {{if or (not $.Expanded) $.Traced}}
1027 <a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1">[Normal/Expanded]</a>
1033 {{if or $.Expanded (not $.Traced)}}
1034 <a href="?fam={{$.Family}}&b={{$.Bucket}}&rtraced=1">[Traced/Summary]</a>
1038 {{if or (not $.Expanded) (not $.Traced)}}
1039 <a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1&rtraced=1">[Traced/Expanded]</a>
1046 <p><em>Showing <b>{{len $.Traces}}</b> of <b>{{$.Total}}</b> traces.</em></p>
1051 {{if $.Active}}Active{{else}}Completed{{end}} Requests
1053 <tr><th>When</th><th>Elapsed (s)</th></tr>
1054 {{range $tr := $.Traces}}
1056 <td class="when">{{$tr.When}}</td>
1057 <td class="elapsed">{{$tr.ElapsedTime}}</td>
1058 <td>{{$tr.Title}}</td>
1059 {{/* TODO: include traceID/spanID */}}
1062 {{range $tr.Events}}
1064 <td class="when">{{.WhenString}}</td>
1065 <td class="elapsed">{{elapsed .Elapsed}}</td>
1066 <td>{{if or $.ShowSensitive (not .Sensitive)}}... {{.What}}{{else}}<em>[redacted]</em>{{end}}</td>
1072 {{end}} {{/* if $.Traces */}}
1075 <h4>Latency (µs) of {{$.Family}} over {{$.HistogramWindow}}</h4>
1077 {{end}} {{/* if $.Histogram */}}
1081 {{end}} {{/* end of Epilog */}}