matoski.com

A lot of times it’s useful to expose metrics for your application so you can know what is going on with the application, there are two ways you can do this one it’s either using a pull or a push mechanism.

Golang has a package called expvar that provides a standardized interface to public variables, such as operation counters in servers, and exports these variables via HTTP in a JSON format.

We can use this package to export all our metrics, custom variables, counters or anything we would want and then visualize our data so we can analyze it and understand how our application works. The package has several types that it exports like float, int, string, map, and we can even define custom types to our metrics by implementing the Var interface

Let’s take a look at a simple implementation of how we can use expvar to export various metrics for our application, and even implementing a custom type that we can use in our application.

package main

import (
	"expvar"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"runtime"
	"time"
)

type TimeVar struct{ v time.Time }

func (o *TimeVar) Set(date time.Time)         { o.v = date }
func (o *TimeVar) Add(duration time.Duration) { o.v = o.v.Add(duration) }
func (o *TimeVar) String() string             { return fmt.Sprintf('"%s"', o.v.Format(time.RFC3339)) }

func NewStats(name string) *expvar.Map {
	stats = expvar.NewMap(name)
	stats.Set("counter", new(expvar.Int))
	stats.Set("success_rate", new(expvar.Float))
	stats.Set("pid", new(expvar.Int))

	go func() {
		t := time.NewTicker(time.Millisecond)
		for {
			select {
			case <-t.C:
				stats.Add("counter", rand.Int63n(100))
				stats.Get("success_rate").(*expvar.Float).Set(rand.Float64())
			}
		}
	}()

	return stats
}

var stats *expvar.Map
var lastUpdate *TimeVar

func init() {
	stats = NewStats("stats")
	lastUpdate = &TimeVar{}
	lastUpdate.Set(time.Now())
	stats.Get("pid").(*expvar.Int).Set(int64(os.Getpid()))
	expvar.Publish("last_update", lastUpdate)
	expvar.Publish("goroutines", expvar.Func(func() interface{} {
		return fmt.Sprintf("%d", runtime.NumGoroutine())
	}))
	expvar.Publish("cgocall", expvar.Func(func() interface{} {
		return fmt.Sprintf("%d", runtime.NumCgoCall())
	}))
	expvar.Publish("cpu", expvar.Func(func() interface{} {
		return fmt.Sprintf("%d", runtime.NumCPU())
	}))
}

func randomGoRoutineSpawn() {

	for {
		go func() {
			r := rand.Intn(10)
			time.Sleep(time.Duration(r) * time.Microsecond)
		}()
		r := rand.Intn(3)
		time.Sleep(time.Duration(r) * time.Microsecond)
	}
}

func main() {
	go randomGoRoutineSpawn()
	http.ListenAndServe(":6000", http.DefaultServeMux)

}

You may have noticed that we are using expvar.Func with Publish, what this one means is that the function will be called anytime we are retrieving the metrics, which makes it perfect for calculating the values only when they are needed and not all the time.

After running the application and querying the endpoint we have the following data.

$ curl localhost:6000/debug/vars

{
 "cgocall": "1",
 "cmdline": ["/tmp/go-build381034895/b001/exe/main"],
 "cpu": "4",
 "goroutines": "8",
 "last_update": "2018-09-30T09:59:08+02:00",
 "memstats": {"Alloc":2729112,"TotalAlloc":67979672,"Sys":72349944,"Lookups":0,"Mallocs":1055537,"Frees":1016347,"HeapAlloc":2729112,"HeapSys":66191360,"HeapIdle":62775296,"HeapInuse":3416064,"HeapReleased":0,"HeapObjects":39190,"StackInuse":917504,"StackSys":917504,"MSpanInuse":64752,"MSpanSys":98304,"MCacheInuse":6912,"MCacheSys":16384,"BuckHashSys":1442870,"GCSys":2445312,"OtherSys":1238210,"NextGC":4194304,"LastGC":1538294385797517989,"PauseTotalNs":1435570,"PauseNs":[79199,42451,50945,155414,71632,34360,112997,78210,80689,94101,197658,110553,47645,97254,37034,58762,86666,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"PauseEnd":[1538294350363250463,1538294352540250881,1538294354731420295,1538294356927816598,1538294358983245342,1538294361184114356,1538294363384264942,1538294365590777905,1538294367789220774,1538294370179305387,1538294372387563780,1538294374656050641,1538294376896879760,1538294379162055815,1538294381539608947,1538294383722948035,1538294385797517989,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"NumGC":17,"NumForcedGC":0,"GCCPUFraction":0.00008913387405867569,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":59,"Frees":24},{"Size":16,"Mallocs":579,"Frees":180},{"Size":32,"Mallocs":133,"Frees":72},{"Size":48,"Mallocs":163,"Frees":48},{"Size":64,"Mallocs":1053800,"Frees":1015659},{"Size":80,"Mallocs":25,"Frees":9},{"Size":96,"Mallocs":60,"Frees":8},{"Size":112,"Mallocs":11,"Frees":8},{"Size":128,"Mallocs":22,"Frees":12},{"Size":144,"Mallocs":7,"Frees":6},{"Size":160,"Mallocs":18,"Frees":5},{"Size":176,"Mallocs":8,"Frees":3},{"Size":192,"Mallocs":8,"Frees":6},{"Size":208,"Mallocs":39,"Frees":16},{"Size":224,"Mallocs":5,"Frees":3},{"Size":240,"Mallocs":0,"Frees":0},{"Size":256,"Mallocs":25,"Frees":7},{"Size":288,"Mallocs":11,"Frees":8},{"Size":320,"Mallocs":2,"Frees":1},{"Size":352,"Mallocs":21,"Frees":14},{"Size":384,"Mallocs":223,"Frees":0},{"Size":416,"Mallocs":8,"Frees":4},{"Size":448,"Mallocs":1,"Frees":0},{"Size":480,"Mallocs":0,"Frees":0},{"Size":512,"Mallocs":28,"Frees":22},{"Size":576,"Mallocs":8,"Frees":6},{"Size":640,"Mallocs":3,"Frees":1},{"Size":704,"Mallocs":2,"Frees":1},{"Size":768,"Mallocs":2,"Frees":0},{"Size":896,"Mallocs":16,"Frees":4},{"Size":1024,"Mallocs":2,"Frees":1},{"Size":1152,"Mallocs":8,"Frees":7},{"Size":1280,"Mallocs":1,"Frees":1},{"Size":1408,"Mallocs":1,"Frees":1},{"Size":1536,"Mallocs":1,"Frees":0},{"Size":1792,"Mallocs":6,"Frees":2},{"Size":2048,"Mallocs":5,"Frees":3},{"Size":2304,"Mallocs":7,"Frees":3},{"Size":2688,"Mallocs":2,"Frees":1},{"Size":3072,"Mallocs":2,"Frees":1},{"Size":3200,"Mallocs":0,"Frees":0},{"Size":3456,"Mallocs":0,"Frees":0},{"Size":4096,"Mallocs":14,"Frees":9},{"Size":4864,"Mallocs":9,"Frees":9},{"Size":5376,"Mallocs":1,"Frees":0},{"Size":6144,"Mallocs":4,"Frees":3},{"Size":6528,"Mallocs":2,"Frees":0},{"Size":6784,"Mallocs":0,"Frees":0},{"Size":6912,"Mallocs":0,"Frees":0},{"Size":8192,"Mallocs":1,"Frees":0},{"Size":9472,"Mallocs":4,"Frees":0},{"Size":9728,"Mallocs":0,"Frees":0},{"Size":10240,"Mallocs":0,"Frees":0},{"Size":10880,"Mallocs":0,"Frees":0},{"Size":12288,"Mallocs":0,"Frees":0},{"Size":13568,"Mallocs":0,"Frees":0},{"Size":14336,"Mallocs":0,"Frees":0},{"Size":16384,"Mallocs":0,"Frees":0},{"Size":18432,"Mallocs":0,"Frees":0},{"Size":19072,"Mallocs":0,"Frees":0}]},
 "stats": {"counter": 1893889, "pid": 23657, "success_rate": 0.45610231133193085}
}

Now we need to store this data somewhere, and we can visualize it and use the data in a lot of useful ways for us.

If you want to try it out you can pull the code from GitHub and run it in a container to see it in action

$ git clone https://github.com/ilijamt/gomeetupamsoct2018
$ cd watch
$ docker-compose up -d
$ export EXPVAR_HOST=http://`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' expvar_app_1`:6000/debug/vars
$ watch -n 1 curl $EXPVAR_HOST -s

This is one of the ways we can do the export of the metrics, another one is to use Prometheus, which I will cover in a future article, with the same example just to see how to implement it in Prometheus way.