Add filtering for browsers, countries and paths

10 files changed, 220 insertions, 304 deletions

diff --git a/cmd/cron/main.go b/cmd/cron/main.go
index af1e734..f9a6b3f 100644
--- a/cmd/cron/main.go
+++ b/cmd/cron/main.go
@@ -12,6 +12,7 @@ import (
 	_ "github.com/jackc/pgx/stdlib"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/mattn/go-sqlite3"
+	detector "github.com/mssola/user_agent"
 	"src.userspace.com.au/sws"
 	"src.userspace.com.au/sws/store"
 )
@@ -97,6 +98,8 @@ func main() {
 		panic(err)
 	}
 
+	seenUAs := make(map[string]bool)
+
 	toUpdate := make([]*sws.Hit, 0)
 	log("updating country code")
 	err = st.HitCursor(func(h *sws.Hit) error {
@@ -114,6 +117,27 @@ func main() {
 			h.CountryCode = cc
 			toUpdate = append(toUpdate, h)
 		}
+
+		// Populate user agent
+		if h.UserAgent != nil {
+			log("hit ID", *h.ID)
+			ua := h.UserAgent
+			if ok := seenUAs[ua.Hash]; ok {
+				return nil
+			}
+			det := detector.New(ua.Name)
+			browser, version := det.Browser()
+
+			h.UserAgent.Browser = browser
+			h.UserAgent.Platform = det.Platform()
+			h.UserAgent.Version = version
+			h.UserAgent.Bot = det.Bot()
+			h.UserAgent.Mobile = strings.Contains(ua.Name, "Mobi") || det.Mobile()
+
+			seenUAs[ua.Hash] = true
+
+			toUpdate = append(toUpdate, h)
+		}
 		return nil
 	})
 	if err != nil {
diff --git a/cmd/server/charts.go b/cmd/server/charts.go
index fce5c86..a8e9238 100644
--- a/cmd/server/charts.go
+++ b/cmd/server/charts.go
@@ -4,7 +4,6 @@ import (
 	"crypto/sha1"
 	"fmt"
 	"net/http"
-	"strconv"
 	"strings"
 	"time"
 
@@ -22,23 +21,9 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
 			return
 		}
 
-		chartType := chi.URLParam(r, "type")
-		dataType := chi.URLParam(r, "data")
-		beginSecs, err := strconv.ParseInt(chi.URLParam(r, "begin"), 10, 64)
-		if err != nil {
-			httpError(w, http.StatusNotFound, err.Error())
-			return
-		}
-		endSecs, err := strconv.ParseInt(chi.URLParam(r, "end"), 10, 64)
-		if err != nil {
-			httpError(w, http.StatusNotFound, err.Error())
-			return
-		}
-		begin := time.Unix(beginSecs, 0)
-		end := time.Unix(endSecs, 0)
-
 		var b strings.Builder
 		b.WriteString(r.URL.Path)
+		b.WriteString(r.URL.Query().Encode())
 		// FIXME
 		b.WriteString(time.Now().Truncate(30 * time.Minute).String())
 		etag := fmt.Sprintf(`"%x"`, sha1.Sum([]byte(b.String())))
@@ -50,10 +35,18 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
 			}
 		}
 
-		filter := map[string]interface{}{
-			"begin": begin,
-			"end":   end,
+		filter := createHitFilter(r)
+		begin, end := extractTimeRange(r)
+		if begin == nil || end == nil {
+			log("charts: empty times")
+			httpError(w, http.StatusNotFound, "not found")
+			return
 		}
+		filter["begin"] = *begin
+		filter["end"] = *end
+
+		chartType := chi.URLParam(r, "type")
+		dataType := chi.URLParam(r, "data")
 
 		hits, err := db.GetHits(*site, filter)
 		if err != nil {
@@ -71,7 +64,7 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
 			return
 		}
 
-		hitSet.Fill(&begin, &end)
+		hitSet.Fill(begin, end)
 		hitSet.SortByDate()
 
 		w.Header().Set("Etag", etag)
@@ -96,6 +89,9 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
 			}
 		case "b":
 			browsers := sws.NewBrowserSet(hitSet)
+			if browsers == nil {
+				return
+			}
 			browsers.SortByHits()
 			w.Header().Set("Content-Type", "image/svg+xml")
 			switch chartType {
diff --git a/cmd/server/country.go b/cmd/server/country.go
deleted file mode 100644
index bb4b22e..0000000
--- a/cmd/server/country.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package main
-
-import (
-	"net/http"
-
-	"src.userspace.com.au/sws"
-)
-
-func handleCountries(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		site, ok := ctx.Value("site").(*sws.Site)
-		if !ok {
-			httpError(w, http.StatusBadRequest, "no site in context")
-			return
-		}
-
-		payload := newTemplateData(r)
-		payload.Site = site
-
-		begin, end := extractTimeRange(r)
-		if begin == nil || end == nil {
-			httpError(w, http.StatusBadRequest, "invalid time range")
-			return
-		}
-		payload.Begin = *begin
-		payload.End = *end
-		debug("begin", *begin)
-		debug("end", *end)
-
-		filter := map[string]interface{}{
-			"begin": *begin,
-			"end":   *end,
-		}
-
-		q := r.URL.Query()
-
-		cc := q.Get("country")
-		if cc != "" {
-			filter["countryCode"] = cc
-		}
-
-		hits, err := db.GetHits(*site, filter)
-		hitSet, err := sws.NewHitSet(sws.FromHits(hits))
-		if err != nil {
-			httpError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-		hitSet.Fill(begin, end)
-		hitSet.SortByDate()
-		if err := expandPayload(hitSet, payload); err != nil {
-			httpError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-
-		// Single or multiple paths
-		if cc == "" {
-			countrySet := sws.NewCountrySet(hitSet)
-			countrySet.SortByHits()
-			payload.CountrySet = countrySet
-		}
-
-		if err := rndr.Render(w, "site", payload); err != nil {
-			httpError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-	}
-}
diff --git a/cmd/server/handlers.go b/cmd/server/handlers.go
index 985955a..00de42b 100644
--- a/cmd/server/handlers.go
+++ b/cmd/server/handlers.go
@@ -3,6 +3,7 @@ package main
 import (
 	"html/template"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 
@@ -15,17 +16,24 @@ type templateData struct {
 	Endpoint string
 	User     *sws.User
 	Flash    template.HTML
-	Begin    time.Time
-	End      time.Time
+	Begin    *time.Time
+	End      *time.Time
+
+	Query url.Values
 
 	Site        *sws.Site
 	Sites       []*sws.Site
+	Hits        *sws.HitSet
 	PageSet     *sws.PageSet
-	Page        *sws.Page
-	Browsers    *sws.BrowserSet
+	BrowserSet  *sws.BrowserSet
 	ReferrerSet *sws.ReferrerSet
 	CountrySet  *sws.CountrySet
-	Hits        *sws.HitSet
+}
+
+func (td templateData) QuerySetEncode(k, v string) template.URL {
+	qs, _ := url.ParseQuery(td.Query.Encode())
+	qs.Set(k, v)
+	return template.URL(qs.Encode())
 }
 
 func newTemplateData(r *http.Request) *templateData {
@@ -34,24 +42,29 @@ func newTemplateData(r *http.Request) *templateData {
 		Payload:  "//" + domain + "/sws.js",
 		Endpoint: "//" + domain + "/sws.gif",
 	}
-	if r != nil {
-		flashes := flashGet(r)
-		var flash strings.Builder
-		for _, f := range flashes {
-			flash.WriteString(`<span class="notification is-`)
-			flash.WriteString(string(f.Level))
-			flash.WriteString(`">`)
-			flash.WriteString(f.Message)
-			flash.WriteString("</span>")
-		}
-		if len(flashes) > 0 {
-			out.Flash = template.HTML(flash.String())
-		}
+	if r == nil {
+		return out
+	}
 
-		if user := r.Context().Value("user"); user != nil {
-			out.User = user.(*sws.User)
-		}
+	flashes := flashGet(r)
+	var flash strings.Builder
+	for _, f := range flashes {
+		flash.WriteString(`<span class="notification is-`)
+		flash.WriteString(string(f.Level))
+		flash.WriteString(`">`)
+		flash.WriteString(f.Message)
+		flash.WriteString("</span>")
 	}
+	if len(flashes) > 0 {
+		out.Flash = template.HTML(flash.String())
+	}
+
+	if user := r.Context().Value("user"); user != nil {
+		out.User = user.(*sws.User)
+	}
+
+	out.Query = r.URL.Query()
+
 	return out
 }
 
diff --git a/cmd/server/helpers.go b/cmd/server/helpers.go
index ff7fd5c..ff19e2b 100644
--- a/cmd/server/helpers.go
+++ b/cmd/server/helpers.go
@@ -12,13 +12,15 @@ import (
 )
 
 var funcMap = template.FuncMap{
-	"piechart": func(siteID int, dataType string, begin, end time.Time) string {
-		return fmt.Sprintf("/sites/%d/charts/p-%s-%d-%d.svg", siteID, dataType, begin.Unix(), end.Unix())
+	"piechart": func(dataType string, pl templateData) string {
+		pl.Query.Add("begin", strconv.Itoa(int(pl.Begin.Unix())))
+		pl.Query.Add("end", strconv.Itoa(int(pl.End.Unix())))
+		return fmt.Sprintf("/sites/%d/charts/p-%s.svg?%s", *pl.Site.ID, dataType, pl.Query.Encode())
 	},
 	"sparkline": func(id int) string {
 		now := time.Now().Truncate(time.Hour)
 		then := now.Add(-168 * time.Hour) // 7 days
-		return fmt.Sprintf("/sites/%d/charts/s-h-%d-%d.svg", id, then.Unix(), now.Unix())
+		return fmt.Sprintf("/sites/%d/charts/s-h.svg?begin=%d&end=%d", id, then.Unix(), now.Unix())
 	},
 	"tz": func(s string, t time.Time) time.Time {
 		tz, _ := time.LoadLocation(s)
@@ -61,9 +63,9 @@ var funcMap = template.FuncMap{
 	"timeRFC": func(t time.Time) string {
 		return t.Format("15:04")
 	},
-	"datetimeRelative": func(d string) int64 {
+	"datetimeRelative": func(d string) string {
 		dur, _ := time.ParseDuration(d)
-		return time.Now().Add(dur).Unix()
+		return strconv.Itoa(int(time.Now().Add(dur).Unix()))
 	},
 	"percent": func(a, b int) float64 {
 		return (float64(a) / float64(b)) * 100
@@ -82,6 +84,32 @@ func httpError(w http.ResponseWriter, code int, msg string) {
 	http.Error(w, http.StatusText(code), code)
 }
 
+func createHitFilter(r *http.Request) map[string]interface{} {
+	filter := make(map[string]interface{})
+
+	q := r.URL.Query()
+
+	path := q.Get("path")
+	if path != "" {
+		filter["path"] = path
+	}
+	country := q.Get("country")
+	if country != "" {
+		filter["country_code"] = country
+	}
+	browser := q.Get("browser")
+	if browser != "" {
+		filter["ua.browser"] = browser
+	}
+	if bots := q.Get("bots"); bots != "" {
+		filter["ua.bot"] = (bots == "1")
+	}
+	if mobile := q.Get("mobile"); mobile != "" {
+		filter["ua.mobile"] = (mobile == "1")
+	}
+	return filter
+}
+
 func extractTimeRange(r *http.Request) (*time.Time, *time.Time) {
 	// Default to 1 week ago
 	begin := timePtr(time.Now().Truncate(time.Hour).Add(-168 * time.Hour))
@@ -100,29 +128,6 @@ func extractTimeRange(r *http.Request) (*time.Time, *time.Time) {
 	return begin, end
 }
 
-func expandPayload(hs *sws.HitSet, pl *templateData) error {
-	pl.Hits = hs
-
-	pageSet, err := sws.NewPageSet(hs)
-	if err != nil {
-		return err
-	}
-
-	if pageSet != nil {
-		pageSet.SortByHits()
-		pl.PageSet = pageSet
-	}
-	pl.Browsers = sws.NewBrowserSet(hs)
-	pl.CountrySet = sws.NewCountrySet(hs)
-
-	refSet := sws.NewReferrerSet(hs)
-	if refSet != nil {
-		refSet.SortByHits()
-		pl.ReferrerSet = refSet
-	}
-	return nil
-}
-
 func stringPtr(s string) *string {
 	return &s
 }
diff --git a/cmd/server/hits.go b/cmd/server/hits.go
index 3a76bea..776e610 100644
--- a/cmd/server/hits.go
+++ b/cmd/server/hits.go
@@ -24,7 +24,7 @@ func handleHitCounter(db sws.CounterStore, mmdbPath string) http.HandlerFunc {
 		panic(err)
 	}
 
-	cache, err := lru.New(100)
+	countryCache, err := lru.New(100)
 	if err != nil {
 		panic(err)
 	}
@@ -67,13 +67,13 @@ func handleHitCounter(db sws.CounterStore, mmdbPath string) http.HandlerFunc {
 
 		if err == nil && hit.Addr != "" {
 			var cc *string
-			if v, ok := cache.Get(hit.Addr); ok {
+			if v, ok := countryCache.Get(hit.Addr); ok {
 				cc = v.(*string)
 			} else if mmdbPath != "" {
 				if cc, err = sws.FetchCountryCode(mmdbPath, hit.Addr); err != nil {
 					log("geoip lookup failed:", err)
 				}
-				cache.Add(hit.Addr, cc)
+				countryCache.Add(hit.Addr, cc)
 			}
 			hit.CountryCode = cc
 			debug("geolocated:", hit.Addr, "to", *hit.CountryCode)
diff --git a/cmd/server/pages.go b/cmd/server/pages.go
deleted file mode 100644
index 702194e..0000000
--- a/cmd/server/pages.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package main
-
-import (
-	"net/http"
-
-	"src.userspace.com.au/sws"
-)
-
-func handlePages(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		site, ok := ctx.Value("site").(*sws.Site)
-		if !ok {
-			httpError(w, http.StatusBadRequest, "no site in context")
-			return
-		}
-
-		payload := newTemplateData(r)
-		payload.Site = site
-
-		begin, end := extractTimeRange(r)
-		if begin == nil || end == nil {
-			httpError(w, http.StatusBadRequest, "invalid time range")
-			return
-		}
-		payload.Begin = *begin
-		payload.End = *end
-		debug("begin", *begin)
-		debug("end", *end)
-
-		filter := map[string]interface{}{
-			"begin": *begin,
-			"end":   *end,
-		}
-
-		q := r.URL.Query()
-
-		path := q.Get("path")
-		if path != "" {
-			filter["path"] = path
-		}
-
-		hits, err := db.GetHits(*site, filter)
-		hitSet, err := sws.NewHitSet(sws.FromHits(hits))
-		if err != nil {
-			httpError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-		hitSet.Fill(begin, end)
-		hitSet.SortByDate()
-
-		payload.Page = sws.NewPage(hitSet)
-		payload.Hits = hitSet
-		payload.Browsers = sws.NewBrowserSet(hitSet)
-		payload.CountrySet = sws.NewCountrySet(hitSet)
-		payload.ReferrerSet = sws.NewReferrerSet(hitSet)
-
-		// Single or multiple paths
-		if path == "" {
-			pageSet, err := sws.NewPageSet(hitSet)
-			if err != nil {
-				httpError(w, http.StatusInternalServerError, err.Error())
-				return
-			}
-
-			pageSet.SortByHits()
-			payload.PageSet = pageSet
-		}
-
-		if err := rndr.Render(w, "pages", payload); err != nil {
-			httpError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-	}
-}
diff --git a/cmd/server/routes.go b/cmd/server/routes.go
index 0e6462b..ae018b6 100644
--- a/cmd/server/routes.go
+++ b/cmd/server/routes.go
@@ -27,7 +27,7 @@ func init() {
 func createRouter(db sws.Store, mmdbPath string) (chi.Router, error) {
 	tmplsCommon := []string{"flash.tmpl", "navbar.tmpl"}
 	tmplsAuthed := append(tmplsCommon, []string{
-		"layout.tmpl", "charts.tmpl", "timerange.tmpl", "hitView.tmpl",
+		"layout.tmpl", "charts.tmpl", "filter.tmpl", "hitView.tmpl",
 	}...)
 	tmplsPublic := append(tmplsCommon, "layout.tmpl")
 
@@ -139,11 +139,10 @@ func createRouter(db sws.Store, mmdbPath string) (chi.Router, error) {
 					r.Use(getSiteCtx(db))
 					r.Get("/", siteHandler)
 					r.Post("/", siteHandler)
-					r.Get("/pages", handlePages(db, rndr))
 					r.Get("/edit", handleSiteEdit(db, rndr))
 
 					r.Route("/charts", func(r chi.Router) {
-						r.Get("/{type:(p|s|b)}-{data:(h|b|c)}-{begin:\\d+}-{end:\\d+}.svg", chartHandler(db))
+						r.Get("/{type:(p|s|b)}-{data:(h|b|c)}.svg", chartHandler(db))
 						//r.Get("/{b:\\d+}-{e:\\d+}.png", svgChartHandler(db))
 					})
 				})
diff --git a/cmd/server/site.go b/cmd/server/site.go
new file mode 100644
index 0000000..36f890f
--- /dev/null
+++ b/cmd/server/site.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+	"net/http"
+	"strings"
+
+	"src.userspace.com.au/sws"
+)
+
+func handleSite(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		site, ok := ctx.Value("site").(*sws.Site)
+		if !ok {
+			httpError(w, 422, "no site in context")
+			return
+		}
+
+		payload := newTemplateData(r)
+		payload.Site = site
+
+		filter := createHitFilter(r)
+		payload.Begin, payload.End = extractTimeRange(r)
+		if payload.Begin == nil || payload.End == nil {
+			httpError(w, http.StatusBadRequest, "invalid time range")
+			return
+		}
+		filter["begin"] = *payload.Begin
+		filter["end"] = *payload.End
+
+		debug("filter", filter)
+
+		hits, err := db.GetHits(*site, filter)
+		if err != nil {
+			httpError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+
+		hitSet, err := sws.NewHitSet(sws.FromHits(hits))
+		if err != nil {
+			httpError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
+		if hitSet != nil {
+			hitSet.Fill(payload.Begin, payload.End)
+			hitSet.SortByDate()
+			payload.Hits = hitSet
+		}
+
+		if _, ok := filter["path"]; !ok {
+			if ps := sws.NewPageSet(hitSet); ps != nil {
+				ps.SortByHits()
+				payload.PageSet = ps
+			}
+		}
+		if _, ok := filter["country"]; !ok {
+			if cs := sws.NewCountrySet(hitSet); cs != nil {
+				cs.SortByHits()
+				payload.CountrySet = cs
+			}
+		}
+		if _, ok := filter["browser"]; !ok {
+			if bs := sws.NewBrowserSet(hitSet); bs != nil {
+				bs.SortByHits()
+				payload.BrowserSet = bs
+			}
+		}
+
+		if err := rndr.Render(w, "site", payload); err != nil {
+			httpError(w, 500, err.Error())
+			return
+		}
+	}
+}
+
+func handleSiteEdit(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		site, ok := ctx.Value("site").(*sws.Site)
+		if !ok {
+			httpError(w, 422, "no site in context")
+			return
+		}
+
+		if r.Method == "POST" {
+			site.Name = r.FormValue("name")
+			site.Description = r.FormValue("description")
+			site.Aliases = r.FormValue("aliases")
+
+			if errs := site.Validate(); len(errs) > 0 {
+				log("invalid site:", errs)
+				r = flashSet(r, flashError, strings.Join(errs, "<br>"))
+			} else if err := db.SaveSite(site); err != nil {
+				httpError(w, 500, err.Error())
+				return
+			}
+			r = flashSet(r, flashSuccess, "site updated")
+		}
+
+		payload := newTemplateData(r)
+		payload.Site = site
+
+		if err := rndr.Render(w, "site", payload); err != nil {
+			httpError(w, 500, err.Error())
+			return
+		}
+	}
+}
diff --git a/cmd/server/sites.go b/cmd/server/sites.go
index 61857a3..cafe788 100644
--- a/cmd/server/sites.go
+++ b/cmd/server/sites.go
@@ -40,89 +40,3 @@ func handleSites(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
 		}
 	}
 }
-
-func handleSite(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		site, ok := ctx.Value("site").(*sws.Site)
-		if !ok {
-			httpError(w, 422, "no site in context")
-			return
-		}
-
-		payload := newTemplateData(r)
-		payload.Site = site
-
-		begin, end := extractTimeRange(r)
-		if begin == nil || end == nil {
-			httpError(w, http.StatusBadRequest, "invalid time range")
-			return
-		}
-		payload.Begin = *begin
-		payload.End = *end
-		debug("begin", *begin)
-		debug("end", *end)
-
-		hits, err := db.GetHits(*site, map[string]interface{}{
-			"begin": *begin,
-			"end":   *end,
-		})
-		if err != nil {
-			httpError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-
-		hitSet, err := sws.NewHitSet(sws.FromHits(hits))
-		if err != nil {
-			httpError(w, http.StatusInternalServerError, err.Error())
-			return
-		}
-		if hitSet != nil {
-			hitSet.Fill(begin, end)
-			hitSet.SortByDate()
-			if err := expandPayload(hitSet, payload); err != nil {
-				httpError(w, http.StatusInternalServerError, err.Error())
-				return
-			}
-		}
-
-		if err := rndr.Render(w, "site", payload); err != nil {
-			httpError(w, 500, err.Error())
-			return
-		}
-	}
-}
-
-func handleSiteEdit(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		site, ok := ctx.Value("site").(*sws.Site)
-		if !ok {
-			httpError(w, 422, "no site in context")
-			return
-		}
-
-		if r.Method == "POST" {
-			site.Name = r.FormValue("name")
-			site.Description = r.FormValue("description")
-			site.Aliases = r.FormValue("aliases")
-
-			if errs := site.Validate(); len(errs) > 0 {
-				log("invalid site:", errs)
-				r = flashSet(r, flashError, strings.Join(errs, "<br>"))
-			} else if err := db.SaveSite(site); err != nil {
-				httpError(w, 500, err.Error())
-				return
-			}
-			r = flashSet(r, flashSuccess, "site updated")
-		}
-
-		payload := newTemplateData(r)
-		payload.Site = site
-
-		if err := rndr.Render(w, "site", payload); err != nil {
-			httpError(w, 500, err.Error())
-			return
-		}
-	}
-}