Outsource UA detection

8 files changed, 99 insertions, 76 deletions

diff --git a/browser.go b/browser.go
deleted file mode 100644
index b843a8b..0000000
--- a/browser.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package sws
-
-import (
-	"time"
-)
-
-type Browser struct {
-	Name       string
-	UserAgent  string
-	LastSeenAt time.Time
-	Engine     string
-	Count      int
-}
-
-func BrowsersFromHits(hits []*Hit) map[string]*Browser {
-	out := make(map[string]*Browser)
-	for _, h := range hits {
-		if h.UserAgentHash != nil {
-			b, ok := out[*h.UserAgentHash]
-			if !ok {
-				b = &Browser{
-					// TODO name
-					UserAgent:  h.UserAgent.Name,
-					LastSeenAt: h.CreatedAt,
-				}
-			}
-			if b.LastSeenAt.Before(h.CreatedAt) {
-				b.LastSeenAt = h.CreatedAt
-			}
-			b.Count++
-			out[*h.UserAgentHash] = b
-		}
-	}
-	return out
-}
diff --git a/charts.go b/charts.go
index 843bd41..1ca245b 100644
--- a/charts.go
+++ b/charts.go
@@ -3,8 +3,6 @@ package sws
 import (
 	"fmt"
 	"io"
-	"strconv"
-	"strings"
 	"time"
 
 	gochart "github.com/wcharczuk/go-chart"
@@ -134,15 +132,3 @@ func HitChartSVG(w io.Writer, data TimeBuckets, d time.Duration) error {
 	graph.Render(gochart.SVG, w)
 	return nil
 }
-
-func HitChartHTML(w io.Writer, data TimeBuckets, d time.Duration) (int, error) {
-	var out strings.Builder
-	out.WriteString(`<div class="chart">`)
-	for _, b := range data.Buckets {
-		h := (b.Count / data.CountMax) * 100
-		toolTip := fmt.Sprintf("%s: %d hits", b.Time, b.Count)
-		out.WriteString(`<div class="bar" title="` + toolTip + `" style="height:` + strconv.Itoa(h) + `" />`)
-	}
-	out.WriteString(`</div>`)
-	return fmt.Fprintf(w, out.String())
-}
diff --git a/cmd/server/sites.go b/cmd/server/sites.go
index 7074670..59bb1f0 100644
--- a/cmd/server/sites.go
+++ b/cmd/server/sites.go
@@ -51,21 +51,21 @@ func handleSite(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
 		}
 
 		pages := sws.PagesFromHits(hits)
-		browsers := sws.BrowsersFromHits(hits)
+		userAgents := sws.UserAgentsFromHits(hits)
 
 		buckets := sws.HitsToTimeBuckets(hits, time.Hour)
 		buckets.Fill(begin, end)
 
 		payload := struct {
-			Site     *sws.Site
-			Pages    map[string]*sws.Page
-			Browsers map[string]*sws.Browser
-			Hits     sws.TimeBuckets
+			Site       *sws.Site
+			Pages      map[string]*sws.Page
+			UserAgents map[string]*sws.UserAgent
+			Hits       sws.TimeBuckets
 		}{
-			Site:     site,
-			Pages:    pages,
-			Browsers: browsers,
-			Hits:     buckets,
+			Site:       site,
+			Pages:      pages,
+			UserAgents: userAgents,
+			Hits:       buckets,
 		}
 		if err := rndr.Render(w, "site", payload); err != nil {
 			log(err)
diff --git a/go.mod b/go.mod
index 7d635f8..e8e98e5 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/jackc/pgx v3.6.2+incompatible
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	github.com/mssola/user_agent v0.5.1
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 // indirect
 	github.com/speps/go-hashids v2.0.0+incompatible
diff --git a/go.sum b/go.sum
index 215df5f..64e4659 100644
--- a/go.sum
+++ b/go.sum
@@ -24,6 +24,8 @@ github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK86
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mssola/user_agent v0.5.1 h1:sJUCUozh+j7c0dR2zMIUX5aJjoY/TNo/gXiNujoH5oY=
+github.com/mssola/user_agent v0.5.1/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 h1:Pm6R878vxWWWR+Sa3ppsLce/Zq+JNTs6aVvRu13jv9A=
diff --git a/templates/charts.tmpl b/templates/charts.tmpl
index aa4ca65..f864263 100644
--- a/templates/charts.tmpl
+++ b/templates/charts.tmpl
@@ -1,5 +1,16 @@
 {{ define "barChart" }}
-	<ul class="chart">
+	<ul class="chart bar">
+		{{ $max := .CountMax }}
+		{{ range .Buckets }}
+			<li class="slot{{ if eq .Time.Hour 0 }} midnight{{ end }}" data-date="{{ .Time|timeHour }}" data-count="{{ .Count }}" data-percent="{{ percent .Count $max }}">
+				<div class="bar" style="height:{{ percent .Count $max }}%" />
+			</li>
+		{{ end }}
+	</ul>
+{{ end }}
+
+{{ define "stackedBarChart" }}
+	<ul class="chart bar stacked">
 		{{ $max := .CountMax }}
 		{{ range .Buckets }}
 			<li class="slot{{ if eq .Time.Hour 0 }} midnight{{ end }}" data-date="{{ .Time|timeHour }}" data-count="{{ .Count }}" data-percent="{{ percent .Count $max }}">
diff --git a/templates/site.tmpl b/templates/site.tmpl
index 6889690..bf1a8af 100644
--- a/templates/site.tmpl
+++ b/templates/site.tmpl
@@ -15,10 +15,10 @@
 				{{ template "pageForList" . }}
 			{{ end }}
 		</ul>
-		<h2>Browsers</h2>
-		<ul class="browsers">
-			{{ range .Browsers }}
-				{{ template "browserForList" . }}
+		<h2>User Agents</h2>
+		<ul class="agents">
+			{{ range .UserAgents }}
+				{{ template "uaForList" . }}
 			{{ end }}
 		</ul>
 	</main>
@@ -33,10 +33,10 @@
 </li>
 {{ end }}
 
-{{ define "browserForList" }}
+{{ define "uaForList" }}
 <li>
-	<h4 class="browser">{{ .Name }}</h4>
-	<span class="user-agent">{{ .UserAgent }}</span>
+	<h4 class="engine">{{ .Browser }}</h4>
+	<span class="user-agent">{{ .Name }}</span>
 	<span class="last-seen">{{ .LastSeenAt|timeLong }}</span>
 	<span class="count">{{ .Count }}</span>
 </li>
diff --git a/user_agent.go b/user_agent.go
index f65f45c..a259538 100644
--- a/user_agent.go
+++ b/user_agent.go
@@ -5,38 +5,96 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 	"time"
-)
 
-var botRegex = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)")
-var botFromSiteRegexp = regexp.MustCompile("http[s]?://.+\\.\\w+")
+	detector "github.com/mssola/user_agent"
+)
 
+// UserAgent of a hit.
 type UserAgent struct {
 	Hash       string    `json:"hash"`
 	Name       string    `json:"name"`
 	LastSeenAt time.Time `json:"last_seen_at" db:"last_seen_at"`
+	Count      int
+
+	ua *detector.UserAgent
 }
 
-func (ua UserAgent) Bot() bool {
-	// TODO a little naive ATM
-	return botRegex.MatchString(ua.Name) || botFromSiteRegexp.MatchString(ua.Name)
+var (
+	reBotWord, reBotSite *regexp.Regexp
+)
+
+type browserMatcher func(string) (string, bool)
+
+func init() {
+	reBotWord = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)")
+	reBotSite = regexp.MustCompile("http[s]?://.+\\.\\w+")
 }
 
+// UserAgentHash is the UA key.
 func UserAgentHash(s string) string {
 	return fmt.Sprintf("%x", sha1.Sum([]byte(s)))
 }
 
+// UserAgentFromRequest extracts a UA from a request.
 func UserAgentFromRequest(r *http.Request) (*UserAgent, error) {
 	q := r.URL.Query()
-	agent := q.Get("u")
-	if agent == "" {
-		return nil, nil
+	ua := q.Get("u")
+	if ua == "" {
+		ua = r.UserAgent()
 	}
-	ua := r.UserAgent()
 
 	return &UserAgent{
 		Name:       ua,
 		LastSeenAt: time.Now(),
 		Hash:       UserAgentHash(ua),
+		ua:         detector.New(ua),
 	}, nil
 }
+
+// UserAgentsFromHits collects the browsers from provided hits.
+func UserAgentsFromHits(hits []*Hit) map[string]*UserAgent {
+	out := make(map[string]*UserAgent)
+	for _, h := range hits {
+		if h.UserAgentHash != nil {
+			b, ok := out[*h.UserAgentHash]
+			if !ok {
+				b = &UserAgent{
+					Name:       h.UserAgent.Name,
+					LastSeenAt: h.CreatedAt,
+					ua:         detector.New(h.UserAgent.Name),
+				}
+			}
+			if b.LastSeenAt.Before(h.CreatedAt) {
+				b.LastSeenAt = h.CreatedAt
+			}
+			b.Count++
+			out[*h.UserAgentHash] = b
+		}
+	}
+	return out
+}
+
+func (ua UserAgent) IsBot() bool {
+	return ua.ua.Bot()
+}
+
+func (ua UserAgent) IsMobile() bool {
+	//return ua.ua.Mobile()
+	return strings.Contains(ua.Name, "Mobi")
+}
+
+func (ua UserAgent) Platform() string {
+	return ua.ua.Platform()
+}
+
+func (ua UserAgent) Browser() string {
+	n, _ := ua.ua.Browser()
+	return n
+}
+
+func (ua UserAgent) BrowserVersion() string {
+	_, v := ua.ua.Browser()
+	return v
+}