aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFelix Hanley <felix@userspace.com.au>2020-02-19 22:49:55 +0000
committerFelix Hanley <felix@userspace.com.au>2020-02-19 22:49:55 +0000
commitd17450c945cd859ee5839802a399dfb9f1e54bfa (patch)
treec29320829d63d3715226f17c2c4dd14212bf3dcb
parentc664ab56a4726690ed233a0d1a98aefd1d6a5ac9 (diff)
downloadsws-d17450c945cd859ee5839802a399dfb9f1e54bfa.tar.gz
sws-d17450c945cd859ee5839802a399dfb9f1e54bfa.tar.bz2
Outsource UA detection
-rw-r--r--browser.go35
-rw-r--r--charts.go14
-rw-r--r--cmd/server/sites.go18
-rw-r--r--go.mod1
-rw-r--r--go.sum2
-rw-r--r--templates/charts.tmpl13
-rw-r--r--templates/site.tmpl14
-rw-r--r--user_agent.go78
8 files changed, 99 insertions, 76 deletions
diff --git a/browser.go b/browser.go
deleted file mode 100644
index b843a8b..0000000
--- a/browser.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package sws
-
-import (
- "time"
-)
-
-type Browser struct {
- Name string
- UserAgent string
- LastSeenAt time.Time
- Engine string
- Count int
-}
-
-func BrowsersFromHits(hits []*Hit) map[string]*Browser {
- out := make(map[string]*Browser)
- for _, h := range hits {
- if h.UserAgentHash != nil {
- b, ok := out[*h.UserAgentHash]
- if !ok {
- b = &Browser{
- // TODO name
- UserAgent: h.UserAgent.Name,
- LastSeenAt: h.CreatedAt,
- }
- }
- if b.LastSeenAt.Before(h.CreatedAt) {
- b.LastSeenAt = h.CreatedAt
- }
- b.Count++
- out[*h.UserAgentHash] = b
- }
- }
- return out
-}
diff --git a/charts.go b/charts.go
index 843bd41..1ca245b 100644
--- a/charts.go
+++ b/charts.go
@@ -3,8 +3,6 @@ package sws
import (
"fmt"
"io"
- "strconv"
- "strings"
"time"
gochart "github.com/wcharczuk/go-chart"
@@ -134,15 +132,3 @@ func HitChartSVG(w io.Writer, data TimeBuckets, d time.Duration) error {
graph.Render(gochart.SVG, w)
return nil
}
-
-func HitChartHTML(w io.Writer, data TimeBuckets, d time.Duration) (int, error) {
- var out strings.Builder
- out.WriteString(`<div class="chart">`)
- for _, b := range data.Buckets {
- h := (b.Count / data.CountMax) * 100
- toolTip := fmt.Sprintf("%s: %d hits", b.Time, b.Count)
- out.WriteString(`<div class="bar" title="` + toolTip + `" style="height:` + strconv.Itoa(h) + `" />`)
- }
- out.WriteString(`</div>`)
- return fmt.Fprintf(w, out.String())
-}
diff --git a/cmd/server/sites.go b/cmd/server/sites.go
index 7074670..59bb1f0 100644
--- a/cmd/server/sites.go
+++ b/cmd/server/sites.go
@@ -51,21 +51,21 @@ func handleSite(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
}
pages := sws.PagesFromHits(hits)
- browsers := sws.BrowsersFromHits(hits)
+ userAgents := sws.UserAgentsFromHits(hits)
buckets := sws.HitsToTimeBuckets(hits, time.Hour)
buckets.Fill(begin, end)
payload := struct {
- Site *sws.Site
- Pages map[string]*sws.Page
- Browsers map[string]*sws.Browser
- Hits sws.TimeBuckets
+ Site *sws.Site
+ Pages map[string]*sws.Page
+ UserAgents map[string]*sws.UserAgent
+ Hits sws.TimeBuckets
}{
- Site: site,
- Pages: pages,
- Browsers: browsers,
- Hits: buckets,
+ Site: site,
+ Pages: pages,
+ UserAgents: userAgents,
+ Hits: buckets,
}
if err := rndr.Render(w, "site", payload); err != nil {
log(err)
diff --git a/go.mod b/go.mod
index 7d635f8..e8e98e5 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
github.com/jackc/pgx v3.6.2+incompatible
github.com/jmoiron/sqlx v1.2.0
github.com/mattn/go-sqlite3 v2.0.3+incompatible
+ github.com/mssola/user_agent v0.5.1
github.com/pkg/errors v0.9.1 // indirect
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 // indirect
github.com/speps/go-hashids v2.0.0+incompatible
diff --git a/go.sum b/go.sum
index 215df5f..64e4659 100644
--- a/go.sum
+++ b/go.sum
@@ -24,6 +24,8 @@ github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK86
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mssola/user_agent v0.5.1 h1:sJUCUozh+j7c0dR2zMIUX5aJjoY/TNo/gXiNujoH5oY=
+github.com/mssola/user_agent v0.5.1/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 h1:Pm6R878vxWWWR+Sa3ppsLce/Zq+JNTs6aVvRu13jv9A=
diff --git a/templates/charts.tmpl b/templates/charts.tmpl
index aa4ca65..f864263 100644
--- a/templates/charts.tmpl
+++ b/templates/charts.tmpl
@@ -1,5 +1,16 @@
{{ define "barChart" }}
- <ul class="chart">
+ <ul class="chart bar">
+ {{ $max := .CountMax }}
+ {{ range .Buckets }}
+ <li class="slot{{ if eq .Time.Hour 0 }} midnight{{ end }}" data-date="{{ .Time|timeHour }}" data-count="{{ .Count }}" data-percent="{{ percent .Count $max }}">
+ <div class="bar" style="height:{{ percent .Count $max }}%" />
+ </li>
+ {{ end }}
+ </ul>
+{{ end }}
+
+{{ define "stackedBarChart" }}
+ <ul class="chart bar stacked">
{{ $max := .CountMax }}
{{ range .Buckets }}
<li class="slot{{ if eq .Time.Hour 0 }} midnight{{ end }}" data-date="{{ .Time|timeHour }}" data-count="{{ .Count }}" data-percent="{{ percent .Count $max }}">
diff --git a/templates/site.tmpl b/templates/site.tmpl
index 6889690..bf1a8af 100644
--- a/templates/site.tmpl
+++ b/templates/site.tmpl
@@ -15,10 +15,10 @@
{{ template "pageForList" . }}
{{ end }}
</ul>
- <h2>Browsers</h2>
- <ul class="browsers">
- {{ range .Browsers }}
- {{ template "browserForList" . }}
+ <h2>User Agents</h2>
+ <ul class="agents">
+ {{ range .UserAgents }}
+ {{ template "uaForList" . }}
{{ end }}
</ul>
</main>
@@ -33,10 +33,10 @@
</li>
{{ end }}
-{{ define "browserForList" }}
+{{ define "uaForList" }}
<li>
- <h4 class="browser">{{ .Name }}</h4>
- <span class="user-agent">{{ .UserAgent }}</span>
+ <h4 class="engine">{{ .Browser }}</h4>
+ <span class="user-agent">{{ .Name }}</span>
<span class="last-seen">{{ .LastSeenAt|timeLong }}</span>
<span class="count">{{ .Count }}</span>
</li>
diff --git a/user_agent.go b/user_agent.go
index f65f45c..a259538 100644
--- a/user_agent.go
+++ b/user_agent.go
@@ -5,38 +5,96 @@ import (
"fmt"
"net/http"
"regexp"
+ "strings"
"time"
-)
-var botRegex = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)")
-var botFromSiteRegexp = regexp.MustCompile("http[s]?://.+\\.\\w+")
+ detector "github.com/mssola/user_agent"
+)
+// UserAgent of a hit.
type UserAgent struct {
Hash string `json:"hash"`
Name string `json:"name"`
LastSeenAt time.Time `json:"last_seen_at" db:"last_seen_at"`
+ Count int
+
+ ua *detector.UserAgent
}
-func (ua UserAgent) Bot() bool {
- // TODO a little naive ATM
- return botRegex.MatchString(ua.Name) || botFromSiteRegexp.MatchString(ua.Name)
+var (
+ reBotWord, reBotSite *regexp.Regexp
+)
+
+type browserMatcher func(string) (string, bool)
+
+func init() {
+ reBotWord = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)")
+ reBotSite = regexp.MustCompile("http[s]?://.+\\.\\w+")
}
+// UserAgentHash is the UA key.
func UserAgentHash(s string) string {
return fmt.Sprintf("%x", sha1.Sum([]byte(s)))
}
+// UserAgentFromRequest extracts a UA from a request.
func UserAgentFromRequest(r *http.Request) (*UserAgent, error) {
q := r.URL.Query()
- agent := q.Get("u")
- if agent == "" {
- return nil, nil
+ ua := q.Get("u")
+ if ua == "" {
+ ua = r.UserAgent()
}
- ua := r.UserAgent()
return &UserAgent{
Name: ua,
LastSeenAt: time.Now(),
Hash: UserAgentHash(ua),
+ ua: detector.New(ua),
}, nil
}
+
+// UserAgentsFromHits collects the browsers from provided hits.
+func UserAgentsFromHits(hits []*Hit) map[string]*UserAgent {
+ out := make(map[string]*UserAgent)
+ for _, h := range hits {
+ if h.UserAgentHash != nil {
+ b, ok := out[*h.UserAgentHash]
+ if !ok {
+ b = &UserAgent{
+ Name: h.UserAgent.Name,
+ LastSeenAt: h.CreatedAt,
+ ua: detector.New(h.UserAgent.Name),
+ }
+ }
+ if b.LastSeenAt.Before(h.CreatedAt) {
+ b.LastSeenAt = h.CreatedAt
+ }
+ b.Count++
+ out[*h.UserAgentHash] = b
+ }
+ }
+ return out
+}
+
+func (ua UserAgent) IsBot() bool {
+ return ua.ua.Bot()
+}
+
+func (ua UserAgent) IsMobile() bool {
+ //return ua.ua.Mobile()
+ return strings.Contains(ua.Name, "Mobi")
+}
+
+func (ua UserAgent) Platform() string {
+ return ua.ua.Platform()
+}
+
+func (ua UserAgent) Browser() string {
+ n, _ := ua.ua.Browser()
+ return n
+}
+
+func (ua UserAgent) BrowserVersion() string {
+ _, v := ua.ua.Browser()
+ return v
+}