aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFelix Hanley <felix@userspace.com.au>2020-03-26 00:30:29 +0000
committerFelix Hanley <felix@userspace.com.au>2020-03-26 00:30:29 +0000
commitfed28a39d4465be006184f5b27f1acaea4796eb5 (patch)
tree7167dcef2952d1813e2b77edb509ec23c629bc22
parent8eb8549b9121e3708c697361c83f39c1dfc846b5 (diff)
downloadsws-fed28a39d4465be006184f5b27f1acaea4796eb5.tar.gz
sws-fed28a39d4465be006184f5b27f1acaea4796eb5.tar.bz2
Add filtering for browsers, countries and paths
-rw-r--r--cmd/cron/main.go24
-rw-r--r--cmd/server/charts.go36
-rw-r--r--cmd/server/country.go68
-rw-r--r--cmd/server/handlers.go55
-rw-r--r--cmd/server/helpers.go61
-rw-r--r--cmd/server/hits.go6
-rw-r--r--cmd/server/pages.go75
-rw-r--r--cmd/server/routes.go5
-rw-r--r--cmd/server/site.go108
-rw-r--r--cmd/server/sites.go86
-rw-r--r--country_codes.go2
-rw-r--r--hit.go13
-rw-r--r--page_set.go6
-rw-r--r--sql/sqlite3/06_page_filtering.sql5
-rw-r--r--store/sqlite3.go44
-rw-r--r--tmpl/filter.tmpl (renamed from tmpl/timerange.tmpl)11
-rw-r--r--tmpl/hitView.tmpl23
-rw-r--r--tmpl/site.tmpl4
-rw-r--r--tmpl/worldMap.tmpl2
-rw-r--r--user_agent.go47
20 files changed, 298 insertions, 383 deletions
diff --git a/cmd/cron/main.go b/cmd/cron/main.go
index af1e734..f9a6b3f 100644
--- a/cmd/cron/main.go
+++ b/cmd/cron/main.go
@@ -12,6 +12,7 @@ import (
_ "github.com/jackc/pgx/stdlib"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
+ detector "github.com/mssola/user_agent"
"src.userspace.com.au/sws"
"src.userspace.com.au/sws/store"
)
@@ -97,6 +98,8 @@ func main() {
panic(err)
}
+ seenUAs := make(map[string]bool)
+
toUpdate := make([]*sws.Hit, 0)
log("updating country code")
err = st.HitCursor(func(h *sws.Hit) error {
@@ -114,6 +117,27 @@ func main() {
h.CountryCode = cc
toUpdate = append(toUpdate, h)
}
+
+ // Populate user agent
+ if h.UserAgent != nil {
+ log("hit ID", *h.ID)
+ ua := h.UserAgent
+ if ok := seenUAs[ua.Hash]; ok {
+ return nil
+ }
+ det := detector.New(ua.Name)
+ browser, version := det.Browser()
+
+ h.UserAgent.Browser = browser
+ h.UserAgent.Platform = det.Platform()
+ h.UserAgent.Version = version
+ h.UserAgent.Bot = det.Bot()
+ h.UserAgent.Mobile = strings.Contains(ua.Name, "Mobi") || det.Mobile()
+
+ seenUAs[ua.Hash] = true
+
+ toUpdate = append(toUpdate, h)
+ }
return nil
})
if err != nil {
diff --git a/cmd/server/charts.go b/cmd/server/charts.go
index fce5c86..a8e9238 100644
--- a/cmd/server/charts.go
+++ b/cmd/server/charts.go
@@ -4,7 +4,6 @@ import (
"crypto/sha1"
"fmt"
"net/http"
- "strconv"
"strings"
"time"
@@ -22,23 +21,9 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
return
}
- chartType := chi.URLParam(r, "type")
- dataType := chi.URLParam(r, "data")
- beginSecs, err := strconv.ParseInt(chi.URLParam(r, "begin"), 10, 64)
- if err != nil {
- httpError(w, http.StatusNotFound, err.Error())
- return
- }
- endSecs, err := strconv.ParseInt(chi.URLParam(r, "end"), 10, 64)
- if err != nil {
- httpError(w, http.StatusNotFound, err.Error())
- return
- }
- begin := time.Unix(beginSecs, 0)
- end := time.Unix(endSecs, 0)
-
var b strings.Builder
b.WriteString(r.URL.Path)
+ b.WriteString(r.URL.Query().Encode())
// FIXME
b.WriteString(time.Now().Truncate(30 * time.Minute).String())
etag := fmt.Sprintf(`"%x"`, sha1.Sum([]byte(b.String())))
@@ -50,10 +35,18 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
}
}
- filter := map[string]interface{}{
- "begin": begin,
- "end": end,
+ filter := createHitFilter(r)
+ begin, end := extractTimeRange(r)
+ if begin == nil || end == nil {
+ log("charts: empty times")
+ httpError(w, http.StatusNotFound, "not found")
+ return
}
+ filter["begin"] = *begin
+ filter["end"] = *end
+
+ chartType := chi.URLParam(r, "type")
+ dataType := chi.URLParam(r, "data")
hits, err := db.GetHits(*site, filter)
if err != nil {
@@ -71,7 +64,7 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
return
}
- hitSet.Fill(&begin, &end)
+ hitSet.Fill(begin, end)
hitSet.SortByDate()
w.Header().Set("Etag", etag)
@@ -96,6 +89,9 @@ func chartHandler(db sws.HitStore) http.HandlerFunc {
}
case "b":
browsers := sws.NewBrowserSet(hitSet)
+ if browsers == nil {
+ return
+ }
browsers.SortByHits()
w.Header().Set("Content-Type", "image/svg+xml")
switch chartType {
diff --git a/cmd/server/country.go b/cmd/server/country.go
deleted file mode 100644
index bb4b22e..0000000
--- a/cmd/server/country.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package main
-
-import (
- "net/http"
-
- "src.userspace.com.au/sws"
-)
-
-func handleCountries(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- ctx := r.Context()
- site, ok := ctx.Value("site").(*sws.Site)
- if !ok {
- httpError(w, http.StatusBadRequest, "no site in context")
- return
- }
-
- payload := newTemplateData(r)
- payload.Site = site
-
- begin, end := extractTimeRange(r)
- if begin == nil || end == nil {
- httpError(w, http.StatusBadRequest, "invalid time range")
- return
- }
- payload.Begin = *begin
- payload.End = *end
- debug("begin", *begin)
- debug("end", *end)
-
- filter := map[string]interface{}{
- "begin": *begin,
- "end": *end,
- }
-
- q := r.URL.Query()
-
- cc := q.Get("country")
- if cc != "" {
- filter["countryCode"] = cc
- }
-
- hits, err := db.GetHits(*site, filter)
- hitSet, err := sws.NewHitSet(sws.FromHits(hits))
- if err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
- hitSet.Fill(begin, end)
- hitSet.SortByDate()
- if err := expandPayload(hitSet, payload); err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
-
- // Single or multiple paths
- if cc == "" {
- countrySet := sws.NewCountrySet(hitSet)
- countrySet.SortByHits()
- payload.CountrySet = countrySet
- }
-
- if err := rndr.Render(w, "site", payload); err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
- }
-}
diff --git a/cmd/server/handlers.go b/cmd/server/handlers.go
index 985955a..00de42b 100644
--- a/cmd/server/handlers.go
+++ b/cmd/server/handlers.go
@@ -3,6 +3,7 @@ package main
import (
"html/template"
"net/http"
+ "net/url"
"strings"
"time"
@@ -15,17 +16,24 @@ type templateData struct {
Endpoint string
User *sws.User
Flash template.HTML
- Begin time.Time
- End time.Time
+ Begin *time.Time
+ End *time.Time
+
+ Query url.Values
Site *sws.Site
Sites []*sws.Site
+ Hits *sws.HitSet
PageSet *sws.PageSet
- Page *sws.Page
- Browsers *sws.BrowserSet
+ BrowserSet *sws.BrowserSet
ReferrerSet *sws.ReferrerSet
CountrySet *sws.CountrySet
- Hits *sws.HitSet
+}
+
+func (td templateData) QuerySetEncode(k, v string) template.URL {
+ qs, _ := url.ParseQuery(td.Query.Encode())
+ qs.Set(k, v)
+ return template.URL(qs.Encode())
}
func newTemplateData(r *http.Request) *templateData {
@@ -34,24 +42,29 @@ func newTemplateData(r *http.Request) *templateData {
Payload: "//" + domain + "/sws.js",
Endpoint: "//" + domain + "/sws.gif",
}
- if r != nil {
- flashes := flashGet(r)
- var flash strings.Builder
- for _, f := range flashes {
- flash.WriteString(`<span class="notification is-`)
- flash.WriteString(string(f.Level))
- flash.WriteString(`">`)
- flash.WriteString(f.Message)
- flash.WriteString("</span>")
- }
- if len(flashes) > 0 {
- out.Flash = template.HTML(flash.String())
- }
+ if r == nil {
+ return out
+ }
- if user := r.Context().Value("user"); user != nil {
- out.User = user.(*sws.User)
- }
+ flashes := flashGet(r)
+ var flash strings.Builder
+ for _, f := range flashes {
+ flash.WriteString(`<span class="notification is-`)
+ flash.WriteString(string(f.Level))
+ flash.WriteString(`">`)
+ flash.WriteString(f.Message)
+ flash.WriteString("</span>")
}
+ if len(flashes) > 0 {
+ out.Flash = template.HTML(flash.String())
+ }
+
+ if user := r.Context().Value("user"); user != nil {
+ out.User = user.(*sws.User)
+ }
+
+ out.Query = r.URL.Query()
+
return out
}
diff --git a/cmd/server/helpers.go b/cmd/server/helpers.go
index ff7fd5c..ff19e2b 100644
--- a/cmd/server/helpers.go
+++ b/cmd/server/helpers.go
@@ -12,13 +12,15 @@ import (
)
var funcMap = template.FuncMap{
- "piechart": func(siteID int, dataType string, begin, end time.Time) string {
- return fmt.Sprintf("/sites/%d/charts/p-%s-%d-%d.svg", siteID, dataType, begin.Unix(), end.Unix())
+ "piechart": func(dataType string, pl templateData) string {
+ pl.Query.Add("begin", strconv.Itoa(int(pl.Begin.Unix())))
+ pl.Query.Add("end", strconv.Itoa(int(pl.End.Unix())))
+ return fmt.Sprintf("/sites/%d/charts/p-%s.svg?%s", *pl.Site.ID, dataType, pl.Query.Encode())
},
"sparkline": func(id int) string {
now := time.Now().Truncate(time.Hour)
then := now.Add(-168 * time.Hour) // 7 days
- return fmt.Sprintf("/sites/%d/charts/s-h-%d-%d.svg", id, then.Unix(), now.Unix())
+ return fmt.Sprintf("/sites/%d/charts/s-h.svg?begin=%d&end=%d", id, then.Unix(), now.Unix())
},
"tz": func(s string, t time.Time) time.Time {
tz, _ := time.LoadLocation(s)
@@ -61,9 +63,9 @@ var funcMap = template.FuncMap{
"timeRFC": func(t time.Time) string {
return t.Format("15:04")
},
- "datetimeRelative": func(d string) int64 {
+ "datetimeRelative": func(d string) string {
dur, _ := time.ParseDuration(d)
- return time.Now().Add(dur).Unix()
+ return strconv.Itoa(int(time.Now().Add(dur).Unix()))
},
"percent": func(a, b int) float64 {
return (float64(a) / float64(b)) * 100
@@ -82,6 +84,32 @@ func httpError(w http.ResponseWriter, code int, msg string) {
http.Error(w, http.StatusText(code), code)
}
+func createHitFilter(r *http.Request) map[string]interface{} {
+ filter := make(map[string]interface{})
+
+ q := r.URL.Query()
+
+ path := q.Get("path")
+ if path != "" {
+ filter["path"] = path
+ }
+ country := q.Get("country")
+ if country != "" {
+ filter["country_code"] = country
+ }
+ browser := q.Get("browser")
+ if browser != "" {
+ filter["ua.browser"] = browser
+ }
+ if bots := q.Get("bots"); bots != "" {
+ filter["ua.bot"] = (bots == "1")
+ }
+ if mobile := q.Get("mobile"); mobile != "" {
+ filter["ua.mobile"] = (mobile == "1")
+ }
+ return filter
+}
+
func extractTimeRange(r *http.Request) (*time.Time, *time.Time) {
// Default to 1 week ago
begin := timePtr(time.Now().Truncate(time.Hour).Add(-168 * time.Hour))
@@ -100,29 +128,6 @@ func extractTimeRange(r *http.Request) (*time.Time, *time.Time) {
return begin, end
}
-func expandPayload(hs *sws.HitSet, pl *templateData) error {
- pl.Hits = hs
-
- pageSet, err := sws.NewPageSet(hs)
- if err != nil {
- return err
- }
-
- if pageSet != nil {
- pageSet.SortByHits()
- pl.PageSet = pageSet
- }
- pl.Browsers = sws.NewBrowserSet(hs)
- pl.CountrySet = sws.NewCountrySet(hs)
-
- refSet := sws.NewReferrerSet(hs)
- if refSet != nil {
- refSet.SortByHits()
- pl.ReferrerSet = refSet
- }
- return nil
-}
-
func stringPtr(s string) *string {
return &s
}
diff --git a/cmd/server/hits.go b/cmd/server/hits.go
index 3a76bea..776e610 100644
--- a/cmd/server/hits.go
+++ b/cmd/server/hits.go
@@ -24,7 +24,7 @@ func handleHitCounter(db sws.CounterStore, mmdbPath string) http.HandlerFunc {
panic(err)
}
- cache, err := lru.New(100)
+ countryCache, err := lru.New(100)
if err != nil {
panic(err)
}
@@ -67,13 +67,13 @@ func handleHitCounter(db sws.CounterStore, mmdbPath string) http.HandlerFunc {
if err == nil && hit.Addr != "" {
var cc *string
- if v, ok := cache.Get(hit.Addr); ok {
+ if v, ok := countryCache.Get(hit.Addr); ok {
cc = v.(*string)
} else if mmdbPath != "" {
if cc, err = sws.FetchCountryCode(mmdbPath, hit.Addr); err != nil {
log("geoip lookup failed:", err)
}
- cache.Add(hit.Addr, cc)
+ countryCache.Add(hit.Addr, cc)
}
hit.CountryCode = cc
debug("geolocated:", hit.Addr, "to", *hit.CountryCode)
diff --git a/cmd/server/pages.go b/cmd/server/pages.go
deleted file mode 100644
index 702194e..0000000
--- a/cmd/server/pages.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package main
-
-import (
- "net/http"
-
- "src.userspace.com.au/sws"
-)
-
-func handlePages(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- ctx := r.Context()
- site, ok := ctx.Value("site").(*sws.Site)
- if !ok {
- httpError(w, http.StatusBadRequest, "no site in context")
- return
- }
-
- payload := newTemplateData(r)
- payload.Site = site
-
- begin, end := extractTimeRange(r)
- if begin == nil || end == nil {
- httpError(w, http.StatusBadRequest, "invalid time range")
- return
- }
- payload.Begin = *begin
- payload.End = *end
- debug("begin", *begin)
- debug("end", *end)
-
- filter := map[string]interface{}{
- "begin": *begin,
- "end": *end,
- }
-
- q := r.URL.Query()
-
- path := q.Get("path")
- if path != "" {
- filter["path"] = path
- }
-
- hits, err := db.GetHits(*site, filter)
- hitSet, err := sws.NewHitSet(sws.FromHits(hits))
- if err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
- hitSet.Fill(begin, end)
- hitSet.SortByDate()
-
- payload.Page = sws.NewPage(hitSet)
- payload.Hits = hitSet
- payload.Browsers = sws.NewBrowserSet(hitSet)
- payload.CountrySet = sws.NewCountrySet(hitSet)
- payload.ReferrerSet = sws.NewReferrerSet(hitSet)
-
- // Single or multiple paths
- if path == "" {
- pageSet, err := sws.NewPageSet(hitSet)
- if err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
-
- pageSet.SortByHits()
- payload.PageSet = pageSet
- }
-
- if err := rndr.Render(w, "pages", payload); err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
- }
-}
diff --git a/cmd/server/routes.go b/cmd/server/routes.go
index 0e6462b..ae018b6 100644
--- a/cmd/server/routes.go
+++ b/cmd/server/routes.go
@@ -27,7 +27,7 @@ func init() {
func createRouter(db sws.Store, mmdbPath string) (chi.Router, error) {
tmplsCommon := []string{"flash.tmpl", "navbar.tmpl"}
tmplsAuthed := append(tmplsCommon, []string{
- "layout.tmpl", "charts.tmpl", "timerange.tmpl", "hitView.tmpl",
+ "layout.tmpl", "charts.tmpl", "filter.tmpl", "hitView.tmpl",
}...)
tmplsPublic := append(tmplsCommon, "layout.tmpl")
@@ -139,11 +139,10 @@ func createRouter(db sws.Store, mmdbPath string) (chi.Router, error) {
r.Use(getSiteCtx(db))
r.Get("/", siteHandler)
r.Post("/", siteHandler)
- r.Get("/pages", handlePages(db, rndr))
r.Get("/edit", handleSiteEdit(db, rndr))
r.Route("/charts", func(r chi.Router) {
- r.Get("/{type:(p|s|b)}-{data:(h|b|c)}-{begin:\\d+}-{end:\\d+}.svg", chartHandler(db))
+ r.Get("/{type:(p|s|b)}-{data:(h|b|c)}.svg", chartHandler(db))
//r.Get("/{b:\\d+}-{e:\\d+}.png", svgChartHandler(db))
})
})
diff --git a/cmd/server/site.go b/cmd/server/site.go
new file mode 100644
index 0000000..36f890f
--- /dev/null
+++ b/cmd/server/site.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+ "net/http"
+ "strings"
+
+ "src.userspace.com.au/sws"
+)
+
+func handleSite(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
+ site, ok := ctx.Value("site").(*sws.Site)
+ if !ok {
+ httpError(w, 422, "no site in context")
+ return
+ }
+
+ payload := newTemplateData(r)
+ payload.Site = site
+
+ filter := createHitFilter(r)
+ payload.Begin, payload.End = extractTimeRange(r)
+ if payload.Begin == nil || payload.End == nil {
+ httpError(w, http.StatusBadRequest, "invalid time range")
+ return
+ }
+ filter["begin"] = *payload.Begin
+ filter["end"] = *payload.End
+
+ debug("filter", filter)
+
+ hits, err := db.GetHits(*site, filter)
+ if err != nil {
+ httpError(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+
+ hitSet, err := sws.NewHitSet(sws.FromHits(hits))
+ if err != nil {
+ httpError(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+ if hitSet != nil {
+ hitSet.Fill(payload.Begin, payload.End)
+ hitSet.SortByDate()
+ payload.Hits = hitSet
+ }
+
+ if _, ok := filter["path"]; !ok {
+ if ps := sws.NewPageSet(hitSet); ps != nil {
+ ps.SortByHits()
+ payload.PageSet = ps
+ }
+ }
+ if _, ok := filter["country"]; !ok {
+ if cs := sws.NewCountrySet(hitSet); cs != nil {
+ cs.SortByHits()
+ payload.CountrySet = cs
+ }
+ }
+ if _, ok := filter["browser"]; !ok {
+ if bs := sws.NewBrowserSet(hitSet); bs != nil {
+ bs.SortByHits()
+ payload.BrowserSet = bs
+ }
+ }
+
+ if err := rndr.Render(w, "site", payload); err != nil {
+ httpError(w, 500, err.Error())
+ return
+ }
+ }
+}
+
+func handleSiteEdit(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
+ site, ok := ctx.Value("site").(*sws.Site)
+ if !ok {
+ httpError(w, 422, "no site in context")
+ return
+ }
+
+ if r.Method == "POST" {
+ site.Name = r.FormValue("name")
+ site.Description = r.FormValue("description")
+ site.Aliases = r.FormValue("aliases")
+
+ if errs := site.Validate(); len(errs) > 0 {
+ log("invalid site:", errs)
+ r = flashSet(r, flashError, strings.Join(errs, "<br>"))
+ } else if err := db.SaveSite(site); err != nil {
+ httpError(w, 500, err.Error())
+ return
+ }
+ r = flashSet(r, flashSuccess, "site updated")
+ }
+
+ payload := newTemplateData(r)
+ payload.Site = site
+
+ if err := rndr.Render(w, "site", payload); err != nil {
+ httpError(w, 500, err.Error())
+ return
+ }
+ }
+}
diff --git a/cmd/server/sites.go b/cmd/server/sites.go
index 61857a3..cafe788 100644
--- a/cmd/server/sites.go
+++ b/cmd/server/sites.go
@@ -40,89 +40,3 @@ func handleSites(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
}
}
}
-
-func handleSite(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- ctx := r.Context()
- site, ok := ctx.Value("site").(*sws.Site)
- if !ok {
- httpError(w, 422, "no site in context")
- return
- }
-
- payload := newTemplateData(r)
- payload.Site = site
-
- begin, end := extractTimeRange(r)
- if begin == nil || end == nil {
- httpError(w, http.StatusBadRequest, "invalid time range")
- return
- }
- payload.Begin = *begin
- payload.End = *end
- debug("begin", *begin)
- debug("end", *end)
-
- hits, err := db.GetHits(*site, map[string]interface{}{
- "begin": *begin,
- "end": *end,
- })
- if err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
-
- hitSet, err := sws.NewHitSet(sws.FromHits(hits))
- if err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
- if hitSet != nil {
- hitSet.Fill(begin, end)
- hitSet.SortByDate()
- if err := expandPayload(hitSet, payload); err != nil {
- httpError(w, http.StatusInternalServerError, err.Error())
- return
- }
- }
-
- if err := rndr.Render(w, "site", payload); err != nil {
- httpError(w, 500, err.Error())
- return
- }
- }
-}
-
-func handleSiteEdit(db sws.SiteStore, rndr Renderer) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- ctx := r.Context()
- site, ok := ctx.Value("site").(*sws.Site)
- if !ok {
- httpError(w, 422, "no site in context")
- return
- }
-
- if r.Method == "POST" {
- site.Name = r.FormValue("name")
- site.Description = r.FormValue("description")
- site.Aliases = r.FormValue("aliases")
-
- if errs := site.Validate(); len(errs) > 0 {
- log("invalid site:", errs)
- r = flashSet(r, flashError, strings.Join(errs, "<br>"))
- } else if err := db.SaveSite(site); err != nil {
- httpError(w, 500, err.Error())
- return
- }
- r = flashSet(r, flashSuccess, "site updated")
- }
-
- payload := newTemplateData(r)
- payload.Site = site
-
- if err := rndr.Render(w, "site", payload); err != nil {
- httpError(w, 500, err.Error())
- return
- }
- }
-}
diff --git a/country_codes.go b/country_codes.go
index 6f518ef..bbeb69d 100644
--- a/country_codes.go
+++ b/country_codes.go
@@ -235,7 +235,7 @@ var CountryCodes = map[string]string{
"UG": "Uganda",
"UA": "Ukraine",
"AE": "United Arab Emirates",
- "GB": "United Kingdom of Great Britain and Northern Ireland",
+ "GB": "United Kingdom", // of Great Britain and Northern Ireland",
"US": "United States of America",
"UM": "United States Minor Outlying Islands",
"UY": "Uruguay",
diff --git a/hit.go b/hit.go
index 1adeae8..97dffef 100644
--- a/hit.go
+++ b/hit.go
@@ -129,17 +129,8 @@ func HitFromRequest(r *http.Request) (*Hit, error) {
out.Referrer = &s
}
- agent := q.Get("u")
- if agent == "" {
- agent = r.UserAgent()
- }
- uaHash := UserAgentHash(agent)
- out.UserAgentHash = &uaHash
- out.UserAgent = &UserAgent{
- Hash: uaHash,
- Name: agent,
- LastSeenAt: time.Now(),
- }
+ out.UserAgent = UserAgentFromRequest(r)
+ out.UserAgentHash = &out.UserAgent.Hash
if view := q.Get("v"); view != "" {
out.ViewPort = &view
diff --git a/page_set.go b/page_set.go
index 043dc7f..ea0da5b 100644
--- a/page_set.go
+++ b/page_set.go
@@ -6,7 +6,7 @@ import (
type PageSet []*Page
-func NewPageSet(hs *HitSet) (*PageSet, error) {
+func NewPageSet(hs *HitSet) *PageSet {
tmp := make(map[string]*Page)
for _, h := range hs.Hits() {
if _, ok := tmp[h.Path]; ok {
@@ -29,7 +29,7 @@ func NewPageSet(hs *HitSet) (*PageSet, error) {
tmp[h.Path] = p
}
if len(tmp) < 1 {
- return nil, nil
+ return nil
}
out := make([]*Page, len(tmp))
i := 0
@@ -38,7 +38,7 @@ func NewPageSet(hs *HitSet) (*PageSet, error) {
i++
}
ps := PageSet(out)
- return &ps, nil
+ return &ps
}
func (ps *PageSet) Count() int {
diff --git a/sql/sqlite3/06_page_filtering.sql b/sql/sqlite3/06_page_filtering.sql
new file mode 100644
index 0000000..818efc7
--- /dev/null
+++ b/sql/sqlite3/06_page_filtering.sql
@@ -0,0 +1,5 @@
+alter table user_agents add column browser varchar not null default '';
+alter table user_agents add column platform varchar not null default '';
+alter table user_agents add column version varchar not null default '';
+alter table user_agents add column bot integer not null default 0;
+alter table user_agents add column mobile integer not null default 0;
diff --git a/store/sqlite3.go b/store/sqlite3.go
index 9525158..568d3fc 100644
--- a/store/sqlite3.go
+++ b/store/sqlite3.go
@@ -72,7 +72,7 @@ func (s *Sqlite3) GetHits(d sws.Site, filter map[string]interface{}) ([]*sws.Hit
filter = make(map[string]interface{})
}
- sql := stmts["hits"]
+ sql := stmts["hits"] + ` where h.site_id = :site_id`
filter["site_id"] = *d.ID
processFilter(&sql, filter)
@@ -93,10 +93,7 @@ func (s *Sqlite3) GetHits(d sws.Site, filter map[string]interface{}) ([]*sws.Hit
}
func (s *Sqlite3) HitCursor(f func(h *sws.Hit) error) error {
- sql := `select h.*,
-ua.hash as "ua.hash", ua.name as "ua.name", ua.last_seen_at as "ua.last_seen_at"
-from hits h
-join user_agents ua on h.user_agent_hash = ua.hash`
+ sql := stmts["hits"]
rows, err := s.db.Queryx(sql)
if err != nil {
@@ -175,9 +172,9 @@ var stmts = map[string]string{
"siteByID": `select * from sites where id = $1 limit 1`,
- "saveSite": `insert into sites (
-name, description, aliases, enabled, subdomains, ignore_ips, created_at, updated_at)
-values (:name, :description, :aliases, :enabled, :subdomains, :ignore_ips, date('now'), date('now'))
+ "saveSite": `insert into sites
+(id, name, description, aliases, enabled, subdomains, ignore_ips, created_at, updated_at)
+values (:id, :name, :description, :aliases, :enabled, :subdomains, :ignore_ips, date('now'), date('now'))
on conflict(id) do update set
name = :name,
description = :description,
@@ -189,21 +186,30 @@ updated_at = date('now')`,
where hash = $1 limit 1`,
"saveUserAgent": `insert into user_agents
-(hash, name, last_seen_at)
-values (:hash, :name, :last_seen_at)
-on conflict(hash) do update set last_seen_at = :last_seen_at`,
-
- "saveHit": `insert into hits (
-site_id, addr, scheme, host, path, query, title, referrer, user_agent_hash,
+(hash, name, last_seen_at, browser, platform, version, bot, mobile)
+values (:hash, :name, :last_seen_at, :browser, :platform, :version, :bot, :mobile)
+on conflict(hash) do update set
+last_seen_at = :last_seen_at,
+browser = :browser,
+platform = :platform,
+version = :version,
+bot = :bot,
+mobile = :mobile`,
+
+ "saveHit": `insert into hits
+(id, site_id, addr, scheme, host, path, query, title, referrer, user_agent_hash,
view_port, country_code, no_script, created_at)
-values (:site_id, :addr, :scheme, :host, :path, :query, :title, :referrer,
-:user_agent_hash, :view_port, :country_code, :no_script, :created_at)`,
+values (:id, :site_id, :addr, :scheme, :host, :path, :query, :title, :referrer,
+:user_agent_hash, :view_port, :country_code, :no_script, :created_at)
+on conflict(id) do nothing`,
+ // The explicit useragent stuff is to work around sqlx filling nested structs
"hits": `select h.*,
-ua.hash as "ua.hash", ua.name as "ua.name", ua.last_seen_at as "ua.last_seen_at"
+ua.hash as "ua.hash", ua.name as "ua.name", ua.last_seen_at as "ua.last_seen_at",
+ua.browser as "ua.browser", ua.platform as "ua.platform", ua.bot as "ua.bot",
+ua.mobile as "ua.mobile"
from hits h
-join user_agents ua on h.user_agent_hash = ua.hash
-where h.site_id = :site_id`,
+join user_agents ua on h.user_agent_hash = ua.hash`,
"userByEmail": `select id, email, first_name, last_name, pw_hash, pw_salt, enabled,
created_at, updated_at, last_login_at
diff --git a/tmpl/timerange.tmpl b/tmpl/filter.tmpl
index 09ea04f..420f74a 100644
--- a/tmpl/timerange.tmpl
+++ b/tmpl/filter.tmpl
@@ -1,4 +1,4 @@
-{{ define "timerange" }}
+{{ define "filter" }}
<div class="timerange">
<!--
<form>
@@ -33,8 +33,11 @@
</select>
</form>
-->
- <a href="?begin={{ datetimeRelative "-24h"}}">last day</a>
- <a href="?begin={{ datetimeRelative "-168h"}}">last 7 days</a>
- <a href="?begin={{ datetimeRelative "-720h"}}">last 30 days</a>
+ <a href="?{{ datetimeRelative "-24h" | .QuerySetEncode "begin" }}">last day</a>
+ <a href="?{{ datetimeRelative "-168h" | .QuerySetEncode "begin" }}">last 7 days</a>
+ <a href="?{{ datetimeRelative "-720h" | .QuerySetEncode "begin" }}">last 30 days</a>
+ <a href="?{{ .QuerySetEncode "bots" "0" }}">without bots</a>
+ <a href="?{{ .QuerySetEncode "bots" "" }}">with bots</a>
+ <a href="?{{ .QuerySetEncode "bots" "1" }}">only bots</a>
</div>
{{ end }}
diff --git a/tmpl/hitView.tmpl b/tmpl/hitView.tmpl
index b5a5d05..2636c5f 100644
--- a/tmpl/hitView.tmpl
+++ b/tmpl/hitView.tmpl
@@ -1,5 +1,6 @@
{{ define "hitView" }}
{{ $siteID := .Site.ID }}
+ {{ $payload := . }}
<section class="panel panel--wide">
<header class="panel__header">
<h3 class="panel__title">Hits</h3>
@@ -29,7 +30,7 @@
{{ if lt $i 10 }}
<tr>
<td class="details__name">
- <a href="/sites/{{ $siteID }}/pages?path={{ .Path }}">{{ .Path }}</a>
+ <a href="/sites/{{ $siteID }}?{{ $payload.QuerySetEncode "path" .Path }}">{{ .Path }}</a>
</td>
<td class="details__count"><span class="details__percent">{{ percent .Count $sum | round 1 }}%</span> ({{ .Count }})</td>
</tr>
@@ -54,7 +55,9 @@
<table class="table is-striped details details--countries">
{{ range .CountrySet }}
<tr>
- <td class="details__name">{{ .Name | countryName }}</td>
+ <td class="details__name">
+ <a href="/sites/{{ $siteID }}?{{ $payload.QuerySetEncode "country" .Name }}">{{ .Name | countryName }}</a>
+ </td>
<td class="details__count"><span class="details__percent">{{ percent .Count $sum | round 1 }}%</span> ({{ .Count }})</td>
</tr>
{{ end }}
@@ -90,16 +93,18 @@
<header class="panel__header">
<h3 class="panel__title">User agents</h3>
</header>
- {{ if .Browsers }}
+ {{ if .BrowserSet }}
<figure class="figure figure--graph">
- <img src="{{ piechart .Site.ID "b" .Begin .End }}" />
- {{ template "barChart" .Browsers }}
+ <img src="{{ piechart "b" . }}" />
+ {{ template "barChart" .BrowserSet }}
</figure>
- {{ $sum := .Browsers.YSum }}
+ {{ $sum := .BrowserSet.YSum }}
<table class="table is-striped details details--browsers">
- {{ range .Browsers }}
+ {{ range .BrowserSet }}
<tr>
- <td class="details__name">{{ .Name }}</td>
+ <td class="details__name">
+ <a href="/sites/{{ $siteID }}?{{ $payload.QuerySetEncode "browser" .Name }}">{{ .Name }}</a>
+ </td>
<td class="details__count"><span class="details__percent">{{ percent .Count $sum | round 1 }}%</span> ({{ .Count }})</td>
</tr>
{{ end }}
@@ -125,7 +130,7 @@
<!-- <span class="summary__comparison"></span> -->
</div>
{{ end }}
- {{ with .Browsers }}
+ {{ with .BrowserSet }}
<div class="summary">
<span class="summary__title">Total Browsers</span>
<span class="summary__count">{{ .Count }}</span>
diff --git a/tmpl/site.tmpl b/tmpl/site.tmpl
index d506eca..b2ef689 100644
--- a/tmpl/site.tmpl
+++ b/tmpl/site.tmpl
@@ -3,7 +3,7 @@
<header class="header--site">
{{ if .Site.ID }}
{{ with .Site }}
- <h1 class="title--site">Site summary for {{ .Name }}</h1>
+ <h1 class="title--site">Hits for {{ .Name }}</h1>
<span class="title__description--site">{{ .Description }}</span>
{{ end }}
{{ else }}
@@ -14,7 +14,7 @@
{{ template "siteSummary" . }}
{{ if .Site.ID }}
- {{ template "timerange" . }}
+ {{ template "filter" . }}
{{ end }}
{{ if .Hits }}
{{ template "hitView" . }}
diff --git a/tmpl/worldMap.tmpl b/tmpl/worldMap.tmpl
index 2da8c17..c476157 100644
--- a/tmpl/worldMap.tmpl
+++ b/tmpl/worldMap.tmpl
@@ -41,7 +41,7 @@
<style>
{{ $max := .YMax }}
{{ range .XSeries }}
- #{{ .Label }} { fill: hsla(230, {{ percent .Count $max }}%, 50%, 1) !important; }
+ #{{ .Label }} { fill: hsla(230, 50%, 50%, {{ percent .Count $max }}%) !important; }
{{ end }}
</style>
<defs
diff --git a/user_agent.go b/user_agent.go
index dfa9a28..e416e3b 100644
--- a/user_agent.go
+++ b/user_agent.go
@@ -16,8 +16,12 @@ type UserAgent struct {
Hash string `json:"hash"`
Name string `json:"name"`
LastSeenAt time.Time `json:"last_seen_at" db:"last_seen_at"`
+ Browser string `json:"browser"`
+ Platform string `json:"platform"`
+ Version string `json:"version"`
+ Bot bool `json:"bot"`
+ Mobile bool `json:"mobile"`
hitSet *HitSet
- ua *detector.UserAgent
}
var (
@@ -37,19 +41,27 @@ func UserAgentHash(s string) string {
}
// UserAgentFromRequest extracts a UA from a request.
-func UserAgentFromRequest(r *http.Request) (*UserAgent, error) {
+func UserAgentFromRequest(r *http.Request) *UserAgent {
q := r.URL.Query()
ua := q.Get("u")
if ua == "" {
ua = r.UserAgent()
}
+ hash := UserAgentHash(ua)
+
+ det := detector.New(ua)
+ browser, version := det.Browser()
return &UserAgent{
Name: ua,
LastSeenAt: time.Now(),
- Hash: UserAgentHash(ua),
- ua: detector.New(ua),
- }, nil
+ Hash: hash,
+ Browser: browser,
+ Platform: det.Platform(),
+ Version: version,
+ Bot: det.Bot(),
+ Mobile: strings.Contains(ua, "Mobi") || det.Mobile(),
+ }
}
func (ua UserAgent) Count() int {
@@ -57,28 +69,5 @@ func (ua UserAgent) Count() int {
}
func (ua UserAgent) Label() string {
- return ua.Browser() // + "/" + ua.BrowserVersion()
-}
-
-func (ua UserAgent) IsBot() bool {
- return ua.ua.Bot()
-}
-
-func (ua UserAgent) IsMobile() bool {
- //return ua.ua.Mobile()
- return strings.Contains(ua.Name, "Mobi")
-}
-
-func (ua UserAgent) Platform() string {
- return ua.ua.Platform()
-}
-
-func (ua UserAgent) Browser() string {
- n, _ := ua.ua.Browser()
- return n
-}
-
-func (ua UserAgent) BrowserVersion() string {
- _, v := ua.ua.Browser()
- return v
+ return ua.Browser
}