diff options
| -rw-r--r-- | Makefile | 43 | ||||
| -rw-r--r-- | README.md | 17 | ||||
| -rw-r--r-- | cmd/dhtsearch/main.go (renamed from cmd/main.go) | 2 | ||||
| -rw-r--r-- | cmd/dhtsearch/tag.go (renamed from cmd/tag.go) | 0 | ||||
| -rw-r--r-- | db/sqlite.go | 3 | ||||
| -rw-r--r-- | go.mod | 16 | ||||
| -rw-r--r-- | go.sum | 27 | ||||
| -rw-r--r-- | main.go | 128 |
8 files changed, 63 insertions, 173 deletions
@@ -1,39 +1,36 @@ -TARGETS = freebsd-amd64 linux-386 linux-amd64 linux-arm linux-arm64 darwin-amd64 windows-386 windows-amd64 -CMD = dhtsearch VERSION ?= $(shell git describe --tags --always) -SRC = $(shell find . -type f -name '*.go') -FLAGS = --tags fts5 -BINARIES = $(patsubst %,$(CMD)-%-v$(VERSION), $(TARGETS)) - -.PHONY: help -help: - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ - |sort \ - |awk 'BEGIN{FS=":.*?## "};{printf "\033[36m%-30s\033[0m %s\n",$$1,$$2}' +SRC := $(shell find . -type f -name '*.go') +FLAGS := --tags fts5 +PLAT := windows darwin linux freebsd openbsd +BINARY := $(patsubst %,dist/%,$(shell find cmd/* -maxdepth 0 -type d -exec basename {} \;)) +RELEASE := $(foreach os, $(PLAT), $(patsubst %,%-$(os), $(BINARY))) .PHONY: build -build: sqlite $(BINARIES) ## Build all binaries +build: sqlite $(BINARY) + +.PHONY: release +release: $(RELEASE) -$(BINARIES): $(SRC) - cd cmd && env GOOS=`echo $@ |cut -d'-' -f2` \ - GOARCH=`echo $@ |cut -d'-' -f3 |cut -d'.' -f1` \ - go build -o ../$@ $(FLAGS) -ldflags="-w -s -X=main.version=$(VERSION)" +dist/%: export GOOS=$(word 2,$(subst -, ,$*)) +dist/%: bin=$(word 1,$(subst -, ,$*)) +dist/%: $(SRC) $(shell find cmd/$(bin) -type f -name '*.go') + go build -ldflags "-X main.version=$(VERSION)" $(FLAGS) \ + -o $@ ./cmd/$(bin) sqlite: - go get -u $(FLAGS) github.com/mattn/go-sqlite3 \ + CGO_ENABLED=1 go get -u $(FLAGS) github.com/mattn/go-sqlite3 \ && go install $(FLAGS) github.com/mattn/go-sqlite3 .PHONY: test -test: ## Run tests and create coverage report +test: go test -short -coverprofile=coverage.out ./... \ && go tool cover -func=coverage.out .PHONY: lint -lint: - revive ./... +lint: ; go vet ./... .PHONY: clean -clean: ## Clean up temp files and binaries - rm -f $(CMD)-*-v* - rm -rf coverage* +clean: + rm -f coverage* + rm -rf dist @@ -24,29 +24,28 @@ hosted by the remote nodes are not retrieved. - **Filtering** can be done by tags. By default all torrents tagged 'adult' are not indexed. See the SkipTags option in the configuration file. -- **Full Text Search** using PostgreSQL's text search vectors. Torrent names - are weighted more than file names. +- **Full Text Search** using PostgreSQL's or Sqlite's text search vectors. + Torrent names are weighted more than file names. - **Statistics** for the crawler process are available when the HTTP server is enabled. Fetch the JSON from the `/status` endpoint. - **Custom tags** can be defined in the configuration file. +- **Cross-platform** builds for Windows, Macos, Linux, FreeBSD, and OpenBSD + ## Installation -All dependencies have been vendored using the -[dep](https://github.com/golang/dep) tool so installation with a recent Go -version should be as simple as: +There is a Makefile for GNU make: ```shell -$ go build +$ make build ``` ## Usage -You will need to create a PostgreSQL database using the `schema.sql` file -provided. You are going to need to sort out any port forwarding if you are -behind NAT so remote nodes can get to yours. +You are going to need to sort out any port forwarding if you are behind NAT so +remote nodes can get to yours. Configuration is done via a [TOML](https://github.com/toml-lang/toml) formatted file or via flags passed to the daemon. diff --git a/cmd/main.go b/cmd/dhtsearch/main.go index b81ea4c..921de77 100644 --- a/cmd/main.go +++ b/cmd/dhtsearch/main.go @@ -58,7 +58,7 @@ func main() { flag.IntVar(&btNodes, "bt-nodes", 3, "number of BT nodes to start") flag.StringVar(&skipTags, "skip-tags", "xxx", "tags of torrents to skip") - flag.StringVar(&dsn, "dsn", "postgres://dhtsearch@localhost/dhtsearch", "database DSN") + flag.StringVar(&dsn, "dsn", "file:dhtsearch.db?cache=shared&mode=memory", "database DSN") flag.BoolVar(&showVersion, "v", false, "show version") diff --git a/cmd/tag.go b/cmd/dhtsearch/tag.go index 6d5c0ea..6d5c0ea 100644 --- a/cmd/tag.go +++ b/cmd/dhtsearch/tag.go diff --git a/db/sqlite.go b/db/sqlite.go index 50baf3b..b7d9c33 100644 --- a/db/sqlite.go +++ b/db/sqlite.go @@ -348,7 +348,8 @@ func (s *Store) prepareStatements() error { join peers_torrents pt on pt.torrent_id = t.id join peers p on p.id = pt.peer_id where t.name is null - group by t.infohash`, + group by t.infohash + limit ?`, ); err != nil { return err } @@ -3,12 +3,20 @@ module src.userspace.com.au/dhtsearch go 1.13 require ( + github.com/cockroachdb/apd v1.1.0 // indirect + github.com/hashicorp/go-version v1.2.1 // indirect github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 + github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect github.com/jackc/pgx v3.1.0+incompatible - github.com/mattn/go-sqlite3 v1.11.0 - github.com/pkg/errors v0.8.0 - golang.org/x/net v0.0.0-20180330215511-b68f30494add + github.com/lib/pq v1.8.0 // indirect + github.com/mattn/go-sqlite3 v1.14.4 + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/pkg/errors v0.8.0 // indirect + github.com/satori/go.uuid v1.2.0 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + golang.org/x/net v0.0.0-20180330215511-b68f30494add // indirect golang.org/x/time v0.0.0-20180314180208-26559e0f760e - src.userspace.com.au/go-bencode v0.3.0 + gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect + src.userspace.com.au/go-bencode v0.3.1 src.userspace.com.au/logger v0.1.1 ) @@ -1,20 +1,33 @@ +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= github.com/jackc/pgx v3.1.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackpal/bencode-go v0.0.0-20180813173944-227668e840fa h1:ym9I4Q1lJG8nu+j5R2H6mHOfVjYbSiwUOzh/AFs3Xfs= github.com/jackpal/bencode-go v0.0.0-20180813173944-227668e840fa/go.mod h1:5FSBQ74yhCl5oQ+QxRPYzWMONFnxbL68/23eezsBI5c= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e h1:KMs6SK8iDSR1+ZzOK10L5wGPpWDByyvOe5nrqk51g2U= github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e/go.mod h1:+AHfJo5+69p+fjvMJTmYajNP9rFBHQcaTDFmuXRRATI= -github.com/mattn/go-sqlite3 v1.6.0 h1:TDwTWbeII+88Qy55nWlof0DclgAtI4LqGujkYMzmQII= -github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= +github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +golang.org/x/net v0.0.0-20180330215511-b68f30494add h1:oGr9qHpQTQvl/BmeWw95ZrQKahW4qdIPUiGfQkJYDsA= golang.org/x/net v0.0.0-20180330215511-b68f30494add/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/time v0.0.0-20180314180208-26559e0f760e h1:aUMCDtB7fbxaw60p2ngy69FCEzU3XpcAEpszqXsdXWg= golang.org/x/time v0.0.0-20180314180208-26559e0f760e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -src.userspace.com.au/dhtsearch v0.0.0-20180325231531-cb1d36ac4386/go.mod h1:IKhUw6hs9WQA7GGrPUV+2aSO83ZiS7J4eJKFyWFNNhM= -src.userspace.com.au/go-bencode v0.3.0 h1:GnJvOtibAmSEc7LVhulXwWJsOWpXfmmO8NwkERnWR0o= -src.userspace.com.au/go-bencode v0.3.0/go.mod h1:x/2aZW6OnN60Ot7VCTShG1B7H47QaWRS7pCxkgCpfaI= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +src.userspace.com.au/go-bencode v0.3.1 h1:O3Qny4bKM4on4U/LjyWb3i7KwQnQnUPIb2PSjGCBnoI= +src.userspace.com.au/go-bencode v0.3.1/go.mod h1:x/2aZW6OnN60Ot7VCTShG1B7H47QaWRS7pCxkgCpfaI= src.userspace.com.au/logger v0.1.1 h1:mfnQv/pi5X2fxgXxMAJS7oDOWww4p2P2ktDuP5AjCtQ= src.userspace.com.au/logger v0.1.1/go.mod h1:PC4uwII4fDAhivRMJMp+6uk+VCT6/EZcdDE8hXG56NI= diff --git a/main.go b/main.go deleted file mode 100644 index a198fcf..0000000 --- a/main.go +++ /dev/null @@ -1,128 +0,0 @@ -// +build ignore - -package dhtsearch - -import ( - "fmt" - "net" - "net/http" - "os" - "time" - - "src.userspace.com.au/logger" -) - -func uptime() interface{} { - return int64(time.Since(start).Seconds()) -} - -func main() { - //defer profile.Start(profile.CPUProfile).Stop() - expvar.Publish("uptime", expvar.Func(uptime)) - - log := logger.New(&logger.Options{ - Name: "dht", - Level: logger.Info, - }) - - // Slice of channels for DHT node output - torrents := make(chan Torrent) - peers := make(chan peer) - - // Close upstreams channels - done := make(chan struct{}) - defer close(done) - - // Persistence - var err error - DB, err = newDB(Config.Dsn) - if err != nil { - os.Exit(1) - } - defer DB.Close() - - // Initialise tags - for tag := range tags { - _, err := createTag(tag) - if err != nil { - fmt.Printf("Error creating tag %s: %q\n", tag, err) - } - } - - // Create DHT nodes - if Config.Debug { - fmt.Printf("Starting %d instance(s)\n", Config.NumNodes) - } - for i := 0; i < Config.NumNodes; i++ { - // Consecutive port numbers - port := Config.BasePort + i - dht := newDHTNode("", port, peers) - err = dht.run(done) - if err != nil { - os.Exit(1) - } - } - - // Create BT node - bt := &btClient{} - bt.log = log.Named("bt") - err = btClient.run(torrents) - if err != nil { - os.Exit(1) - } - - // HTTP Server - if !Config.NoHttp { - http.HandleFunc("/", indexHandler) - http.HandleFunc("/stats", statsHandler) - http.HandleFunc("/search", searchHandler) - sock, _ := net.Listen("tcp", Config.HttpAddress) - go func() { - if !Config.Quiet { - fmt.Printf("HTTP now available at %s\n", Config.HttpAddress) - } - http.Serve(sock, nil) - }() - } - - // Simple cache of most recent - var p peer - var t Torrent - - for { - select { - case t = <-torrents: - length := t.Size - - // Add tags - tagTorrent(&t) - - // Not sure if I like continue labels, so this - var notWanted = false - for _, tag := range Config.SkipTags { - if hasTag(t, tag) { - if !Config.Quiet { - fmt.Printf("Skipping torrent tagged '%s': %q\n", tag, t.Name) - } - notWanted = true - break - } - } - if notWanted { - torrentsSkipped.Add(1) - continue - } - - err := t.save() - if err != nil { - fmt.Printf("Error saving torrent: %q\n", err) - continue - } - if !Config.Quiet { - fmt.Printf("Torrrent added, length: %d, name: %q, tags: %s, url: magnet:?xt=urn:btih:%s\n", length, t.Name, t.Tags, t.Infohash) - } - torrentsSaved.Add(1) - torrentsTotal.Add(1) - } - } -} |