Begin splitting up libraries

14 files changed, 78 insertions, 85 deletions

diff --git a/bencode.go b/bencode/bencode.go
index 16ec993..6b2373a 100644
--- a/bencode.go
+++ b/bencode/bencode.go
@@ -1,4 +1,4 @@
-package dhtsearch
+package bencode
 
 // Lifted from github.com/shiyanhui/dht
 
diff --git a/btclient.go b/bittorrent/client.go
index 957dc89..f4ac28e 100644
--- a/btclient.go
+++ b/bittorrent/client.go
@@ -14,10 +14,16 @@ import (
 	"strings"
 	"time"
 
+	"github.com/felix/dhtsearch/bencode"
 	"github.com/felix/logger"
 )
 
 const (
+	TCPTimeout = 5
+	UDPTimeout = 5
+)
+
+const (
 	// MsgRequest represents request message type
 	MsgRequest = iota
 	// MsgData represents data message type
@@ -175,7 +181,7 @@ func (bt *btClient) fetchMetadata(p peer) (out []byte, err error) {
 				return out, errors.New("no pieces found")
 			}
 
-			d, index, err := DecodeDict(payload, 0)
+			d, index, err := bencode.DecodeDict(payload, 0)
 			if err != nil {
 				return out, err
 			}
@@ -217,7 +223,7 @@ func (bt *btClient) fetchMetadata(p peer) (out []byte, err error) {
 }
 
 func decodeMetadata(p peer, md []byte) (*Torrent, error) {
-	metadata, err := Decode(md)
+	metadata, err := bencode.Decode(md)
 	if err != nil {
 		return nil, err
 	}
@@ -334,7 +340,7 @@ func onHandshake(data []byte) (err error) {
 func sendExtHandshake(conn *net.TCPConn) error {
 	data := append(
 		[]byte{MsgExtended, HandshakeBit},
-		Encode(map[string]interface{}{
+		bencode.Encode(map[string]interface{}{
 			"m": map[string]interface{}{"ut_metadata": 1},
 		})...,
 	)
@@ -344,7 +350,7 @@ func sendExtHandshake(conn *net.TCPConn) error {
 
 // getUTMetaSize returns the ut_metadata and metadata_size.
 func getUTMetaSize(data []byte) (utMetadata int, metadataSize int, err error) {
-	v, err := Decode(data)
+	v, err := bencode.Decode(data)
 	if err != nil {
 		return utMetadata, metadataSize, err
 	}
@@ -381,7 +387,7 @@ func (bt *btClient) requestPieces(conn *net.TCPConn, utMetadata int, metadataSiz
 		buffer[0] = MsgExtended
 		buffer[1] = byte(utMetadata)
 
-		msg := Encode(map[string]interface{}{
+		msg := bencode.Encode(map[string]interface{}{
 			"msg_type": MsgRequest,
 			"piece":    i,
 		})
diff --git a/config.go b/config.go
deleted file mode 100644
index b08557d..0000000
--- a/config.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package dhtsearch
-
-// Config -
-type Config struct {
-	BasePort    int               `toml:"base_port"`
-	Debug       bool              `toml:"debug"`
-	NoHTTP      bool              `toml:"no_http"`
-	HTTPAddress string            `toml:"http_address"`
-	DSN         string            `toml:"dsn"`
-	NumNodes    int               `toml:"num_nodes"`
-	Tags        map[string]string `toml:"tags"`
-	SkipTags    []string          `toml:"skip_tags"`
-
-	// Advanced
-	MaxBTWorkers    int `toml:"max_bt_workers"`
-	MaxDHTWorkers   int `toml:"max_dht_workers"`
-	PeerCacheSize   int `toml:"peer_cache_size"`
-	ResultsPageSize int `toml:"results_page_size"`
-}
-
-// DefaultConfig sets the defaults
-func DefaultConfig() Config {
-	return Config{
-		BasePort:        6881,
-		NumNodes:        1,
-		Debug:           false,
-		DSN:             "postgres://dht:dht@localhost/dht?sslmode=disable",
-		NoHTTP:          false,
-		HTTPAddress:     "localhost:6880",
-		MaxBTWorkers:    10,
-		MaxDHTWorkers:   256,
-		PeerCacheSize:   200,
-		ResultsPageSize: 50,
-	}
-}
diff --git a/server.go b/crawler/crawler.go
index 5a7da30..bfd9785 100644
--- a/server.go
+++ b/crawler/crawler.go
@@ -1,4 +1,4 @@
-package dhtsearch
+package crawler
 
 import (
 	"regexp"
@@ -7,11 +7,11 @@ import (
 )
 
 const (
-	TCPTimeout = 10
-	UDPTimeout = 10
+	TCPTimeout = 5
+	UDPTimeout = 5
 )
 
-type Server struct {
+type Crawler struct {
 	port        int
 	nodes       int
 	httpAddress string
@@ -20,10 +20,11 @@ type Server struct {
 }
 
 // Option are options for the server
-type Option func(*Server) error
+type Option func(*Crawler) error
 
-func NewServer(dsn string, opts ...Option) (*Server, error) {
-	s := &Server{
+// NewCrawler creates a set of DHT nodes to crawl the network
+func NewCrawer(opts ...Option) (*Crawler, error) {
+	s := &Crawler{
 		port:        6881,
 		nodes:       1,
 		httpAddress: "localhost:6880",
@@ -32,7 +33,7 @@ func NewServer(dsn string, opts ...Option) (*Server, error) {
 
 	// Default logger
 	logOpts := &logger.Options{
-		Name:  "dhtsearch",
+		Name:  "crawler",
 		Level: logger.Info,
 	}
 	s.log = logger.New(logOpts)
@@ -79,7 +80,7 @@ func NewServer(dsn string, opts ...Option) (*Server, error) {
 
 // SetLogger sets the server
 func SetLogger(l logger.Logger) Option {
-	return func(s *Server) error {
+	return func(s *Crawler) error {
 		s.log = l
 		return nil
 	}
@@ -87,7 +88,7 @@ func SetLogger(l logger.Logger) Option {
 
 // SetPort sets the base port
 func SetPort(p int) Option {
-	return func(s *Server) error {
+	return func(s *Crawler) error {
 		s.port = p
 		return nil
 	}
@@ -95,7 +96,7 @@ func SetPort(p int) Option {
 
 // SetNodes determines the number of nodes to start
 func SetNodes(n int) Option {
-	return func(s *Server) error {
+	return func(s *Crawler) error {
 		s.nodes = n
 		return nil
 	}
@@ -103,7 +104,7 @@ func SetNodes(n int) Option {
 
 // SetHTTPAddress determines the listening address for HTTP
 func SetHTTPAddress(a string) Option {
-	return func(s *Server) error {
+	return func(s *Crawler) error {
 		s.httpAddress = a
 		return nil
 	}
@@ -111,7 +112,7 @@ func SetHTTPAddress(a string) Option {
 
 // SetTags determines the listening address for HTTP
 func SetTags(tags map[string]string) Option {
-	return func(s *Server) error {
+	return func(s *Crawler) error {
 		// Merge user tags
 		err := mergeTagRegexps(s.tagREs, tags)
 		if err != nil {
@@ -121,7 +122,7 @@ func SetTags(tags map[string]string) Option {
 	}
 }
 
-func (s *Server) Stats() Stats {
+func (s *Crawler) Stats() Stats {
 	s.statlock.RLock()
 	defer s.statlock.RUnlock()
 	return s.stats
diff --git a/dht.go b/crawler/dht.go
index 1a8dcf2..51d478e 100644
--- a/dht.go
+++ b/crawler/dht.go
@@ -1,4 +1,4 @@
-package dhtsearch
+package crawler
 
 import (
 	"math"
diff --git a/dht_worker.go b/crawler/dht_worker.go
index ef86dc1..29f3bc5 100644
--- a/dht_worker.go
+++ b/crawler/dht_worker.go
@@ -1,4 +1,4 @@
-package dhtsearch
+package crawler
 
 import (
 	"net"
diff --git a/crawler/krpc.go b/crawler/krpc.go
new file mode 100644
index 0000000..67150c0
--- /dev/null
+++ b/crawler/krpc.go
@@ -0,0 +1,37 @@
+package crawler
+
+import (
+	"errors"
+	"fmt"
+	"net"
+)
+
+// makeQuery returns a query-formed data.
+func makeQuery(t, q string, a map[string]interface{}) map[string]interface{} {
+	return map[string]interface{}{
+		"t": t,
+		"y": "q",
+		"q": q,
+		"a": a,
+	}
+}
+
+// makeResponse returns a response-formed data.
+func makeResponse(t string, r map[string]interface{}) map[string]interface{} {
+	return map[string]interface{}{
+		"t": t,
+		"y": "r",
+		"r": r,
+	}
+}
+
+// parseKeys parses keys. It just wraps parseKey.
+func parseKeys(data map[string]interface{}, pairs [][]string) error {
+	for _, args := range pairs {
+		key, t := args[0], args[1]
+		if err := parseKey(data, key, t); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/packet.go b/crawler/packet.go
index 7c5368b..1fa3318 100644
--- a/packet.go
+++ b/crawler/packet.go
@@ -1,4 +1,4 @@
-package dhtsearch
+package crawler
 
 import "net"
 
diff --git a/crawler/peer.go b/crawler/peer.go
new file mode 100644
index 0000000..06be506
--- /dev/null
+++ b/crawler/peer.go
@@ -0,0 +1,9 @@
+package crawler
+
+import "net"
+
+// Peer on DHT network
+type Peer struct {
+	Address net.UDPAddr
+	ID      string
+}
diff --git a/remote_node.go b/crawler/remote_node.go
index 0a1083d..bfbc5ac 100644
--- a/remote_node.go
+++ b/crawler/remote_node.go
@@ -1,4 +1,4 @@
-package dhtsearch
+package crawler
 
 import (
 	"fmt"
diff --git a/routing_table.go b/crawler/routing_table.go
index 5f5ed04..8bb0d3c 100644
--- a/routing_table.go
+++ b/crawler/routing_table.go
@@ -1,4 +1,4 @@
-package dhtsearch
+package crawler
 
 import (
 	"net"
diff --git a/example-config.toml b/example-config.toml
deleted file mode 100644
index 47e0986..0000000
--- a/example-config.toml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Move to config.toml in the same directory as the binary.
-# All command line arguments are accepted plus some more.
-
-# The following are the defaults:
-
-base-port = 6881
-num-nodes = 1
-debug = false
-quiet = false
-dsn = "postgres://dht:dht@localhost/dht?sslmode=disable"
-no-http = false
-http-address = "localhost:6880"
-skip-tags = ["adult"]
-
-[tags]
-test = "\\.asdfasdfsdfsfdsfs"
diff --git a/peer.go b/peer.go
deleted file mode 100644
index 4fc2b95..0000000
--- a/peer.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package dhtsearch
-
-import "net"
-
-// Annouced peer
-type peer struct {
-	address net.UDPAddr
-	id      string
-}
diff --git a/krpc.go b/util.go
index 1548fab..1548fab 100644
--- a/krpc.go
+++ b/util.go