diff --git a/.drone.yml b/.drone.yml index 47fff11..ff60a69 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,32 +3,33 @@ type: docker name: default steps: -- name: static_check - image: golang:1.16 - commands: - - go get honnef.co/go/tools/cmd/staticcheck - - staticcheck ./pkg/... - volumes: - - name: gopath - path: /go + - name: static_check + image: golang:1.17 + commands: + - go install honnef.co/go/tools/cmd/staticcheck@latest + - staticcheck -checks all ./pkg/... + volumes: + - name: env_cache + path: /go -- name: lint - image: golang:1.16 - commands: - - go get golang.org/x/lint/golint - - golint -set_exit_status ./pkg/... - volumes: - - name: gopath - path: /go + - name: lint + image: golang:1.17 + commands: + - go install golang.org/x/lint/golint@latest + - golint -set_exit_status ./pkg/... + volumes: + - name: env_cache + path: /go -- name: vet - image: golang:1.16 - commands: - - go vet ./pkg/... - volumes: - - name: gopath - path: /go + - name: vet + image: golang:1.17 + commands: + - go vet ./pkg/... + volumes: + - name: env_cache + path: /go volumes: -- name: gopath - temp: {} \ No newline at end of file + - name: env_cache + host: + path: /tmp/drone/envs/vegvisir \ No newline at end of file diff --git a/.gitignore b/.gitignore index 557751c..741cfd8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.idea + vegvisir.json *.prof -.idea \ No newline at end of file + +*.local* diff --git a/go.mod b/go.mod index 6a06123..3a51d08 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module vegvisir +module git.pbiernat.dev/golang/vegvisir go 1.17 diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 3fcc09a..2285534 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -7,11 +7,12 @@ // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License // Repo: https://git.pbiernat.dev/golang/vegvisir +// Package cache whole cache functionality package cache import ( + "git.pbiernat.dev/golang/vegvisir/pkg/config" "log" - "vegvisir/pkg/config" ) const ( diff --git a/pkg/cache/manager.go b/pkg/cache/manager.go index 3f253f1..0cefdc6 100644 --- a/pkg/cache/manager.go +++ b/pkg/cache/manager.go @@ -27,6 +27,7 @@ import ( "time" ) +// Manager main struct type Manager struct { sync.RWMutex datastore Datastore @@ -46,7 +47,8 @@ type entry struct { ready chan struct{} // closed when res is ready } -type HandlerFunc func(url, method string, route *RouteCache) (error, *ResponseCache) +// HandlerFunc non-blocking cache handler func signature definition +type HandlerFunc func(url, method string, route *RouteCache) (*ResponseCache, error) type queue struct { items chan queueItem @@ -64,9 +66,10 @@ type queueItem struct { //var reqsSend = 0 -func NewCachedManager(datastore Datastore, prefix string, ttl int, handler HandlerFunc) Manager { +// NewCachedManager Creates instance of non-blocking cache manager +func NewCachedManager(datastore Datastore, prefix string, ttl int, handler HandlerFunc) *Manager { //log.Printf("Create DS: %s %d", prefix, ttl) - m := Manager{ + m := &Manager{ datastore: datastore, prefix: prefix, ttl: ttl, @@ -78,7 +81,8 @@ func NewCachedManager(datastore Datastore, prefix string, ttl int, handler Handl return m } -func HttpRequestHandler(url, method string, route *RouteCache) (error, *ResponseCache) { // FIXME: Refactor|Move to handler/ +// HTTPRequestHandler HTTP Handler for non-blocking cache reads +func HTTPRequestHandler(url, method string, route *RouteCache) (*ResponseCache, error) { // FIXME: Refactor|Move to handler/ //start := time.Now() bckReq := fasthttp.AcquireRequest() bckResp := fasthttp.AcquireResponse() @@ -91,7 +95,7 @@ func HttpRequestHandler(url, method string, route *RouteCache) (error, *Response err := fasthttp.Do(bckReq, bckResp) if err != nil { - return err, nil + return nil, err } body, code := string(bckResp.Body()), bckResp.StatusCode() @@ -101,11 +105,12 @@ func HttpRequestHandler(url, method string, route *RouteCache) (error, *Response // ^^ FIXME: rename NewResponseCache(with all depend struct) and move to handler/ (without dependencies) //reqsSend++ - log.Println(time.Now().Nanosecond(), "HttpRequestHandler() done:", route.TargetURL, url) + log.Println(time.Now().Nanosecond(), "HTTPRequestHandler() done:", route.TargetURL, url) - return nil, respCache + return respCache, nil } +// Fetch Non-blocking cache response read // FIXME: #68 - refactor func (m *Manager) Fetch(url, method string, route *RouteCache) (*ResponseCache, error) { //log.Println("Response CM: Fetch()") @@ -116,11 +121,12 @@ func (m *Manager) Fetch(url, method string, route *RouteCache) (*ResponseCache, return res.body, res.err } +// Close exec while server shutting down func (m *Manager) Close() { close(m.queue.items) } -func (m *Manager) load(url, method string, route *RouteCache/*, output interface{}*/) (bool, *entry) { // FIXME second return type( interface{} ) +func (m *Manager) load(url, method string, route *RouteCache /*, output interface{}*/) (bool, *entry) { // FIXME second return type( interface{} ) //log.Println("Response CM: load()") key := m.prefix + method + "_" + url @@ -173,7 +179,7 @@ func (m *Manager) save(name string, r interface{}) bool { // FIXME second argume func (m *Manager) server(handler HandlerFunc) { for item := range m.queue.items { m.Lock() - ok, e := m.load(item.url, item.method, item.route/*, &ResponseCache{}*/) // FIXME: &responseCache{} tmp fix + ok, e := m.load(item.url, item.method, item.route /*, &ResponseCache{}*/) // FIXME: &responseCache{} tmp fix m.Unlock() if !ok { //e = &entry{ready: make(chan struct{})/*, called: make(chan struct{})*/} @@ -188,7 +194,7 @@ func (m *Manager) server(handler HandlerFunc) { func (m *Manager) call(e *entry, f HandlerFunc, url, method string, route *RouteCache) { e.res.name = method + "_" + url // FIXME: hardcoded key pattern m.RLock() - e.res.err, e.res.body = f(url, method, route) + e.res.body, e.res.err = f(url, method, route) m.save(e.res.name, e.res.body) m.RUnlock() diff --git a/pkg/cache/memory_datastore.go b/pkg/cache/memory_datastore.go index d099513..c69f4e9 100644 --- a/pkg/cache/memory_datastore.go +++ b/pkg/cache/memory_datastore.go @@ -33,7 +33,7 @@ func NewMemoryDatastore() *MemoryDatastore { type MemoryDatastore struct { cache map[string]interface{} ts map[string]TTLItem - lock sync.RWMutex + lock sync.RWMutex } // SetKey function diff --git a/pkg/cache/redis_datastore.go b/pkg/cache/redis_datastore.go index fc95486..71a557b 100644 --- a/pkg/cache/redis_datastore.go +++ b/pkg/cache/redis_datastore.go @@ -10,12 +10,11 @@ package cache import ( + "github.com/go-redis/redis" "log" "os" "strconv" "time" - - "github.com/go-redis/redis" ) // NewRedisDatastore function diff --git a/pkg/cache/response.go b/pkg/cache/response.go index 2581ad1..0b87704 100644 --- a/pkg/cache/response.go +++ b/pkg/cache/response.go @@ -23,6 +23,7 @@ type ResponseCache struct { Headers *fasthttp.ResponseHeader } +// NewResponseCache Creates new ResponseCache structure func NewResponseCache(url, method, body string, code int, headers *fasthttp.ResponseHeader) *ResponseCache { return &ResponseCache{url, method, body, code, headers} } diff --git a/pkg/cache/route.go b/pkg/cache/route.go index b74c10b..82d40b4 100644 --- a/pkg/cache/route.go +++ b/pkg/cache/route.go @@ -9,11 +9,13 @@ package cache +// RouteCache struct corresponding route cache item type RouteCache struct { SourceURL string TargetURL string } +// NewRouteCache Creates new router instance func NewRouteCache(source, target string) *RouteCache { return &RouteCache{ SourceURL: source, diff --git a/pkg/client/http.go b/pkg/client/http.go index 624dc5f..784dbcb 100644 --- a/pkg/client/http.go +++ b/pkg/client/http.go @@ -7,6 +7,7 @@ // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License // Repo: https://git.pbiernat.dev/golang/vegvisir +// Package client all available clients lives here (httpo, https, grpc etc.) package client import "github.com/valyala/fasthttp" diff --git a/pkg/config/config.go b/pkg/config/config.go index 52a3c5e..3c82f4b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,6 +7,7 @@ // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License // Repo: https://git.pbiernat.dev/golang/vegvisir +// Package config main config file structs package config import ( diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 7e14d54..193a14c 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -7,4 +7,5 @@ // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License // Repo: https://git.pbiernat.dev/golang/vegvisir +// Package handler all handlers will be placed here in future... package handler diff --git a/pkg/main.go b/pkg/main.go index 7b882e0..dabfa7a 100644 --- a/pkg/main.go +++ b/pkg/main.go @@ -11,14 +11,14 @@ package main import ( "flag" + "git.pbiernat.dev/golang/vegvisir/pkg/server" "log" "net/http" + _ "net/http/pprof" "os" "runtime" "runtime/pprof" - "vegvisir/pkg/server" ) -import _ "net/http/pprof" var ( cPath = flag.String("c", "vegvisir.json", "Path to config file") diff --git a/pkg/server/router.go b/pkg/server/router.go index aff96e2..0ee04f3 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -12,27 +12,24 @@ package server import ( "fmt" + "git.pbiernat.dev/golang/vegvisir/pkg/cache" + "git.pbiernat.dev/golang/vegvisir/pkg/config" "log" "regexp" "strings" - "vegvisir/pkg/cache" - "vegvisir/pkg/config" ) // Router struct type Router struct { config *config.Config - //cache map[string]cache.RouteCache cache *cache.MemoryDatastore - //cache *cache.RedisDatastore } // NewRouter function -func NewRouter(config *config.Config, ds cache.Datastore, ttl int) *Router { +func NewRouter(config *config.Config, ds *cache.MemoryDatastore, ttl int) *Router { return &Router{ config: config, - //cache: make(map[string]cache.RouteCache), - cache: cache.NewMemoryDatastore(), + cache: ds, //cache: cache.NewRedisDatastore(config.Cache.Host, config.Cache.Password, config.Cache.Database, config.Cache.Port), } } @@ -65,7 +62,7 @@ func (r *Router) FindByRequestURL(url []byte) (bool, *cache.RouteCache) { //routeJSON, _ := ffjson.Marshal(&route) // FIXME: TMP - if err := r.cache.SetKey("route_" + sURL, route, r.config.Cache.RouteTTL); err != nil { + if err := r.cache.SetKey("route_"+sURL, route, r.config.Cache.RouteTTL); err != nil { // FIXME: ^^ use cache.Manager* after refactor log.Println("Error saving route cache:", sURL) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 3589af4..f9065e1 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -7,20 +7,20 @@ // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License // Repo: https://git.pbiernat.dev/golang/vegvisir +// Package server contains main sever files package server import ( "context" "fmt" + "git.pbiernat.dev/golang/vegvisir/pkg/cache" + "git.pbiernat.dev/golang/vegvisir/pkg/config" + "github.com/valyala/fasthttp" "log" "os" "os/signal" "syscall" "time" - "vegvisir/pkg/cache" - "vegvisir/pkg/config" - - "github.com/valyala/fasthttp" ) const ( @@ -34,7 +34,7 @@ const ( type Server struct { config *config.Config router *Router - respCM cache.Manager + respCM *cache.Manager daemon *fasthttp.Server } @@ -52,7 +52,7 @@ func NewServer(cPath string) *Server { config: cfg, router: NewRouter(cfg, cache.NewMemoryDatastore(), cfg.Cache.RouteTTL), // ^^ INFO: Routes always use memoryCache cause low size and fast read time (optimalization) - respCM: cache.NewCachedManager(*datastore, "response_", cfg.Cache.ResponseTTL, cache.HttpRequestHandler), + respCM: cache.NewCachedManager(*datastore, "response_", cfg.Cache.ResponseTTL, cache.HTTPRequestHandler), // ^^ FIXME: add handler detection option by config etc... } } @@ -65,12 +65,12 @@ func (s *Server) Run() { go func() { serverAddress := s.config.Server.Address + ":" + fmt.Sprint(s.config.Server.Port) s.daemon = &fasthttp.Server{ - Handler: s.mainHandler, - ReadTimeout: 15 * time.Second, - WriteTimeout: 20 * time.Second, - MaxConnsPerIP: 500, - MaxRequestsPerConn: 500, - IdleTimeout: 15 * time.Second, // aka KeepAlive + Handler: s.mainHandler, + ReadTimeout: 15 * time.Second, + WriteTimeout: 20 * time.Second, + MaxConnsPerIP: 500, + MaxRequestsPerConn: 500, + IdleTimeout: 15 * time.Second, // aka KeepAlive //CloseOnShutdown: true, }