Merge branch 'develop-memo-cache' of git.pbiernat.dev:golang/vegvisir into develop-memo-cache

.drone.yml

@@ -3,32 +3,33 @@ type: docker
 name: default
 
 steps:
-- name: static_check
+  - name: static_check
-  image: golang:1.16
+    image: golang:1.17
-  commands:
+    commands:
-  - go get honnef.co/go/tools/cmd/staticcheck
+    - go install honnef.co/go/tools/cmd/staticcheck@latest
-  - staticcheck ./pkg/...
+    - staticcheck -checks all ./pkg/...
-  volumes:
+    volumes:
-  - name: gopath
+    - name: env_cache
-    path: /go
+      path: /go
 
-- name: lint
+  - name: lint
-  image: golang:1.16
+    image: golang:1.17
-  commands:
+    commands:
-  - go get golang.org/x/lint/golint
+    - go install golang.org/x/lint/golint@latest
-  - golint -set_exit_status ./pkg/...
+    - golint -set_exit_status ./pkg/...
-  volumes:
+    volumes:
-  - name: gopath
+    - name: env_cache
-    path: /go
+      path: /go
 
-- name: vet
+  - name: vet
-  image: golang:1.16
+    image: golang:1.17
-  commands:
+    commands:
-  - go vet ./pkg/...
+    - go vet ./pkg/...
-  volumes:
+    volumes:
-  - name: gopath
+    - name: env_cache
-    path: /go
+      path: /go
 
 volumes:
-- name: gopath
+  - name: env_cache
-  temp: {}
+    host:
+      path: /tmp/drone/envs/vegvisir

.gitignore

@@ -1,3 +1,6 @@
+.idea
+
 vegvisir.json
 *.prof
-.idea
+
+*.local*

go.mod

@@ -1,4 +1,4 @@
-module vegvisir
+module git.pbiernat.dev/golang/vegvisir
 
 go 1.17
 

pkg/cache/cache.go

@@ -7,11 +7,12 @@
 // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License
 // Repo: https://git.pbiernat.dev/golang/vegvisir
 
+// Package cache whole cache functionality
 package cache
 
 import (
+	"git.pbiernat.dev/golang/vegvisir/pkg/config"
 	"log"
-	"vegvisir/pkg/config"
 )
 
 const (

pkg/cache/manager.go

@@ -27,6 +27,7 @@ import (
 	"time"
 )
 
+// Manager main struct
 type Manager struct {
 	sync.RWMutex
 	datastore Datastore
@@ -46,7 +47,8 @@ type entry struct {
 	ready chan struct{} // closed when res is ready
 }
 
-type HandlerFunc func(url, method string, route *RouteCache) (error, *ResponseCache)
+// HandlerFunc non-blocking cache handler func signature definition
+type HandlerFunc func(url, method string, route *RouteCache) (*ResponseCache, error)
 
 type queue struct {
 	items chan queueItem
@@ -64,9 +66,10 @@ type queueItem struct {
 
 //var reqsSend = 0
 
-func NewCachedManager(datastore Datastore, prefix string, ttl int, handler HandlerFunc) Manager {
+// NewCachedManager Creates instance of non-blocking cache manager
+func NewCachedManager(datastore Datastore, prefix string, ttl int, handler HandlerFunc) *Manager {
 	//log.Printf("Create DS: %s %d", prefix, ttl)
-	m := Manager{
+	m := &Manager{
 		datastore: datastore,
 		prefix:    prefix,
 		ttl:       ttl,
@@ -78,7 +81,8 @@ func NewCachedManager(datastore Datastore, prefix string, ttl int, handler Handl
 	return m
 }
 
-func HttpRequestHandler(url, method string, route *RouteCache) (error, *ResponseCache) { // FIXME: Refactor|Move to handler/
+// HTTPRequestHandler HTTP Handler for non-blocking cache reads
+func HTTPRequestHandler(url, method string, route *RouteCache) (*ResponseCache, error) { // FIXME: Refactor|Move to handler/
 	//start := time.Now()
 	bckReq := fasthttp.AcquireRequest()
 	bckResp := fasthttp.AcquireResponse()
@@ -91,7 +95,7 @@ func HttpRequestHandler(url, method string, route *RouteCache) (error, *Response
 
 	err := fasthttp.Do(bckReq, bckResp)
 	if err != nil {
-		return err, nil
+		return nil, err
 	}
 
 	body, code := string(bckResp.Body()), bckResp.StatusCode()
@@ -101,11 +105,12 @@ func HttpRequestHandler(url, method string, route *RouteCache) (error, *Response
 	// ^^ FIXME: rename NewResponseCache(with all depend struct) and move to handler/ (without dependencies)
 
 	//reqsSend++
-	log.Println(time.Now().Nanosecond(), "HttpRequestHandler() done:", route.TargetURL, url)
+	log.Println(time.Now().Nanosecond(), "HTTPRequestHandler() done:", route.TargetURL, url)
 
-	return nil, respCache
+	return respCache, nil
 }
 
+// Fetch Non-blocking cache response read
 // FIXME: #68 - refactor
 func (m *Manager) Fetch(url, method string, route *RouteCache) (*ResponseCache, error) {
 	//log.Println("Response CM: Fetch()")
@@ -116,11 +121,12 @@ func (m *Manager) Fetch(url, method string, route *RouteCache) (*ResponseCache,
 	return res.body, res.err
 }
 
+// Close exec while server shutting down
 func (m *Manager) Close() {
 	close(m.queue.items)
 }
 
-func (m *Manager) load(url, method string, route *RouteCache/*, output interface{}*/) (bool, *entry) { // FIXME second return type( interface{} )
+func (m *Manager) load(url, method string, route *RouteCache /*, output interface{}*/) (bool, *entry) { // FIXME second return type( interface{} )
 	//log.Println("Response CM: load()")
 
 	key := m.prefix + method + "_" + url
@@ -173,7 +179,7 @@ func (m *Manager) save(name string, r interface{}) bool { // FIXME second argume
 func (m *Manager) server(handler HandlerFunc) {
 	for item := range m.queue.items {
 		m.Lock()
-		ok, e := m.load(item.url, item.method, item.route/*, &ResponseCache{}*/) // FIXME: &responseCache{} tmp fix
+		ok, e := m.load(item.url, item.method, item.route /*, &ResponseCache{}*/) // FIXME: &responseCache{} tmp fix
 		m.Unlock()
 		if !ok {
 			//e = &entry{ready: make(chan struct{})/*, called: make(chan struct{})*/}
@@ -188,7 +194,7 @@ func (m *Manager) server(handler HandlerFunc) {
 func (m *Manager) call(e *entry, f HandlerFunc, url, method string, route *RouteCache) {
 	e.res.name = method + "_" + url // FIXME: hardcoded key pattern
 	m.RLock()
-	e.res.err, e.res.body = f(url, method, route)
+	e.res.body, e.res.err = f(url, method, route)
 	m.save(e.res.name, e.res.body)
 	m.RUnlock()
 

pkg/cache/memory_datastore.go

@@ -33,7 +33,7 @@ func NewMemoryDatastore() *MemoryDatastore {
 type MemoryDatastore struct {
 	cache map[string]interface{}
 	ts    map[string]TTLItem
-	lock sync.RWMutex
+	lock  sync.RWMutex
 }
 
 // SetKey function

pkg/cache/redis_datastore.go

@@ -10,12 +10,11 @@
 package cache
 
 import (
+	"github.com/go-redis/redis"
 	"log"
 	"os"
 	"strconv"
 	"time"
-
-	"github.com/go-redis/redis"
 )
 
 // NewRedisDatastore function

pkg/cache/response.go

@@ -23,6 +23,7 @@ type ResponseCache struct {
 	Headers *fasthttp.ResponseHeader
 }
 
+// NewResponseCache Creates new ResponseCache structure
 func NewResponseCache(url, method, body string, code int, headers *fasthttp.ResponseHeader) *ResponseCache {
 	return &ResponseCache{url, method, body, code, headers}
 }

pkg/cache/route.go

@@ -9,11 +9,13 @@
 
 package cache
 
+// RouteCache struct corresponding route cache item
 type RouteCache struct {
 	SourceURL string
 	TargetURL string
 }
 
+// NewRouteCache Creates new router instance
 func NewRouteCache(source, target string) *RouteCache {
 	return &RouteCache{
 		SourceURL: source,

pkg/client/http.go

@@ -7,6 +7,7 @@
 // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License
 // Repo: https://git.pbiernat.dev/golang/vegvisir
 
+// Package client all available clients lives here (httpo, https, grpc etc.)
 package client
 
 import "github.com/valyala/fasthttp"

pkg/config/config.go

@@ -7,6 +7,7 @@
 // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License
 // Repo: https://git.pbiernat.dev/golang/vegvisir
 
+// Package config main config file structs
 package config
 
 import (

pkg/handler/handler.go

@@ -7,4 +7,5 @@
 // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License
 // Repo: https://git.pbiernat.dev/golang/vegvisir
 
+// Package handler all handlers will be placed here in future...
 package handler

pkg/main.go

@@ -11,14 +11,14 @@ package main
 
 import (
 	"flag"
+	"git.pbiernat.dev/golang/vegvisir/pkg/server"
 	"log"
 	"net/http"
+	_ "net/http/pprof"
 	"os"
 	"runtime"
 	"runtime/pprof"
-	"vegvisir/pkg/server"
 )
-import _ "net/http/pprof"
 
 var (
 	cPath = flag.String("c", "vegvisir.json", "Path to config file")

pkg/server/router.go

@@ -12,27 +12,24 @@ package server
 
 import (
 	"fmt"
+	"git.pbiernat.dev/golang/vegvisir/pkg/cache"
+	"git.pbiernat.dev/golang/vegvisir/pkg/config"
 	"log"
 	"regexp"
 	"strings"
-	"vegvisir/pkg/cache"
-	"vegvisir/pkg/config"
 )
 
 // Router struct
 type Router struct {
 	config *config.Config
-	//cache  map[string]cache.RouteCache
 	cache  *cache.MemoryDatastore
-	//cache  *cache.RedisDatastore
 }
 
 // NewRouter function
-func NewRouter(config *config.Config, ds cache.Datastore, ttl int) *Router {
+func NewRouter(config *config.Config, ds *cache.MemoryDatastore, ttl int) *Router {
 	return &Router{
 		config: config,
-		//cache: make(map[string]cache.RouteCache),
+		cache:  ds,
-		cache: cache.NewMemoryDatastore(),
 		//cache: cache.NewRedisDatastore(config.Cache.Host, config.Cache.Password, config.Cache.Database, config.Cache.Port),
 	}
 }
@@ -65,7 +62,7 @@ func (r *Router) FindByRequestURL(url []byte) (bool, *cache.RouteCache) {
 
 				//routeJSON, _ := ffjson.Marshal(&route) // FIXME: TMP
 
-				if err := r.cache.SetKey("route_" + sURL, route, r.config.Cache.RouteTTL); err != nil {
+				if err := r.cache.SetKey("route_"+sURL, route, r.config.Cache.RouteTTL); err != nil {
 					// FIXME: ^^ use cache.Manager* after refactor
 					log.Println("Error saving route cache:", sURL)
 				}

pkg/server/server.go

@@ -7,20 +7,20 @@
 // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License
 // Repo: https://git.pbiernat.dev/golang/vegvisir
 
+// Package server contains main sever files
 package server
 
 import (
 	"context"
 	"fmt"
+	"git.pbiernat.dev/golang/vegvisir/pkg/cache"
+	"git.pbiernat.dev/golang/vegvisir/pkg/config"
+	"github.com/valyala/fasthttp"
 	"log"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
-	"vegvisir/pkg/cache"
-	"vegvisir/pkg/config"
-
-	"github.com/valyala/fasthttp"
 )
 
 const (
@@ -34,7 +34,7 @@ const (
 type Server struct {
 	config *config.Config
 	router *Router
-	respCM cache.Manager
+	respCM *cache.Manager
 	daemon *fasthttp.Server
 }
 
@@ -52,7 +52,7 @@ func NewServer(cPath string) *Server {
 		config: cfg,
 		router: NewRouter(cfg, cache.NewMemoryDatastore(), cfg.Cache.RouteTTL),
 		// ^^ INFO: Routes always use memoryCache cause low size and fast read time (optimalization)
-		respCM: cache.NewCachedManager(*datastore, "response_", cfg.Cache.ResponseTTL, cache.HttpRequestHandler),
+		respCM: cache.NewCachedManager(*datastore, "response_", cfg.Cache.ResponseTTL, cache.HTTPRequestHandler),
 		// ^^ FIXME: add handler detection option by config etc...
 	}
 }
@@ -65,12 +65,12 @@ func (s *Server) Run() {
 	go func() {
 		serverAddress := s.config.Server.Address + ":" + fmt.Sprint(s.config.Server.Port)
 		s.daemon = &fasthttp.Server{
-			Handler: 			  s.mainHandler,
+			Handler:            s.mainHandler,
-			ReadTimeout:          15 * time.Second,
+			ReadTimeout:        15 * time.Second,
-			WriteTimeout:         20 * time.Second,
+			WriteTimeout:       20 * time.Second,
-			MaxConnsPerIP:        500,
+			MaxConnsPerIP:      500,
-			MaxRequestsPerConn:   500,
+			MaxRequestsPerConn: 500,
-			IdleTimeout:		  15 * time.Second, // aka KeepAlive
+			IdleTimeout:        15 * time.Second, // aka KeepAlive
 			//CloseOnShutdown: true,
 		}