From f8b05510b02f756f35205e0075a61f64112ee4f2 Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Fri, 9 Jul 2021 15:15:54 +0200 Subject: [PATCH 1/8] Added Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 0418ba9..cdd5078 100644 --- a/Makefile +++ b/Makefile @@ -8,3 +8,4 @@ build: test: go test . + From 0675fc61d52f8b14db53762a6a6c2b10f7de23a5 Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Thu, 8 Jul 2021 22:32:49 +0200 Subject: [PATCH 2/8] Base config support --- config/config.go | 17 +++++++++++++++++ config/test/sample.json | 18 ++++++++++++++++++ go.mod | 2 ++ main.go | 6 ++++++ server/server.go | 31 +++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+) create mode 100644 config/config.go create mode 100644 config/test/sample.json create mode 100644 server/server.go diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..226151e --- /dev/null +++ b/config/config.go @@ -0,0 +1,17 @@ +package config + +type Config struct { + Server Server + Services map[string]Service +} + +type Server struct { + Address string + Port int +} + +type Service struct { + BaseURL string + ForwardTo string + Protocol string +} diff --git a/config/test/sample.json b/config/test/sample.json new file mode 100644 index 0000000..b1c6df8 --- /dev/null +++ b/config/test/sample.json @@ -0,0 +1,18 @@ +{ + "server": { + "address": "127.0.0.1", + "port": 8080 + }, + "services": { + "test-app1": { + "baseUrl": "test/", + "forwardTo": "127.0.0.1:3030/", + "protocol": "http" + }, + "test-app2": { + "baseUrl": "test2/", + "forwardTo": "httpbin.org/", + "protocol": "http" + } + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index b60d587..f9239a4 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module vegvisir go 1.13 + +require github.com/opentracing/opentracing-go v1.2.0 // indirect diff --git a/main.go b/main.go index da29a2c..a376935 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,10 @@ package main +import ( + "vegvisir/server" +) + func main() { + s := server.NewServer() + err := s.LoadConfig("config/test/sample.json") } diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..c9ffca7 --- /dev/null +++ b/server/server.go @@ -0,0 +1,31 @@ +package server + +import ( + "encoding/json" + "io/ioutil" + "os" + "vegvisir/config" +) + +type Server struct { + Config config.Config +} + +func NewServer() Server { + return Server{} +} + +func (s *Server) LoadConfig(file string) error { + if _, err := os.Stat(file); err != nil { + return err + } + + data, err := ioutil.ReadFile(file) + if err != nil { + return err + } + + json.Unmarshal(data, &s.Config) + + return nil +} From 8ba2c8b18def77e05285038936916140fa596b46 Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Fri, 9 Jul 2021 18:32:27 +0200 Subject: [PATCH 3/8] Added draft for starting server as separate thread --- main.go | 9 +++++++++ server/server.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 server/server.go diff --git a/main.go b/main.go index da29a2c..37f6fd6 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,13 @@ package main +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ + +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + func main() { } diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..229b060 --- /dev/null +++ b/server/server.go @@ -0,0 +1,51 @@ +package server + +import ( + "encoding/json" + "io/ioutil" + "os" + "vegvisir/config" +) + +type Server struct { + Config config.Config +} + +func NewServer() Server { + return Server{} +} + +func (s *Server) LoadConfig(file string) error { + if _, err := os.Stat(file); err != nil { + return err + } + + data, err := ioutil.ReadFile(file) + if err != nil { + return err + } + + json.Unmarshal(data, &s.Config) + + return nil +} + +// func Run(s *Server) { +// s.StartListener() + +// log.Println("Server started successfully...") + +// Wait for an interrupt +// c := make(chan os.Signal, 1) +// signal.Notify(c, os.Interrupt, os.Kill) +// <-c + +// log.Error("SIGKILL or SIGINT caught, shutting down...") + +// Attempt a graceful shutdown +// ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +// defer cancel() + +// s.shutdownServers(ctx) +// log.Error("all listeners shut down.") +// } From 9d897accb62d4d5dd038c5540629b9a5ee751f32 Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Sat, 10 Jul 2021 11:34:51 +0200 Subject: [PATCH 4/8] Added modd config --- .gitignore | 3 +++ modd.conf | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 .gitignore create mode 100644 modd.conf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..884e49b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +go.sum +vegvisir +vegvisir_tmp diff --git a/modd.conf b/modd.conf new file mode 100644 index 0000000..fd9a7bd --- /dev/null +++ b/modd.conf @@ -0,0 +1,5 @@ +# Exclude all test files of the form *_test.go +**/*.go **/vegvisir.json !**/*_test.go { + prep: go build -o vegvisir_tmp + daemon +sigterm: ./vegvisir_tmp +} From ba6b5b556418b0e84524633043d7c1100ab7bfea Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Sat, 10 Jul 2021 23:57:45 +0200 Subject: [PATCH 5/8] The seed of reverse-proxy functionality --- config/config.go | 30 +++++++-- config/test/sample.json | 18 ----- go.mod | 6 +- main.go | 10 ++- server/server.go | 143 ++++++++++++++++++++++++++++++++++------ vegvisir.json | 33 ++++++++++ 6 files changed, 194 insertions(+), 46 deletions(-) delete mode 100644 config/test/sample.json create mode 100644 vegvisir.json diff --git a/config/config.go b/config/config.go index 226151e..a82e0c0 100644 --- a/config/config.go +++ b/config/config.go @@ -1,8 +1,17 @@ package config +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ + +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + type Config struct { Server Server - Services map[string]Service + Backends map[string]Backend } type Server struct { @@ -10,8 +19,19 @@ type Server struct { Port int } -type Service struct { - BaseURL string - ForwardTo string - Protocol string +type Backend struct { + PrefixUrl string + BackendAddress string + Protocol string + Routes []Route +} + +type Route struct { + Pattern string + Target string +} + +var DefaultRoute = Route{ + Pattern: "(.*)", + Target: "", } diff --git a/config/test/sample.json b/config/test/sample.json deleted file mode 100644 index b1c6df8..0000000 --- a/config/test/sample.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "server": { - "address": "127.0.0.1", - "port": 8080 - }, - "services": { - "test-app1": { - "baseUrl": "test/", - "forwardTo": "127.0.0.1:3030/", - "protocol": "http" - }, - "test-app2": { - "baseUrl": "test2/", - "forwardTo": "httpbin.org/", - "protocol": "http" - } - } -} \ No newline at end of file diff --git a/go.mod b/go.mod index f9239a4..e2fcb7c 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,8 @@ module vegvisir go 1.13 -require github.com/opentracing/opentracing-go v1.2.0 // indirect +require ( + github.com/andybalholm/brotli v1.0.3 // indirect + github.com/klauspost/compress v1.13.1 // indirect + github.com/valyala/fasthttp v1.28.0 +) diff --git a/main.go b/main.go index 7fa23c6..3033f9d 100644 --- a/main.go +++ b/main.go @@ -10,10 +10,16 @@ package main // Repo: https://git.pbiernat.dev/golang/vegvisir import ( + "flag" "vegvisir/server" ) +var ( + cFile = flag.String("c", "vegvisir.json", "Path to config file") +) + func main() { - s := server.NewServer() - err := s.LoadConfig("config/test/sample.json") + flag.Parse() + + server.NewServer(*cFile).Run() } diff --git a/server/server.go b/server/server.go index 3f34598..33c9267 100644 --- a/server/server.go +++ b/server/server.go @@ -10,51 +10,154 @@ package server // Repo: https://git.pbiernat.dev/golang/vegvisir import ( + "context" "encoding/json" + "fmt" "io/ioutil" + "log" "os" + "os/signal" + "regexp" + "strings" + "time" "vegvisir/config" + + "github.com/valyala/fasthttp" +) + +const ( + Version = "0.1" + Name = "Vegvisir/" + Version ) type Server struct { Config config.Config + + cFilePath string + foundBackend *config.Backend + foundRoute *config.Route } -func NewServer() Server { - return Server{} +func NewServer(cPath string) *Server { + return &Server{ + cFilePath: cPath, + } } -func (s *Server) LoadConfig(file string) error { - if _, err := os.Stat(file); err != nil { +func (s *Server) Run() { + if err := s.loadConfig(); err != nil { + log.Fatalln("Unable to find config file: ", s.cFilePath, err) + } + + go func() { + serverAddress := s.Config.Server.Address + ":" + fmt.Sprint(s.Config.Server.Port) + if err := fasthttp.ListenAndServe(serverAddress, s.mainHandler); err != nil { + log.Fatalf("Server panic! Error message: %s", err) + } + }() + + log.Println("Server started") + + // Wait for an interrupt + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt, os.Kill) + <-interrupt + + log.Println("SIGKILL or SIGINT caught, shutting down...") + + // Attempt a graceful shutdown + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + s.Shutdown(ctx) + log.Println("Server shutdown successfully.") +} + +func (s *Server) Shutdown(ctx context.Context) { // TODO: wait for all connections to finish + log.Println("Shuting down finished") +} + +func (s *Server) loadConfig() error { + if _, err := os.Stat(s.cFilePath); err != nil { return err } - data, err := ioutil.ReadFile(file) + data, err := ioutil.ReadFile(s.cFilePath) if err != nil { return err } - json.Unmarshal(data, &s.Config) + err = json.Unmarshal(data, &s.Config) + if err != nil { + return err + } return nil } -// func Run(s *Server) { -// s.StartListener() +func (s *Server) mainHandler(ctx *fasthttp.RequestCtx) { + ctx.Response.Header.Add(fasthttp.HeaderServer, Name) -// log.Println("Server started successfully...") + // move all below logic to concrete handler or sth....? + reqURI, sReqURI, sReqMethod := ctx.RequestURI(), string(ctx.RequestURI()), string(ctx.Method()) + log.Println("Incomming request:", sReqMethod, sReqURI) -// Wait for an interrupt -// c := make(chan os.Signal, 1) -// signal.Notify(c, os.Interrupt, os.Kill) -// <-c + found, rgxp := s.findBackendAndRouteByRequestURI(reqURI) + if !found { + // FIXME: return 404/5xx error in repsonse, maybe define it in Backend config? + ctx.Response.SetStatusCode(fasthttp.StatusNotFound) + return + } -// log.Error("SIGKILL or SIGINT caught, shutting down...") + bckReqURI := s.buildBackendTargetURI(*rgxp, sReqURI) -// Attempt a graceful shutdown -// ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) -// defer cancel() + // prepare to send request to backend - separate + bckReq := fasthttp.AcquireRequest() + bckResp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(bckReq) + defer fasthttp.ReleaseResponse(bckResp) -// s.shutdownServers(ctx) -// log.Error("all listeners shut down.") -// } + // copy headers from backend response and prepare request for backend - separate + bckReq.SetRequestURI(bckReqURI) + bckReq.Header.SetMethod(sReqMethod) + + fasthttp.Do(bckReq, bckResp) + + ctx.Response.Header.SetBytesV(fasthttp.HeaderContentType, bckResp.Header.ContentType()) + ctx.SetBody(bckResp.Body()) +} + +func (s *Server) findBackendAndRouteByRequestURI(uri []byte) (bool, *regexp.Regexp) { + for _, bck := range s.Config.Backends { + if !strings.Contains(string(uri), bck.PrefixUrl) { + continue + } + s.foundBackend = &bck // possibly errorneus... + + for _, r := range bck.Routes { + rgxp := regexp.MustCompile(fmt.Sprintf("%s%s", bck.PrefixUrl, r.Pattern)) + // fmt.Println("pattern: ", bck.PrefixUrl+r.Pattern, rgxp.Match(uri)) + if rgxp.Match(uri) { + s.foundBackend = &bck // 100% safe here + s.foundRoute = &r + + return true, rgxp + } + } + } + + return false, nil +} + +func (s *Server) buildBackendTargetURI(rgxp regexp.Regexp, uri string) string { + baseURI := s.foundBackend.BackendAddress + url := rgxp.ReplaceAllString(uri, s.foundRoute.Target) + + // if !strings.HasSuffix(baseURI, "/") { + // baseURI += "/" + // } + + log.Println("Send req to backend url: ", baseURI+url) + + return baseURI + url +} diff --git a/vegvisir.json b/vegvisir.json new file mode 100644 index 0000000..8394cc5 --- /dev/null +++ b/vegvisir.json @@ -0,0 +1,33 @@ +{ + "server": { + "address": "127.0.0.1", + "port": 8080 + }, + "backends": { + "news-app": { + "prefixUrl": "news/", + "backendAddress": "http://127.0.0.1:3030", + "routes": [{ + "pattern": "new/(.*)", + "target": "$1" + },{ + "pattern": "new1/(.*)/([0-9]*)", + "target": "new-url1/$1/$2" + },{ + "pattern": "(.+)", + "target": "url" + }] + }, + "article-app": { + "prefixUrl": "art/", + "backendAddress": "http://127.0.0.1:3030", + "routes": [{ + "pattern": "art1/(.*)", + "target": "art-url1/$1" + },{ + "pattern": "([a-zA-Z0-9]*)", + "target": "article-global/$1" + }] + } + } +} \ No newline at end of file From b5e1d55c21d7b70b047518942f3d26d1375a23be Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Fri, 23 Jul 2021 02:00:02 +0200 Subject: [PATCH 6/8] Drone CI test config --- .drone.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..92f9976 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,9 @@ +kind: pipeline +type: docker +name: default + +steps: +- name: test + image: golang + commands: + - ls -lah \ No newline at end of file From 1200928fc195f00d9e001037d4ca6a6f1ee1a93c Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Fri, 23 Jul 2021 16:04:54 +0200 Subject: [PATCH 7/8] Update of environment and added docker container config --- .gitignore | 5 ++- Dockerfile | 11 ++++++ Makefile | 18 ++++++---- docker-compose.yml | 24 ++++++++++++++ go.mod | 6 ++++ go.sum | 83 ++++++++++++++++++++++++++++++++++++++++++++++ modd.conf | 3 +- vegvisir.json | 20 +++++++---- vegvisir.json.dist | 33 ++++++++++++++++++ 9 files changed, 185 insertions(+), 18 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 go.sum create mode 100644 vegvisir.json.dist diff --git a/.gitignore b/.gitignore index 884e49b..e895468 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -go.sum -vegvisir -vegvisir_tmp +vegvisir.json +*.prof diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2004bdc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.16-alpine + +RUN mkdir /go/src/vegvisir +WORKDIR /go/src/vegvisir + +COPY vegvisir.json go.mod go.sum ./ +COPY pkg/ ./pkg + +RUN go mod download + +CMD ["go", "run", "pkg/main.go"] \ No newline at end of file diff --git a/Makefile b/Makefile index cdd5078..eefd431 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,17 @@ - - run: - go run . + go run pkg/main.go + +run-profiler: + go run pkg/main.go -cpuprofile cpu.prof -memprofile mem.prof + +profile-cpu-web: + go tool pprof -http=127.0.0.1:9990 cpu.prof + +profile-mem-web: + go tool pprof -http=127.0.0.1:9991 mem.prof build: - go build . + go build pkg/main.go test: - go test . - + go test pkg/main.go diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d9b180b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3.7' + +services: + app: + container_name: vegvisir_app + build: . + ports: + - 8080:8080 + networks: + - vegvisir-network + links: + - redis + + redis: + container_name: vegvisir_redis + image: redis:6.2-alpine + ports: + - 6379:6379 + networks: + - vegvisir-network + +networks: + vegvisir-network: + driver: bridge diff --git a/go.mod b/go.mod index e2fcb7c..3664d03 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,12 @@ go 1.13 require ( github.com/andybalholm/brotli v1.0.3 // indirect + github.com/go-redis/redis v6.15.9+incompatible + github.com/google/go-cmp v0.5.6 // indirect github.com/klauspost/compress v1.13.1 // indirect + github.com/onsi/ginkgo v1.15.0 // indirect + github.com/onsi/gomega v1.10.5 // indirect github.com/valyala/fasthttp v1.28.0 + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..35d2356 --- /dev/null +++ b/go.sum @@ -0,0 +1,83 @@ +github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/modd.conf b/modd.conf index fd9a7bd..900dd04 100644 --- a/modd.conf +++ b/modd.conf @@ -1,5 +1,4 @@ # Exclude all test files of the form *_test.go **/*.go **/vegvisir.json !**/*_test.go { - prep: go build -o vegvisir_tmp - daemon +sigterm: ./vegvisir_tmp + daemon +sigterm: make run } diff --git a/vegvisir.json b/vegvisir.json index 8394cc5..7bd5829 100644 --- a/vegvisir.json +++ b/vegvisir.json @@ -6,13 +6,13 @@ "backends": { "news-app": { "prefixUrl": "news/", - "backendAddress": "http://127.0.0.1:3030", + "backendAddress": "http://172.17.0.1:3030", "routes": [{ + "pattern": "new/(.*)/([0-9]*)", + "target": "new-url/$1/$2" + },{ "pattern": "new/(.*)", "target": "$1" - },{ - "pattern": "new1/(.*)/([0-9]*)", - "target": "new-url1/$1/$2" },{ "pattern": "(.+)", "target": "url" @@ -20,14 +20,20 @@ }, "article-app": { "prefixUrl": "art/", - "backendAddress": "http://127.0.0.1:3030", + "backendAddress": "http://172.17.0.1:3030", "routes": [{ "pattern": "art1/(.*)", - "target": "art-url1/$1" + "target": "art-url/$1" },{ "pattern": "([a-zA-Z0-9]*)", - "target": "article-global/$1" + "target": "art-global/$1" }] } + }, + "cache": { + "ttl": 300, + "route_ttl": 300, + "response_ttl": 300, + "headers_ttl": 300 } } \ No newline at end of file diff --git a/vegvisir.json.dist b/vegvisir.json.dist new file mode 100644 index 0000000..8394cc5 --- /dev/null +++ b/vegvisir.json.dist @@ -0,0 +1,33 @@ +{ + "server": { + "address": "127.0.0.1", + "port": 8080 + }, + "backends": { + "news-app": { + "prefixUrl": "news/", + "backendAddress": "http://127.0.0.1:3030", + "routes": [{ + "pattern": "new/(.*)", + "target": "$1" + },{ + "pattern": "new1/(.*)/([0-9]*)", + "target": "new-url1/$1/$2" + },{ + "pattern": "(.+)", + "target": "url" + }] + }, + "article-app": { + "prefixUrl": "art/", + "backendAddress": "http://127.0.0.1:3030", + "routes": [{ + "pattern": "art1/(.*)", + "target": "art-url1/$1" + },{ + "pattern": "([a-zA-Z0-9]*)", + "target": "article-global/$1" + }] + } + } +} \ No newline at end of file From ea6c90236aa3713a036280a24eebbeaf7757db45 Mon Sep 17 00:00:00 2001 From: Piotr Biernat Date: Fri, 23 Jul 2021 16:10:37 +0200 Subject: [PATCH 8/8] Renamed src/ to pkg/ Added cache with base redis support Added profiling in main.go --- main.go => pkg/cache/datastore.go | 19 +--- pkg/cache/redis_datastore.go | 57 ++++++++++ pkg/cache/response.go | 65 +++++++++++ pkg/cache/route.go | 63 +++++++++++ pkg/client/http.go | 10 ++ pkg/client/tcp.go | 10 ++ {config => pkg/config}/config.go | 30 ++++- pkg/handler/handler.go | 10 ++ pkg/main.go | 59 ++++++++++ pkg/server/server.go | 176 ++++++++++++++++++++++++++++++ pkg/server/struct.go | 10 ++ server/server.go | 163 --------------------------- 12 files changed, 492 insertions(+), 180 deletions(-) rename main.go => pkg/cache/datastore.go (67%) create mode 100644 pkg/cache/redis_datastore.go create mode 100644 pkg/cache/response.go create mode 100644 pkg/cache/route.go create mode 100644 pkg/client/http.go create mode 100644 pkg/client/tcp.go rename {config => pkg/config}/config.go (71%) create mode 100644 pkg/handler/handler.go create mode 100644 pkg/main.go create mode 100644 pkg/server/server.go create mode 100644 pkg/server/struct.go delete mode 100644 server/server.go diff --git a/main.go b/pkg/cache/datastore.go similarity index 67% rename from main.go rename to pkg/cache/datastore.go index 3033f9d..f61593d 100644 --- a/main.go +++ b/pkg/cache/datastore.go @@ -1,25 +1,16 @@ -package main - // ___ ____ ___ ___ // \ \ / / | _ | __| \ \ / / || | __ || || _ | // \ \/ / |___ | |__ \ \/ / || |___ || ||___| // \ / | _ | _ | \ / || __ | || ||\\ // \/ |___ |___ | \/ || ____| || || \\ - +// // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License // Repo: https://git.pbiernat.dev/golang/vegvisir -import ( - "flag" - "vegvisir/server" -) +package cache -var ( - cFile = flag.String("c", "vegvisir.json", "Path to config file") -) +type CacheDatastore interface { + SetKey(string, interface{}, int) error -func main() { - flag.Parse() - - server.NewServer(*cFile).Run() + GetKey(string) (interface{}, error) } diff --git a/pkg/cache/redis_datastore.go b/pkg/cache/redis_datastore.go new file mode 100644 index 0000000..4fac007 --- /dev/null +++ b/pkg/cache/redis_datastore.go @@ -0,0 +1,57 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ +// +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package cache + +import ( + "strconv" + "time" + + "github.com/go-redis/redis" +) + +func NewRedisDatastore(host string, port int) *RedisDatastore { + return &RedisDatastore{ + client: redis.NewClient(&redis.Options{ + Addr: host + ":" + strconv.Itoa(port), + Password: "", // FIXME: use env or param + DB: 0, // FIXME: use env or param + }), + cache: make(map[string]interface{}), + } +} + +type RedisDatastore struct { + client *redis.Client + cache map[string]interface{} +} + +func (ds *RedisDatastore) SetKey(key string, data interface{}, ttl int) error { + err := ds.client.Set(key, data, time.Duration(ttl)*time.Second).Err() + if err != nil { + return err + } + + // ds.cache[key] = data + + return nil +} + +func (ds *RedisDatastore) GetKey(key string) (interface{}, error) { + // if data, ok := ds.cache[key]; ok { + // return data, nil + // } + + data, err := ds.client.Get(key).Result() + if err != nil { + return nil, err + } + + return data, nil +} diff --git a/pkg/cache/response.go b/pkg/cache/response.go new file mode 100644 index 0000000..c213157 --- /dev/null +++ b/pkg/cache/response.go @@ -0,0 +1,65 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ +// +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package cache + +import ( + "encoding/json" + "log" +) + +type ResponseCache struct { + URL string + Body string + // Headers map[string]string +} + +type ResponseCacheManager struct { + datastore CacheDatastore + prefix string + ttl int +} + +func NewResponseCacheManager(datastore CacheDatastore, ttl int) ResponseCacheManager { + return ResponseCacheManager{ + datastore: datastore, + prefix: "response_", + ttl: ttl, + } +} + +func (rm *ResponseCacheManager) Save(name string, r ResponseCache) { + data, err := json.Marshal(r) + if err != nil { + log.Println("JSON:", err) // FIXME + } + + name = rm.prefix + name + err = rm.datastore.SetKey(name, string(data), rm.ttl) + if err != nil { + log.Println("REDIS:", err, name) // FIXME + } +} + +func (rm *ResponseCacheManager) Load(name string) (bool, *ResponseCache) { + name = rm.prefix + name + + data, err := rm.datastore.GetKey(name) + if err != nil { + return false, &ResponseCache{} + } + + rc := &ResponseCache{} + err = json.Unmarshal([]byte(data.(string)), &rc) + if err != nil { + log.Println("JSON:", err) // FIXME + } + + return true, rc +} diff --git a/pkg/cache/route.go b/pkg/cache/route.go new file mode 100644 index 0000000..ff28a1c --- /dev/null +++ b/pkg/cache/route.go @@ -0,0 +1,63 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ +// +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package cache + +import ( + "encoding/json" + "log" +) + +type RouteCache struct { + SourceUrl string + TargetUrl string +} + +type RouteCacheManager struct { + datastore CacheDatastore + ttl int +} + +func NewRouteCacheManager(datastore CacheDatastore, ttl int) RouteCacheManager { + return RouteCacheManager{ + datastore: datastore, + ttl: ttl, + } +} + +func (rm *RouteCacheManager) Save(name string, r RouteCache) { + data, err := json.Marshal(r) + if err != nil { + log.Println("JSON:", err) // FIXME + } + + err = rm.datastore.SetKey("route_"+name, data, rm.ttl) + if err != nil { + log.Println("REDIS:", err, name) // FIXME + } +} + +func (rm *RouteCacheManager) Load(name string) (bool, RouteCache) { + name = "route_" + name + + data, err := rm.datastore.GetKey(name) + if err != nil { + log.Println("REDIS:", err, name) // FIXME + + return false, RouteCache{} + } + + rc := RouteCache{} + err = json.Unmarshal([]byte(data.(string)), &rc) + if err != nil { + log.Println("JSON:", err) // FIXME + } + + return true, rc +} diff --git a/pkg/client/http.go b/pkg/client/http.go new file mode 100644 index 0000000..0510e9a --- /dev/null +++ b/pkg/client/http.go @@ -0,0 +1,10 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ +// +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package client diff --git a/pkg/client/tcp.go b/pkg/client/tcp.go new file mode 100644 index 0000000..4bd534b --- /dev/null +++ b/pkg/client/tcp.go @@ -0,0 +1,10 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ + +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package client diff --git a/config/config.go b/pkg/config/config.go similarity index 71% rename from config/config.go rename to pkg/config/config.go index a82e0c0..6fb1785 100644 --- a/config/config.go +++ b/pkg/config/config.go @@ -1,14 +1,20 @@ -package config - // ___ ____ ___ ___ // \ \ / / | _ | __| \ \ / / || | __ || || _ | // \ \/ / |___ | |__ \ \/ / || |___ || ||___| // \ / | _ | _ | \ / || __ | || ||\\ // \/ |___ |___ | \/ || ____| || || \\ - +// // Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License // Repo: https://git.pbiernat.dev/golang/vegvisir +package config + +import ( + "encoding/json" + "io/ioutil" + "os" +) + type Config struct { Server Server Backends map[string]Backend @@ -35,3 +41,21 @@ var DefaultRoute = Route{ Pattern: "(.*)", Target: "", } + +func (c *Config) Load(cPath string) error { + if _, err := os.Stat(cPath); err != nil { + return err + } + + data, err := ioutil.ReadFile(cPath) + if err != nil { + return err + } + + err = json.Unmarshal(data, &c) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go new file mode 100644 index 0000000..7e14d54 --- /dev/null +++ b/pkg/handler/handler.go @@ -0,0 +1,10 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ + +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package handler diff --git a/pkg/main.go b/pkg/main.go new file mode 100644 index 0000000..27e15bf --- /dev/null +++ b/pkg/main.go @@ -0,0 +1,59 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ +// +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package main + +import ( + "flag" + "log" + "os" + "runtime" + "runtime/pprof" + "vegvisir/pkg/server" +) + +var ( + cFile = flag.String("c", "vegvisir.json", "Path to config file") + + // for profiling... + cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`") + memprofile = flag.String("memprofile", "", "write memory profile to `file`") +) + +func main() { + + flag.Parse() + // cpu profiling + if *cpuprofile != "" { + f, err := os.Create(*cpuprofile) + if err != nil { + log.Fatal("could not create CPU profile: ", err) + } + defer f.Close() // error handling omitted for example + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("could not start CPU profile: ", err) + } + defer pprof.StopCPUProfile() + } + + server.NewServer(*cFile).Run() + + // memory profiling + if *memprofile != "" { + f, err := os.Create(*memprofile) + if err != nil { + log.Fatal("could not create memory profile: ", err) + } + defer f.Close() // error handling omitted for example + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + log.Fatal("could not write memory profile: ", err) + } + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go new file mode 100644 index 0000000..ee9f769 --- /dev/null +++ b/pkg/server/server.go @@ -0,0 +1,176 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ +// +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package server + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "regexp" + "strings" + "time" + "vegvisir/pkg/cache" + "vegvisir/pkg/config" + + "github.com/valyala/fasthttp" +) + +const ( + Version = "0.1" + Name = "Vegvisir/" + Version +) + +type Server struct { + Config config.Config + + cFilePath string + rCache map[string]cache.RouteCache // Internal route cache + routeCM cache.RouteCacheManager // Redis route cache + respCM cache.ResponseCacheManager // Redis response cache +} + +func NewServer(cPath string) *Server { + datastore := cache.NewRedisDatastore("127.0.0.7", 6379) // FIXME use config or env... + + return &Server{ + cFilePath: cPath, + rCache: make(map[string]cache.RouteCache), + routeCM: cache.NewRouteCacheManager(datastore, 30), //FIXME use ttl(seconds) from config or env... + respCM: cache.NewResponseCacheManager(datastore, 30), //FIXME use ttl(seconds) from config or env... + } +} + +func (s *Server) Run() { + if err := s.Config.Load(s.cFilePath); err != nil { + log.Fatalln("Unable to find config file: ", s.cFilePath, err) + } + + go func() { + serverAddress := s.Config.Server.Address + ":" + fmt.Sprint(s.Config.Server.Port) + if err := fasthttp.ListenAndServe(serverAddress, s.mainHandler); err != nil { + log.Fatalf("Server panic! Error message: %s", err) + } + }() + + log.Println("Server started") + + // Wait for an interrupt + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt, os.Kill) + <-interrupt + + log.Println("SIGKILL or SIGINT caught, shutting down...") + + // Attempt a graceful shutdown + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + s.Shutdown(ctx) + log.Println("Server shutdown successfully.") +} + +func (s *Server) Shutdown(ctx context.Context) { // TODO: wait for all connections to finish + log.Println("Shuting down finished") +} + +func (s *Server) mainHandler(ctx *fasthttp.RequestCtx) { + ctx.Response.Header.Add(fasthttp.HeaderServer, Name) + + // move all below logic to concrete handler or sth.... + reqUri, sReqUri, sReqMethod := ctx.RequestURI(), string(ctx.RequestURI()), string(ctx.Method()) + log.Println("Incomming request:", sReqMethod, sReqUri) + + found, route := s.findRouteByRequestURI(reqUri) + if !found { + // FIXME: return 404/5xx error in repsonse, maybe define it in Backend config? + ctx.SetStatusCode(fasthttp.StatusNotFound) + return + } + + // handle response caching + if ok, data := s.respCM.Load(sReqUri); ok { + log.Println("Read resp from cache: ", route.TargetUrl) + + ctx.SetBody([]byte(data.Body)) + } else { + log.Println("Send req to backend url: ", route.TargetUrl) + + // prepare to send request to backend - separate + bckReq := fasthttp.AcquireRequest() + bckResp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseRequest(bckReq) + defer fasthttp.ReleaseResponse(bckResp) + + // copy headers from backend response and prepare request for backend - separate + bckReq.SetRequestURI(route.TargetUrl) + bckReq.Header.SetMethod(sReqMethod) + + err := fasthttp.Do(bckReq, bckResp) + if err != nil { + ctx.SetStatusCode(fasthttp.StatusInternalServerError) + return + } + + ctx.Response.Header.SetBytesV(fasthttp.HeaderContentType, bckResp.Header.ContentType()) + ctx.SetStatusCode(bckResp.StatusCode()) + ctx.SetBody(bckResp.Body()) + + // save response to cache + respCache := cache.ResponseCache{ + URL: sReqUri, + Body: string(bckResp.Body()), + // Headers: [] + } // FIXME: prepare resp cache struct in respCM.Save method or other service... + s.respCM.Save(sReqUri, respCache) + } +} + +func (s *Server) findRouteByRequestURI(uri []byte) (bool, cache.RouteCache) { + var sUri string = string(uri) + + for bId := range s.Config.Backends { + bck := s.Config.Backends[bId] + if !strings.Contains(sUri, bck.PrefixUrl) { + continue + } + + for rId := range bck.Routes { + route := &bck.Routes[rId] + + // if ok, cRoute := s.rCacheManager.Load(sUri); ok { + // return true, cRoute + // } + if cRoute, ok := s.rCache[sUri]; ok { + return true, cRoute + } + + rgxp := regexp.MustCompile(fmt.Sprintf("%s%s", bck.PrefixUrl, route.Pattern)) + if rgxp.Match(uri) { + targetUrl := bck.BackendAddress + rgxp.ReplaceAllString(sUri, route.Target) + + cRoute := cache.RouteCache{ // FIXME: data duplication and use short alias for backend and route! + // Backend: bck, + // Route: *route, + SourceUrl: sUri, + TargetUrl: targetUrl, + } + + // s.rCacheManager.Save(sUri, cRoute) + s.rCache[sUri] = cRoute + + return true, cRoute + } + } + } + + return false, cache.RouteCache{} +} diff --git a/pkg/server/struct.go b/pkg/server/struct.go new file mode 100644 index 0000000..8dbb0e4 --- /dev/null +++ b/pkg/server/struct.go @@ -0,0 +1,10 @@ +// ___ ____ ___ ___ +// \ \ / / | _ | __| \ \ / / || | __ || || _ | +// \ \/ / |___ | |__ \ \/ / || |___ || ||___| +// \ / | _ | _ | \ / || __ | || ||\\ +// \/ |___ |___ | \/ || ____| || || \\ +// +// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License +// Repo: https://git.pbiernat.dev/golang/vegvisir + +package server diff --git a/server/server.go b/server/server.go deleted file mode 100644 index 33c9267..0000000 --- a/server/server.go +++ /dev/null @@ -1,163 +0,0 @@ -package server - -// ___ ____ ___ ___ -// \ \ / / | _ | __| \ \ / / || | __ || || _ | -// \ \/ / |___ | |__ \ \/ / || |___ || ||___| -// \ / | _ | _ | \ / || __ | || ||\\ -// \/ |___ |___ | \/ || ____| || || \\ - -// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License -// Repo: https://git.pbiernat.dev/golang/vegvisir - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "os" - "os/signal" - "regexp" - "strings" - "time" - "vegvisir/config" - - "github.com/valyala/fasthttp" -) - -const ( - Version = "0.1" - Name = "Vegvisir/" + Version -) - -type Server struct { - Config config.Config - - cFilePath string - foundBackend *config.Backend - foundRoute *config.Route -} - -func NewServer(cPath string) *Server { - return &Server{ - cFilePath: cPath, - } -} - -func (s *Server) Run() { - if err := s.loadConfig(); err != nil { - log.Fatalln("Unable to find config file: ", s.cFilePath, err) - } - - go func() { - serverAddress := s.Config.Server.Address + ":" + fmt.Sprint(s.Config.Server.Port) - if err := fasthttp.ListenAndServe(serverAddress, s.mainHandler); err != nil { - log.Fatalf("Server panic! Error message: %s", err) - } - }() - - log.Println("Server started") - - // Wait for an interrupt - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt, os.Kill) - <-interrupt - - log.Println("SIGKILL or SIGINT caught, shutting down...") - - // Attempt a graceful shutdown - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - s.Shutdown(ctx) - log.Println("Server shutdown successfully.") -} - -func (s *Server) Shutdown(ctx context.Context) { // TODO: wait for all connections to finish - log.Println("Shuting down finished") -} - -func (s *Server) loadConfig() error { - if _, err := os.Stat(s.cFilePath); err != nil { - return err - } - - data, err := ioutil.ReadFile(s.cFilePath) - if err != nil { - return err - } - - err = json.Unmarshal(data, &s.Config) - if err != nil { - return err - } - - return nil -} - -func (s *Server) mainHandler(ctx *fasthttp.RequestCtx) { - ctx.Response.Header.Add(fasthttp.HeaderServer, Name) - - // move all below logic to concrete handler or sth....? - reqURI, sReqURI, sReqMethod := ctx.RequestURI(), string(ctx.RequestURI()), string(ctx.Method()) - log.Println("Incomming request:", sReqMethod, sReqURI) - - found, rgxp := s.findBackendAndRouteByRequestURI(reqURI) - if !found { - // FIXME: return 404/5xx error in repsonse, maybe define it in Backend config? - ctx.Response.SetStatusCode(fasthttp.StatusNotFound) - return - } - - bckReqURI := s.buildBackendTargetURI(*rgxp, sReqURI) - - // prepare to send request to backend - separate - bckReq := fasthttp.AcquireRequest() - bckResp := fasthttp.AcquireResponse() - defer fasthttp.ReleaseRequest(bckReq) - defer fasthttp.ReleaseResponse(bckResp) - - // copy headers from backend response and prepare request for backend - separate - bckReq.SetRequestURI(bckReqURI) - bckReq.Header.SetMethod(sReqMethod) - - fasthttp.Do(bckReq, bckResp) - - ctx.Response.Header.SetBytesV(fasthttp.HeaderContentType, bckResp.Header.ContentType()) - ctx.SetBody(bckResp.Body()) -} - -func (s *Server) findBackendAndRouteByRequestURI(uri []byte) (bool, *regexp.Regexp) { - for _, bck := range s.Config.Backends { - if !strings.Contains(string(uri), bck.PrefixUrl) { - continue - } - s.foundBackend = &bck // possibly errorneus... - - for _, r := range bck.Routes { - rgxp := regexp.MustCompile(fmt.Sprintf("%s%s", bck.PrefixUrl, r.Pattern)) - // fmt.Println("pattern: ", bck.PrefixUrl+r.Pattern, rgxp.Match(uri)) - if rgxp.Match(uri) { - s.foundBackend = &bck // 100% safe here - s.foundRoute = &r - - return true, rgxp - } - } - } - - return false, nil -} - -func (s *Server) buildBackendTargetURI(rgxp regexp.Regexp, uri string) string { - baseURI := s.foundBackend.BackendAddress - url := rgxp.ReplaceAllString(uri, s.foundRoute.Target) - - // if !strings.HasSuffix(baseURI, "/") { - // baseURI += "/" - // } - - log.Println("Send req to backend url: ", baseURI+url) - - return baseURI + url -}