[fix] Better usage of goroutines - beta

This commit is contained in:
Piotr Biernat 2021-11-10 01:36:33 +01:00
parent 4f10933caa
commit 8a4429cb3f
17 changed files with 2045 additions and 329 deletions

4
.gitignore vendored
View File

@ -1,2 +1,6 @@
.idea
vegvisir.json vegvisir.json
*.prof *.prof
*.local*

View File

@ -1,6 +1,9 @@
run: run:
go run pkg/main.go go run pkg/main.go
race:
go run --race pkg/main.go
run-profiler: run-profiler:
go run pkg/main.go -cpuprofile cpu.prof -memprofile mem.prof go run pkg/main.go -cpuprofile cpu.prof -memprofile mem.prof

View File

@ -14,6 +14,9 @@ services:
redis: redis:
container_name: vegvisir_redis container_name: vegvisir_redis
image: redis:6.2-alpine image: redis:6.2-alpine
volumes:
- ./docker/etc/redis.conf:/etc/redis.conf
command: "/usr/local/bin/redis-server /etc/redis.conf"
ports: ports:
- 6379:6379 - 6379:6379
networks: networks:

1630
docker/etc/redis.conf Normal file

File diff suppressed because it is too large Load Diff

20
go.mod
View File

@ -1,15 +1,17 @@
module vegvisir module vegvisir
go 1.13 go 1.17
require ( require (
github.com/andybalholm/brotli v1.0.3 // indirect
github.com/go-redis/redis v6.15.9+incompatible github.com/go-redis/redis v6.15.9+incompatible
github.com/google/go-cmp v0.5.6 // indirect github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7
github.com/klauspost/compress v1.13.1 // indirect github.com/valyala/fasthttp v1.31.0
github.com/onsi/ginkgo v1.15.0 // indirect )
github.com/onsi/gomega v1.10.5 // indirect
github.com/valyala/fasthttp v1.28.0 require (
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect github.com/andybalholm/brotli v1.0.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect github.com/klauspost/compress v1.13.4 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.17.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
) )

49
go.sum
View File

@ -1,43 +1,51 @@
github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E=
github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20=
github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.28.0 h1:ruVmTmZaBR5i67NqnjvvH5gEv0zwHfWtbjoyW98iho4= github.com/valyala/fasthttp v1.31.0 h1:lrauRLII19afgCs2fnWRJ4M5IkV0lo2FqA61uGkNBfE=
github.com/valyala/fasthttp v1.28.0/go.mod h1:cmWIqlu99AO/RKcp1HWaViTqc57FswJOfYYdPJBl8BA= github.com/valyala/fasthttp v1.31.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@ -50,8 +58,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I= golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -68,9 +76,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@ -82,20 +89,20 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

9
pkg/cache/cache.go vendored
View File

@ -28,12 +28,13 @@ type Datastore interface {
IsConnected() bool IsConnected() bool
} }
// GetCacheDatastore function // GetDatastore function
func GetCacheDatastore(config config.Cache) *Datastore { func GetDatastore(cfg config.Cache) *Datastore {
var datastore Datastore var datastore Datastore
if config.Type == typeRedis { log.Printf("Cache datastore type: %s", cfg.Type)
datastore = NewRedisDatastore(config.Host, config.Port) if cfg.Type == typeRedis {
datastore = NewRedisDatastore(cfg.Host, cfg.Password, cfg.Database, cfg.Port)
} else { } else {
datastore = NewMemoryDatastore() datastore = NewMemoryDatastore()
} }

201
pkg/cache/manager.go vendored Normal file
View File

@ -0,0 +1,201 @@
// ___ ____ ___ ___
// \ \ / / | _ | __| \ \ / / || | __ || || _ |
// \ \/ / |___ | |__ \ \/ / || |___ || ||___|
// \ / | _ | _ | \ / || __ | || ||\\
// \/ |___ |___ | \/ || ____| || || \\
//
// Copyright (c) 2021 Piotr Biernat. https://pbiernat.dev. MIT License
// Repo: https://git.pbiernat.dev/golang/vegvisir
// Based on:
// Package memo provides a concurrency-safe non-blocking memoization
// of a function. Requests for different keys proceed in parallel.
// Concurrent requests for the same url block until the first completes.
// This implementation uses a monitor goroutine.
// See page 278.
// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/
package cache
import (
"github.com/pquerna/ffjson/ffjson"
"github.com/valyala/fasthttp"
"log"
"sync"
"time"
)
type Manager struct {
sync.RWMutex
datastore Datastore
prefix string
ttl int
queue *queue
}
type result struct {
name string // cache item key
body *ResponseCache // cache item body FIXME: use interface to cover all types of cache items
err error
}
type entry struct {
res result
ready chan struct{} // closed when res is ready
}
type HandlerFunc func(url, method string, route *RouteCache) (error, *ResponseCache)
type queue struct {
items chan queueItem
}
type queueItem struct {
url string
method string
route *RouteCache
// FIXME: Refactor ^^
//key string
response chan<- result // the client wants a single result
}
//var reqsSend = 0
func NewCachedManager(datastore Datastore, prefix string, ttl int, handler HandlerFunc) Manager {
//log.Printf("Create DS: %s %d", prefix, ttl)
m := Manager{
datastore: datastore,
prefix: prefix,
ttl: ttl,
queue: &queue{items: make(chan queueItem)},
}
go m.server(handler)
return m
}
func HttpRequestHandler(url, method string, route *RouteCache) (error, *ResponseCache) { // FIXME: Refactor|Move to handler/
//start := time.Now()
bckReq := fasthttp.AcquireRequest()
bckResp := fasthttp.AcquireResponse()
defer fasthttp.ReleaseRequest(bckReq)
defer fasthttp.ReleaseResponse(bckResp)
// copy headers from backend response and prepare request for backend - separate
bckReq.SetRequestURI(route.TargetURL)
bckReq.Header.SetMethod(method)
err := fasthttp.Do(bckReq, bckResp)
if err != nil {
return err, nil
}
body, code := string(bckResp.Body()), bckResp.StatusCode()
//log.Printf("%s, %s, %d bytes\n", url, time.Since(start), len(body))
// save response to cache
respCache := NewResponseCache(url, method, body, code, &bckResp.Header)
// ^^ FIXME: rename NewResponseCache(with all depend struct) and move to handler/ (without dependencies)
//reqsSend++
log.Println(time.Now().Nanosecond(), "HttpRequestHandler() done:", route.TargetURL, url)
return nil, respCache
}
// FIXME: #68 - refactor
func (m *Manager) Fetch(url, method string, route *RouteCache) (*ResponseCache, error) {
//log.Println("Response CM: Fetch()")
response := make(chan result)
m.queue.items <- queueItem{url, method, route, response}
res := <-response
return res.body, res.err
}
func (m *Manager) Close() {
close(m.queue.items)
}
func (m *Manager) load(url, method string, route *RouteCache/*, output interface{}*/) (bool, *entry) { // FIXME second return type( interface{} )
//log.Println("Response CM: load()")
key := m.prefix + method + "_" + url
var data, err = m.datastore.GetKey(key)
if err != nil {
//log.Println(time.Now().Nanosecond(), "Cache item:", key, "not found")
return false, &entry{ready: make(chan struct{})}
}
response := &ResponseCache{} // FIXME #67: Simplify
jsonErr := ffjson.Unmarshal([]byte(data.(string)), response)
if jsonErr != nil {
log.Println("Converting error: ", err) // FIXME
return false, &entry{ready: make(chan struct{})}
}
e := &entry{result{key, response, nil}, make(chan struct{})}
close(e.ready)
return true, e
}
func (m *Manager) save(name string, r interface{}) bool { // FIXME second argument type( interface{} )
//log.Println(time.Now().Nanosecond(), "Saving Response")
data, err := ffjson.Marshal(&r)
if err != nil {
log.Println("JSON:", err) // FIXME
return false
}
name = m.prefix + name
err = m.datastore.SetKey(name, string(data), m.ttl)
//log.Printf("%d Saved cache in to ds: %T %s", time.Now().Nanosecond(), m.datastore, name)
if err != nil {
log.Println("Error saving cache item:", name) // FIXME
return false
}
return true
}
// FIXME: #68 - refactor
// FIXME:
// crashing on https://vps.pbiernat.dev/test-api/news (missing keep-alive header or some http/2 issue...)
// no-errors when Connection: close header is applied on remote
func (m *Manager) server(handler HandlerFunc) {
for item := range m.queue.items {
m.Lock()
ok, e := m.load(item.url, item.method, item.route/*, &ResponseCache{}*/) // FIXME: &responseCache{} tmp fix
m.Unlock()
if !ok {
//e = &entry{ready: make(chan struct{})/*, called: make(chan struct{})*/}
//log.Println("e.call:", item.url, item.method, item.route)
go m.call(e, handler, item.url, item.method, item.route)
}
//log.Println("e.deliver:", item.response)
go m.deliver(e, item.response)
}
}
func (m *Manager) call(e *entry, f HandlerFunc, url, method string, route *RouteCache) {
e.res.name = method + "_" + url // FIXME: hardcoded key pattern
m.RLock()
e.res.err, e.res.body = f(url, method, route)
m.save(e.res.name, e.res.body)
m.RUnlock()
close(e.ready)
}
func (m *Manager) deliver(e *entry, response chan<- result) {
<-e.ready
response <- e.res
}

View File

@ -11,6 +11,7 @@ package cache
import ( import (
"errors" "errors"
"sync"
"time" "time"
) )
@ -32,23 +33,27 @@ func NewMemoryDatastore() *MemoryDatastore {
type MemoryDatastore struct { type MemoryDatastore struct {
cache map[string]interface{} cache map[string]interface{}
ts map[string]TTLItem ts map[string]TTLItem
lock sync.RWMutex
} }
// SetKey function // SetKey function
func (ds *MemoryDatastore) SetKey(key string, data interface{}, ttl int) error { func (ds *MemoryDatastore) SetKey(key string, data interface{}, ttl int) error {
ds.lock.Lock()
ds.cache[key] = data ds.cache[key] = data
ds.ts[key] = TTLItem{ ds.ts[key] = TTLItem{
ts: time.Now().Second(), ts: time.Now().Second(),
ttl: ttl, ttl: ttl,
} }
ds.lock.Unlock()
return nil return nil
} }
// GetKey function // GetKey function
func (ds *MemoryDatastore) GetKey(key string) (interface{}, error) { func (ds *MemoryDatastore) GetKey(key string) (interface{}, error) {
ds.lock.RLock()
defer ds.lock.RUnlock()
ds.gc(key) // remove key is time of creation is outdated ds.gc(key) // remove key is time of creation is outdated
if data, ok := ds.cache[key]; ok { if data, ok := ds.cache[key]; ok {
return data, nil return data, nil
} }

View File

@ -10,6 +10,8 @@
package cache package cache
import ( import (
"log"
"os"
"strconv" "strconv"
"time" "time"
@ -17,12 +19,18 @@ import (
) )
// NewRedisDatastore function // NewRedisDatastore function
func NewRedisDatastore(host string, port int) *RedisDatastore { func NewRedisDatastore(host, password, db string, port int) *RedisDatastore {
dbNum, err := strconv.Atoi(db)
if err != nil {
log.Println("Config: Invalid redis database!")
os.Exit(1) // FIXME: move up so in main we use os.Exit ONLY!
}
return &RedisDatastore{ return &RedisDatastore{
client: redis.NewClient(&redis.Options{ client: redis.NewClient(&redis.Options{
Addr: host + ":" + strconv.Itoa(port), Addr: host + ":" + strconv.Itoa(port),
Password: "", // FIXME: use env or param Password: password,
DB: 0, // FIXME: use env or param DB: dbNum,
}), }),
} }
} }

77
pkg/cache/response.go vendored
View File

@ -9,10 +9,7 @@
package cache package cache
import ( import "github.com/valyala/fasthttp"
"encoding/json"
"log"
)
// Headers type // Headers type
type Headers map[string]string type Headers map[string]string
@ -20,68 +17,20 @@ type Headers map[string]string
// ResponseCache struct // ResponseCache struct
type ResponseCache struct { type ResponseCache struct {
URL string URL string
Method string
Body string Body string
Headers Headers Code int
Headers *fasthttp.ResponseHeader
} }
// ResponseCacheManager class func NewResponseCache(url, method, body string, code int, headers *fasthttp.ResponseHeader) *ResponseCache {
type ResponseCacheManager struct { return &ResponseCache{url, method, body, code, headers}
datastore Datastore
prefix string
ttl int
} }
// NewResponseCacheManager function //func (r *ResponseCache) StatusCode() int {
func NewResponseCacheManager(datastore Datastore, ttl int) ResponseCacheManager { // return r.Code
return ResponseCacheManager{ //}
datastore: datastore, //
prefix: "response_", //func (r *ResponseCache) Body() []byte {
ttl: ttl, // return r.Body
} //}
}
// Save function
func (rm *ResponseCacheManager) Save(name string, r ResponseCache) bool {
if !rm.datastore.IsConnected() {
log.Println("Response-cache:", "Not connected to server")
return false
}
data, err := json.Marshal(r)
if err != nil {
log.Println("Response-cache", "JSON encode:", err)
return false
}
name = rm.prefix + name
err = rm.datastore.SetKey(name, string(data), rm.ttl)
if err != nil {
log.Println("Response-cache:", err, name)
return false
}
return true
}
// Load function
func (rm *ResponseCacheManager) Load(name string) (bool, *ResponseCache) {
if !rm.datastore.IsConnected() {
log.Println("Response-cache:", "Not connected to server")
return false, &ResponseCache{}
}
name = rm.prefix + name
data, err := rm.datastore.GetKey(name)
if err != nil {
return false, &ResponseCache{}
}
rc := &ResponseCache{}
err = json.Unmarshal([]byte(data.(string)), &rc)
if err != nil {
log.Println("Response-cache:", "JSON docode:", err)
return false, &ResponseCache{}
}
return true, rc
}

70
pkg/cache/route.go vendored
View File

@ -9,76 +9,14 @@
package cache package cache
import (
"encoding/json"
"log"
)
// RouteCache struct
type RouteCache struct { type RouteCache struct {
SourceURL string SourceURL string
TargetURL string TargetURL string
} }
// RouteCacheManager class func NewRouteCache(source, target string) *RouteCache {
type RouteCacheManager struct { return &RouteCache{
datastore Datastore SourceURL: source,
prefix string TargetURL: target,
ttl int
}
// NewRouteCacheManager function
func NewRouteCacheManager(datastore Datastore, ttl int) RouteCacheManager {
return RouteCacheManager{
datastore: datastore,
prefix: "route_",
ttl: ttl,
} }
} }
// Save function
func (rm *RouteCacheManager) Save(name string, r RouteCache) bool {
if !rm.datastore.IsConnected() {
log.Println("Route-cache:", "Not connected to server")
return false
}
data, err := json.Marshal(r)
if err != nil {
log.Println("Route-cache", "JSON encode:", err)
return false
}
name = rm.prefix + name
err = rm.datastore.SetKey(name, data, rm.ttl)
if err != nil {
log.Println("Route-cache:", "Unable to save", name, err)
return false
}
return true
}
// Load function
func (rm *RouteCacheManager) Load(name string) (bool, *RouteCache) {
if !rm.datastore.IsConnected() {
log.Println("Route-cache:", "Not connected to server")
return false, &RouteCache{}
}
name = rm.prefix + name
data, err := rm.datastore.GetKey(name)
if err != nil {
log.Println("Route-cache:", "Unable to load", name, err)
return false, &RouteCache{}
}
rc := &RouteCache{}
err = json.Unmarshal([]byte(data.(string)), &rc)
if err != nil {
log.Println("Route-cache:", "JSON decode:", err)
return false, &RouteCache{}
}
return true, rc
}

View File

@ -12,11 +12,13 @@ package main
import ( import (
"flag" "flag"
"log" "log"
"net/http"
"os" "os"
"runtime" "runtime"
"runtime/pprof" "runtime/pprof"
"vegvisir/pkg/server" "vegvisir/pkg/server"
) )
import _ "net/http/pprof"
var ( var (
cPath = flag.String("c", "vegvisir.json", "Path to config file") cPath = flag.String("c", "vegvisir.json", "Path to config file")
@ -26,6 +28,10 @@ var (
) )
func main() { func main() {
go func() {
log.Println(http.ListenAndServe("localhost:6060", nil))
}()
flag.Parse() flag.Parse()
// cpu profiling // cpu profiling
if *cpuprofile != "" { if *cpuprofile != "" {

View File

@ -1,3 +1,4 @@
// ___ ____ ___ ___ // ___ ____ ___ ___
// \ \ / / | _ | __| \ \ / / || | __ || || _ | // \ \ / / | _ | __| \ \ / / || | __ || || _ |
// \ \/ / |___ | |__ \ \/ / || |___ || ||___| // \ \/ / |___ | |__ \ \/ / || |___ || ||___|
@ -11,6 +12,7 @@ package server
import ( import (
"fmt" "fmt"
"log"
"regexp" "regexp"
"strings" "strings"
"vegvisir/pkg/cache" "vegvisir/pkg/cache"
@ -20,28 +22,24 @@ import (
// Router struct // Router struct
type Router struct { type Router struct {
config *config.Config config *config.Config
rcm cache.RouteCacheManager // Redis cache //cache map[string]cache.RouteCache
rCache map[string]cache.RouteCache // Internal route cache - TMP? cache *cache.MemoryDatastore
} //cache *cache.RedisDatastore
// Route struct
type Route struct {
SourceURL string
TargetURL string
} }
// NewRouter function // NewRouter function
func NewRouter(conf *config.Config, cacheDS cache.Datastore, ttl int) *Router { func NewRouter(config *config.Config, ds cache.Datastore, ttl int) *Router {
return &Router{ return &Router{
config: conf, config: config,
rcm: cache.NewRouteCacheManager(cacheDS, ttl), //cache: make(map[string]cache.RouteCache),
rCache: make(map[string]cache.RouteCache), cache: cache.NewMemoryDatastore(),
//cache: cache.NewRedisDatastore(config.Cache.Host, config.Cache.Password, config.Cache.Database, config.Cache.Port),
} }
} }
// FindByRequestURL function // FindByRequestURL function
func (r *Router) FindByRequestURL(url []byte) (bool, Route) { func (r *Router) FindByRequestURL(url []byte) (bool, *cache.RouteCache) {
var sURL string = string(url) var sURL = string(url)
for bID := range r.config.Backends { for bID := range r.config.Backends {
bck := r.config.Backends[bID] bck := r.config.Backends[bID]
@ -50,38 +48,33 @@ func (r *Router) FindByRequestURL(url []byte) (bool, Route) {
} }
for rID := range bck.Routes { for rID := range bck.Routes {
routeCfg := &bck.Routes[rID] rCfg := &bck.Routes[rID]
// if ok, cRoute := s.rCacheManager.Load(sUri); ok { //if cRoute, ok := r.cache[sURL]; ok {
// return true, cRoute if cRoute, err := r.cache.GetKey("route_" + sURL); err == nil {
// } return true, cRoute.(*cache.RouteCache)
if cRoute, ok := r.rCache[sURL]; ok { //response := &cache.RouteCache{} // FIXME #67: Simplify
route := Route{ //ffjson.Unmarshal([]byte(cRoute.(string)), response)
SourceURL: cRoute.SourceURL, //return true, response // FIXME: redis cache tmp
TargetURL: cRoute.TargetURL,
}
return true, route
} }
rgxp := regexp.MustCompile(fmt.Sprintf("%s%s", bck.PrefixURL, routeCfg.Pattern)) regxp := regexp.MustCompile(fmt.Sprintf("%s%s", bck.PrefixURL, rCfg.Pattern))
if rgxp.Match(url) { if regxp.Match(url) {
targetURL := bck.BackendAddress + rgxp.ReplaceAllString(sURL, routeCfg.Target) tURL := bck.BackendAddress + regxp.ReplaceAllString(sURL, rCfg.Target)
route := cache.NewRouteCache(sURL, tURL)
route := Route{ // FIXME: duplicated #1 //routeJSON, _ := ffjson.Marshal(&route) // FIXME: TMP
SourceURL: sURL,
TargetURL: targetURL,
}
// s.rCacheManager.Save(sUri, cRoute) if err := r.cache.SetKey("route_" + sURL, route, r.config.Cache.RouteTTL); err != nil {
r.rCache[sURL] = cache.RouteCache{ // FIXME: duplicated #1 // FIXME: ^^ use cache.Manager* after refactor
SourceURL: route.SourceURL, log.Println("Error saving route cache:", sURL)
TargetURL: route.TargetURL,
} }
log.Println("Saved route:", sURL, route.TargetURL)
return true, route return true, route
} }
} }
} }
return false, Route{} return false, &cache.RouteCache{}
} }

View File

@ -34,45 +34,56 @@ const (
type Server struct { type Server struct {
config *config.Config config *config.Config
router *Router router *Router
respCM cache.ResponseCacheManager // Redis response cache respCM cache.Manager
daemon *fasthttp.Server
} }
// NewServer function // NewServer function
func NewServer(cPath string) *Server { func NewServer(cPath string) *Server {
config := config.New(cPath) cfg := config.New(cPath)
if err := config.Load(); err != nil { if err := cfg.Load(); err != nil {
log.Fatalln("Unable to find config file: ", cPath, err) log.Fatalln("Unable to find config file: ", cPath, err)
} }
datastore := cache.GetCacheDatastore(config.Cache) datastore := cache.GetDatastore(cfg.Cache)
return &Server{ return &Server{
config: config, config: cfg,
router: NewRouter(config, *datastore, config.Cache.RouteTTL), router: NewRouter(cfg, cache.NewMemoryDatastore(), cfg.Cache.RouteTTL),
respCM: cache.NewResponseCacheManager(*datastore, config.Cache.ResponseTTL), // ^^ INFO: Routes always use memoryCache cause low size and fast read time (optimalization)
respCM: cache.NewCachedManager(*datastore, "response_", cfg.Cache.ResponseTTL, cache.HttpRequestHandler),
// ^^ FIXME: add handler detection option by config etc...
} }
} }
// Run function // Run function
func (s *Server) Run() { func (s *Server) Run() {
//s.memo = cache.New(s.processUrl)
log.Println("Starting server...")
go func() { go func() {
serverAddress := s.config.Server.Address + ":" + fmt.Sprint(s.config.Server.Port) serverAddress := s.config.Server.Address + ":" + fmt.Sprint(s.config.Server.Port)
if err := fasthttp.ListenAndServe(serverAddress, s.mainHandler); err != nil { s.daemon = &fasthttp.Server{
Handler: s.mainHandler,
ReadTimeout: 15 * time.Second,
WriteTimeout: 20 * time.Second,
MaxConnsPerIP: 500,
MaxRequestsPerConn: 500,
IdleTimeout: 15 * time.Second, // aka KeepAlive
//CloseOnShutdown: true,
}
if err := s.daemon.ListenAndServe(serverAddress); err != nil {
log.Fatalf("Server panic! Error message: %s", err) log.Fatalf("Server panic! Error message: %s", err)
} }
}() }()
log.Println("Server started") // Wait for an interrupt and attempt a graceful shutdown
// Wait for an interrupt
interrupt := make(chan os.Signal, 1) interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
<-interrupt <-interrupt
log.Println("SIGKILL or SIGINT caught, shutting down...") log.Println("SIGKILL or SIGINT caught, shutting down...")
// Attempt a graceful shutdown
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel() defer cancel()
@ -82,7 +93,10 @@ func (s *Server) Run() {
// Shutdown function // Shutdown function
func (s *Server) Shutdown(ctx context.Context) { // TODO: wait for all connections to finish func (s *Server) Shutdown(ctx context.Context) { // TODO: wait for all connections to finish
log.Println("Shuting down...") //defer s.respCM.Close()
//defer s.daemon.Shutdown()
log.Println("Shutting down...")
} }
func (s *Server) mainHandler(ctx *fasthttp.RequestCtx) { func (s *Server) mainHandler(ctx *fasthttp.RequestCtx) {
@ -94,60 +108,61 @@ func (s *Server) mainHandler(ctx *fasthttp.RequestCtx) {
found, route := s.router.FindByRequestURL(reqURL) found, route := s.router.FindByRequestURL(reqURL)
if !found { if !found {
// FIXME: return 404/5xx error in repsonse, maybe define it in Backend config? // FIXME: return 404 or 5xx error in response? Maybe define it in concrete Backend config?
ctx.SetStatusCode(fasthttp.StatusNotFound) ctx.SetStatusCode(fasthttp.StatusNotFound)
//ctx.SetConnectionClose()
log.Println("404:", sReqMethod, sReqURL)
return return
} }
go func() { response, err := s.respCM.Fetch(sReqURL, sReqMethod, route)
cacheFound, data := s.respCM.Load(sReqURL) //err, response := s.processUrl(sReqURL, sReqMethod, route)
if cacheFound { // get response from cache
// log.Println("Read resp from cache: ", route.TargetUrl)
// copy headers and body from cache
ctx.Response.Header.DisableNormalizing()
for key, value := range data.Headers {
ctx.Response.Header.Set(key, value)
}
ctx.SetBody([]byte(data.Body))
} else { // send request to backend and save to cache
log.Println("Send req to backend url: ", route.TargetURL)
// prepare to send request to backend - separate
bckReq := fasthttp.AcquireRequest()
bckResp := fasthttp.AcquireResponse()
defer fasthttp.ReleaseRequest(bckReq)
defer fasthttp.ReleaseResponse(bckResp)
// copy headers from backend response and prepare request for backend - separate
bckReq.SetRequestURI(route.TargetURL)
bckReq.Header.SetMethod(sReqMethod)
err := fasthttp.Do(bckReq, bckResp)
if err != nil { if err != nil {
// FIXME: Response read error(sending 500 error response)
ctx.SetStatusCode(fasthttp.StatusInternalServerError) ctx.SetStatusCode(fasthttp.StatusInternalServerError)
//ctx.SetConnectionClose() // not sure if change abything with connection: keep-alive/close issue ^^
log.Println("Response read error(sending 500 error response)", err)
return return
} }
headers := make(cache.Headers) ctx.Response.Header.SetBytesV(fasthttp.HeaderContentType, response.Headers.ContentType())
// rewrite headers from backend to gateway response ctx.SetStatusCode(response.Code)
bckResp.Header.Set(fasthttp.HeaderServer, ServerName) ctx.SetBodyString(response.Body)
bckResp.Header.Del(fasthttp.HeaderXPoweredBy) //ctx.SetConnectionClose()
ctx.Response.Header.DisableNormalizing() }
bckResp.Header.VisitAll(func(key, value []byte) {
headers[string(key)] = string(value)
ctx.Response.Header.SetBytesKV(key, value)
})
// ctx.SetStatusCode(bckResp.StatusCode())
ctx.SetBody(bckResp.Body())
// save response to cache //func (s *Server) processUrl(url, method string, route *cache.RouteCache) (error, *cache.ResponseCache) {
respCache := cache.ResponseCache{ // // handle response caching
URL: sReqURL, // cacheKey := method + "_" + url
Body: string(bckResp.Body()), // if ok, data := s.respCM.load(cacheKey, &cache.ResponseCache{}); ok {
Headers: headers, // log.Println("Read resp from cache: ", route.TargetURL, url)
} // FIXME: prepare resp cache struct in respCM.Save method or other service... //
s.respCM.Save(sReqURL, respCache) // return nil, data.(*cache.ResponseCache) // FIXME #67 - simplify
} // } else {
}() // //log.Println("Send req to backend url: ", route.TargetURL, url)
} // ////start := time.Now()
// //bckReq := fasthttp.AcquireRequest()
// //bckResp := fasthttp.AcquireResponse()
// //defer fasthttp.ReleaseRequest(bckReq)
// //defer fasthttp.ReleaseResponse(bckResp)
// //
// //// copy headers from backend response and prepare request for backend - separate
// //bckReq.SetRequestURI(route.TargetURL)
// //bckReq.Header.SetMethod(method)
// //
// //err := fasthttp.Do(bckReq, bckResp)
// //if err != nil {
// // return err, nil
// //}
// //
// //body, code := bckResp.Body(), bckResp.StatusCode()
// ////log.Printf("%s, %s, %d bytes\n", url, time.Since(start), len(body))
// //// save response to cache
// //respCache := cache.NewResponseCache(url, method, body, code, &bckResp.Header)
// //s.respCM.save(cacheKey, respCache)
// //
// //return nil, respCache
// }
//}

View File

@ -1,43 +0,0 @@
{
"server": {
"address": "127.0.0.1",
"port": 8080
},
"backends": {
"news-app": {
"prefixURL": "news/",
"backendAddress": "http://172.17.0.1:3030",
"routes": [{
"pattern": "new/(.*)/([0-9]*)",
"target": "new-url/$1/$2"
},{
"pattern": "new/(.*)",
"target": "$1"
},{
"pattern": "(.+)",
"target": "url"
}]
},
"article-app": {
"prefixURL": "art/",
"backendAddress": "http://172.17.0.1:3030",
"routes": [{
"pattern": "art1/(.*)",
"target": "art-url/$1"
},{
"pattern": "([a-zA-Z0-9]*)",
"target": "art-global/$1"
}]
}
},
"cache": {
"type": "redis",
"host": "localhost",
"port": 6379,
"username": "",
"password": "",
"database": "0",
"routeTTL": 10,
"responseTTL": 10
}
}

View File

@ -6,27 +6,21 @@
"backends": { "backends": {
"news-app": { "news-app": {
"prefixURL": "news/", "prefixURL": "news/",
"backendAddress": "http://127.0.0.1:3030", "backendAddress": "http://172.17.0.1:3030/api/news",
"routes": [{ "routes": [{
"pattern": "new/(.*)", "pattern": "archive/([0-9]*)",
"target": "$1" "target": "archive/$1"
},{ },{
"pattern": "new1/(.*)/([0-9]*)", "pattern": "(.*)",
"target": "new-url1/$1/$2" "target": "new/$1"
},{
"pattern": "(.+)",
"target": "url"
}] }]
}, },
"article-app": { "article-app": {
"prefixURL": "art/", "prefixURL": "art/",
"backendAddress": "http://127.0.0.1:3030", "backendAddress": "http://172.17.0.1:3030/api/articles",
"routes": [{ "routes": [{
"pattern": "art1/(.*)", "pattern": "(.*)",
"target": "art-url1/$1" "target": "$1"
},{
"pattern": "([a-zA-Z0-9]*)",
"target": "article-global/$1"
}] }]
} }
}, },
@ -37,7 +31,7 @@
"username": "", "username": "",
"password": "", "password": "",
"database": "0", "database": "0",
"routeTTL": 300, "routeTTL": 10,
"responseTTL": 300 "responseTTL": 5
} }
} }