Added KV Config support and updated build scripts

2022-12-05 23:25:56 +01:00 · 2022-12-05 23:25:56 +01:00 · 39b94713b8
commit 39b94713b8
parent c322316430
13 changed files with 139 additions and 45 deletions
--- a/Dockerfile.builder
+++ b/Dockerfile.builder
@ -11,4 +11,3 @@ COPY src ./
 RUN export CGO_ENABLED=0 ; export GOOS=linux ; export GOARCH=amd64 && \
    go build -ldflags="-w -s" -o "$BIN_OUTPUT/server" $GO_SERVER && \
    go build -ldflags="-w -s" -o "$BIN_OUTPUT/worker" $GO_WORKER
-    
--- a/Dockerfile.target
+++ b/Dockerfile.target
@ -1,11 +1,12 @@
 # Builder
-ARG BUILDER_IMAGE="git.pbiernat.dev/egommerce/catalog-builder:latest"
+ARG BUILDER_IMAGE
 FROM ${BUILDER_IMAGE} AS builder

 # Destination image - server
 # FROM gcr.io/distroless/base-debian10
 FROM alpine:3.17

+ARG BUILD_TIME
 ARG BIN_OUTPUT
 ARG SVC_NAME
 ARG SVC_VER
@ -14,6 +15,7 @@ LABEL dev.egommerce.image.author="Piotr Biernat"
 LABEL dev.egommerce.image.vendor="Egommerce"
 LABEL dev.egommerce.image.service=${SVC_NAME}
 LABEL dev.egommerce.image.version=${SVC_VER}
+LABEL dev.egommerce.image.build_time=${BUILD_TIME}

 WORKDIR /
 COPY --from=builder $BIN_OUTPUT /app
--- a/deploy/image-build.sh
+++ b/deploy/image-build.sh
@ -2,39 +2,39 @@
 # RUN IN REPO ROOT DIR !!

 export IMAGE_PREFIX="git.pbiernat.dev/egommerce/catalog"
-export BUILDER_IMAGE="$IMAGE_PREFIX-builder:tmp"
+export BUILDER_IMAGE="egommerce-builder:catalog"
+export BUILD_TIME=$(date +"%Y%m%d%H%M%S")
 export SERVER_IMAGE="$IMAGE_PREFIX-svc"
 export WORKER_IMAGE="$IMAGE_PREFIX-worker"
+export DOCKER_BUILDKIT=1

 TARGET=${1:-latest}

-[ ! -d \"src/vendor\" ] && sh -c "cd src; go mod vendor"
-
-export DOCKER_BUILDKIT=1
-
-docker build -t "$BUILDER_IMAGE" -f Dockerfile.builder . && echo "Successfully tagged $BUILDER_IMAGE"
+[ ! -d "src/vendor" ] && sh -c "cd src; go mod vendor"

 echo "Building target $IMAGE_PREFIX images..."
+docker build --rm -t "$BUILDER_IMAGE" -f Dockerfile.builder .
+
 if [ $TARGET = "latest" ]
 then
    # SERVER
    docker build --build-arg SVC_NAME=catalog-svc --build-arg SVC_VER="1.0" --build-arg BIN_OUTPUT=/go/bin/server \
-        --rm --build-arg BUILDER_IMAGE --cache-from "$SERVER_IMAGE:$TARGET" -t "$SERVER_IMAGE:$TARGET" \
-        -f Dockerfile.target . >/dev/null 2>&1 && echo "Successfully tagged $SERVER_IMAGE:$TARGET" &
+        --build-arg BUILDER_IMAGE=$BUILDER_IMAGE --build-arg BUILD_TIME --rm --cache-from "$SERVER_IMAGE:$TARGET" -t "$SERVER_IMAGE:$TARGET" \
+        -f Dockerfile.target . >/dev/null 2>&1 && echo "Successfully tagged $SERVER_IMAGE:$TARGET"
    
    # WORKER
    docker build --build-arg SVC_NAME=catalog-worker --build-arg SVC_VER="1.0" --build-arg BIN_OUTPUT=/go/bin/worker \
-        --rm --build-arg BUILDER_IMAGE --cache-from "$WORKER_IMAGE:$TARGET" -t "$WORKER_IMAGE:$TARGET" \
+        --build-arg BUILDER_IMAGE=$BUILDER_IMAGE --build-arg BUILD_TIME --rm --cache-from "$WORKER_IMAGE:$TARGET" -t "$WORKER_IMAGE:$TARGET" \
        -f Dockerfile.target . >/dev/null 2>&1 && echo "Successfully tagged $WORKER_IMAGE:$TARGET"
 else
    # SERVER
    docker build --build-arg SVC_NAME=catalog-svc --build-arg SVC_VER="dev" --build-arg BIN_OUTPUT=/go/bin/server \
-        --rm --build-arg BUILDER_IMAGE --no-cache -t "$SERVER_IMAGE:$TARGET" \
-        -f Dockerfile.target . >/dev/null 2>&1 && echo "Successfully tagged $SERVER_IMAGE:$TARGET" &
+        --build-arg BUILDER_IMAGE=$BUILDER_IMAGE --build-arg BUILD_TIME --rm --no-cache -t "$SERVER_IMAGE:$TARGET" \
+        -f Dockerfile.target . >/dev/null 2>&1 && echo "Successfully tagged $SERVER_IMAGE:$TARGET"

    # WORKER
    docker build --build-arg SVC_NAME=catalog-worker --build-arg SVC_VER="dev" --build-arg BIN_OUTPUT=/go/bin/worker \
-        --rm --build-arg BUILDER_IMAGE --no-cache -t "$WORKER_IMAGE:$TARGET" \
+        --build-arg BUILDER_IMAGE=$BUILDER_IMAGE --build-arg BUILD_TIME --rm --no-cache -t "$WORKER_IMAGE:$TARGET" \
        -f Dockerfile.target . >/dev/null 2>&1 && echo "Successfully tagged $WORKER_IMAGE:$TARGET"
 fi

--- a/deploy/image-push.sh
+++ b/deploy/image-push.sh
@ -8,5 +8,6 @@ export WORKER_IMAGE="$IMAGE_BASE-worker"
 TARGET=${1:-latest}

 echo $DOCKER_PASSWORD | docker login git.pbiernat.dev -u $DOCKER_USERNAME --password-stdin
+
 docker push "$SERVER_IMAGE:$TARGET"
 docker push "$WORKER_IMAGE:$TARGET"
--- a/src/cmd/server/main.go
+++ b/src/cmd/server/main.go
@ -23,6 +23,7 @@ const (
 	defEventBusURL   = "amqp://guest:guest@api-eventbus:5672"
 	ebEventsExchange = "api-events"
 	ebEventsQueue    = "catalog-svc"
+	defKVNmspc       = "dev.egommerce/service/catalog-svc"
 )

 func main() {
@ -41,6 +42,7 @@ func main() {
 	c.DbURL = config.GetEnv("DATABASE_URL", defDbURL)
 	c.EventBusURL = config.GetEnv("EVENTBUS_URL", defEventBusURL)
 	c.EventBusExchange = ebEventsExchange
+	c.KVNamespace = config.GetEnv("APP_KV_NAMESPACE", defKVNmspc)

 	logHost, logPort := fluentd.ParseAddr(c.LoggerAddr)
 	logger := fluentd.NewLogger(c.GetAppFullName(), logHost, logPort)
--- a/src/cmd/worker/main.go
+++ b/src/cmd/worker/main.go
@ -1,17 +1,22 @@
 package main

 import (
+	"bytes"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"

 	"git.pbiernat.dev/egommerce/catalog-service/internal/app/config"
 	"git.pbiernat.dev/egommerce/catalog-service/internal/app/database"
 	"git.pbiernat.dev/egommerce/catalog-service/internal/app/event"
 	"git.pbiernat.dev/egommerce/catalog-service/internal/app/server"
+	discovery "git.pbiernat.dev/egommerce/go-api-pkg/consul"
 	"git.pbiernat.dev/egommerce/go-api-pkg/fluentd"
 	amqp "git.pbiernat.dev/egommerce/go-api-pkg/rabbitmq"
 )
@ -19,11 +24,13 @@ import (
 const (
 	defAppName       = "catalog-worker"
 	defLoggerAddr    = "api-logger:24224"
+	defRegistryAddr  = "api-registry:8500"
 	defDbURL         = "postgres://postgres:12345678@postgres-db:5432/egommerce"
 	defMongoDbURL    = "mongodb://mongodb:12345678@mongo-db:27017"
 	defEventBusURL   = "amqp://guest:guest@api-eventbus:5672"
 	ebEventsExchange = "api-events"
 	ebEventsQueue    = "catalog-worker"
+	defKVNmspc       = "dev.egommerce/service/catalog-worker"
 )

 func main() {
@ -35,15 +42,33 @@ func main() {
 	c.AppID, _ = os.Hostname()
 	c.AppName = config.GetEnv("APP_NAME", defAppName)
 	c.LoggerAddr = config.GetEnv("LOGGER_ADDR", defLoggerAddr)
+	c.RegistryAddr = config.GetEnv("REGISTRY_ADDR", defRegistryAddr)
 	c.DbURL = config.GetEnv("DATABASE_URL", defDbURL)
 	c.EventBusURL = config.GetEnv("EVENTBUS_URL", defEventBusURL)
 	c.EventBusExchange = ebEventsExchange
 	c.EventBusQueue = ebEventsQueue
+	c.KVNamespace = config.GetEnv("APP_KV_NAMESPACE", defKVNmspc)

 	logHost, logPort := fluentd.ParseAddr(c.LoggerAddr)
 	logger := fluentd.NewLogger(c.GetAppFullName(), logHost, logPort)
 	defer logger.Close()

+	consul, err := discovery.NewService(c.RegistryAddr, c.AppID, c.AppName, c.AppID, "", 0)
+	if err != nil {
+		logger.Log("Error connecting to %s: %v", c.RegistryAddr, err)
+	}
+
+	go func(consul *discovery.Service) {
+		interval := time.Second * 30
+		ticker := time.NewTicker(interval)
+		for range ticker.C {
+			err := updateKVConfig(consul, c) // FIXME: duplicated in internal/app/server/server.go
+			if err != nil {
+				logger.Log("KV config update error (skipping): %v\n", err)
+			}
+		}
+	}(consul)
+
 	// db conn
 	dbConn, err := database.Connect(c.DbURL)
 	if err != nil { // fixme: add wait-for-db...
@ -140,3 +165,22 @@ func main() {
 	logger.Log("Waiting for messages...")
 	<-forever
 }
+
+func updateKVConfig(s *discovery.Service, oldCnf *server.Config) error { // FIXME: duplicated in internal/app/server/server.go
+	data, _, err := s.KV().Get(oldCnf.KVNamespace, nil)
+	if err != nil {
+		return err
+	}
+
+	if data == nil {
+		return errors.New("empty KV config data. Skipping")
+	}
+
+	buf := bytes.NewBuffer(data.Value)
+	decoder := json.NewDecoder(buf)
+	if err := decoder.Decode(oldCnf); err != nil {
+		return err
+	}
+
+	return nil
+}
--- a/src/go.mod
+++ b/src/go.mod
@ -3,7 +3,7 @@ module git.pbiernat.dev/egommerce/catalog-service
 go 1.18

 require (
-	git.pbiernat.dev/egommerce/go-api-pkg v0.0.101
+	git.pbiernat.dev/egommerce/go-api-pkg v0.0.108
 	github.com/gofiber/fiber/v2 v2.40.1
 	github.com/google/uuid v1.3.0
 	github.com/jackc/pgx/v4 v4.17.2
--- a/src/go.sum
+++ b/src/go.sum
@ -1,13 +1,15 @@
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.32 h1:ArB/n30m927WMAM4u51guH+qR0Lu4NGyYnYdi7OhlzY=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.32/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.33 h1:1tm+pvUeS6OZLvHmLM3BwFS0Ty/eA3jDRuB60OicosA=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.33/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.34 h1:UO1x6O+cyU7yYYbDCDyhhAypuf4QGIXcmWcBEEjLuYM=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.34/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.100 h1:jw4fiGbZTsfJXJpGV+HQiYeMGZ7DMRMoepjuIwY6FIU=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.100/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.101 h1:NZCFAAlC94+LcN1gjrENnWUHvpWgaNksyB2N4Fiy8C4=
-git.pbiernat.dev/egommerce/go-api-pkg v0.0.101/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.103 h1:tVSHVQOBDe1Ofcbodaa/R5gHRD4gYO/d1tw7rVuLJuA=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.103/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.104 h1:YymR7Zyo9xjIZ9S75o2nfyNHp69n2FXHyGbTxtV1p/A=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.104/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.105 h1:8w4p4QNaSF58iL3YiGvqXC4UjUVeeu5D10OQmImA/Z0=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.105/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.106 h1:kOqDvQfk8MzmyQonMMLmZKhW7I5YeDyw6N8YIYHIVwA=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.106/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.107 h1:yigpHD40ocyiamWc7GUVPgeaVtmz1VQwOucagwbuQfE=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.107/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.108 h1:gr5kzKNR3sCxTz+nbqtOM7vdIely5ZWb8itSLAjTo0I=
+git.pbiernat.dev/egommerce/go-api-pkg v0.0.108/go.mod h1:nAwcw2MZtn/54YKq8VQK6RJAsiuoLUtPuazXg8JcqK8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
--- a/src/internal/app/config/env.go
+++ b/src/internal/app/config/env.go
@ -12,7 +12,7 @@ func init() {
 	ErrLoadingEnvs = godotenv.Load()
 }

-func GetEnv(name string, defVal string) string { // FIXME defVal and return types
+func GetEnv(name string, defVal string) string {
 	env := os.Getenv(name)
 	if env == "" {
 		return defVal
--- a/src/internal/app/server/config.go
+++ b/src/internal/app/server/config.go
@ -8,13 +8,19 @@ type Config struct {
 	AppDomain    string
 	NetAddr      string
 	Port         int
-	LoggerAddr       string
 	RegistryAddr string
-	DbURL            string
-	MongoDbUrl       string
-	EventBusURL      string
-	EventBusExchange string
-	EventBusQueue    string
+	KVNamespace  string
+
+	LoggerAddr       string `json:"logger_addr"`
+	DbURL            string `json:"db_url"`
+	MongoDbUrl       string `json:"mongodb_url"`
+	EventBusURL      string `json:"eventbus_url"`
+	EventBusExchange string `json:"eventbus_exchange"`
+	EventBusQueue    string `json:"eventbus_queue"`
+	HttpReadTimeout  int    `json:"http_read_timeout"`
+	HttpWriteTimeout int    `json:"http_write_timeout"`
+	HttpIdleTimeout  int    `json:"http_idle_timeout"`
+	// Fields with json mapping are available trough ConsulKV
 }

 func (c *Config) GetAppFullName() string {
--- a/src/internal/app/server/health_handler.go
+++ b/src/internal/app/server/health_handler.go
@ -1,9 +1,8 @@
 package server

 import (
-	"github.com/gofiber/fiber/v2"
-
 	def "git.pbiernat.dev/egommerce/catalog-service/internal/app/definition"
+	"github.com/gofiber/fiber/v2"
 )

 func (s *Server) HealthHandler(c *fiber.Ctx) error {
@ -11,3 +10,7 @@ func (s *Server) HealthHandler(c *fiber.Ctx) error {
 		Status: "OK",
 	})
 }
+
+func (s *Server) ConfigHandler(c *fiber.Ctx) error {
+	return c.JSON(s.conf)
+}
--- a/src/internal/app/server/router.go
+++ b/src/internal/app/server/router.go
@ -9,6 +9,7 @@ import (

 func SetupRoutes(s *Server) {
 	s.App.Get("/health", s.HealthHandler)
+	s.App.Get("/config", s.ConfigHandler)

 	api := s.App.Group("/api")
 	v1 := api.Group("/v1")
--- a/src/internal/app/server/server.go
+++ b/src/internal/app/server/server.go
@ -1,6 +1,9 @@
 package server

 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
@ -16,12 +19,14 @@ import (

 type Server struct {
 	*fiber.App
+	conf      *Config
 	log       *fluentd.Logger
 	db        *pgxpool.Pool
 	ebCh      *amqp.Channel
 	discovery *discovery.Service
 	name      string
 	addr      string
+	kvNmspc   string
 }

 type Headers struct {
@ -29,14 +34,13 @@ type Headers struct {
 }

 func NewServer(conf *Config, logger *fluentd.Logger, db *pgxpool.Pool, ebCh *amqp.Channel) *Server {
-	logger.Log("API_ID: %s", conf.AppID)
-	discovery, err := discovery.NewService(conf.RegistryAddr, conf.AppID, conf.AppName, conf.AppID, conf.AppDomain, conf.Port)
+	consul, err := discovery.NewService(conf.RegistryAddr, conf.AppID, conf.AppName, conf.AppID, conf.AppDomain, conf.Port)
 	if err != nil {
 		logger.Log("Error connecting to %s: %v", conf.RegistryAddr, err)
 	}

-	logger.Log("Registering service with name: %s, address: %s", discovery.Name, discovery.Address)
-	err = discovery.Register()
+	logger.Log("Registering service with name: %s, address: %s", consul.Name, consul.Address)
+	err = consul.Register()
 	if err != nil {
 		logger.Log("register error: %v", err)
 	}
@ -50,14 +54,27 @@ func NewServer(conf *Config, logger *fluentd.Logger, db *pgxpool.Pool, ebCh *amq
 	}
 	s := &Server{
 		fiber.New(cnf),
+		conf,
 		logger,
 		db,
 		ebCh,
-		discovery,
+		consul,
 		conf.AppName,
 		conf.NetAddr,
+		conf.KVNamespace,
 	}

+	go func(s *Server) { // Consul KV config updater
+		interval := time.Second * 30
+		ticker := time.NewTicker(interval)
+		for range ticker.C {
+			err := s.updateKVConfig()
+			if err != nil {
+				logger.Log("KV config update error (skipping): %v\n", err)
+			}
+		}
+	}(s)
+
 	SetupMiddlewares(s)
 	SetupRoutes(s)

@ -99,6 +116,23 @@ func (s *Server) GetRequestID(c *fiber.Ctx) (string, error) {
 	return hdr.RequestID, nil
 }

+func (s *Server) updateKVConfig() error { // FIXME: duplicated in cmd/worker/main.go
+	data, _, err := s.discovery.KV().Get(s.kvNmspc, nil)
+	if err != nil {
+		fmt.Println(err)
+
+		return err
+	}
+
+	kvCnf := bytes.NewBuffer(data.Value)
+	decoder := json.NewDecoder(kvCnf)
+	if err := decoder.Decode(&s.conf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (s *Server) gracefulShutdown() error {
 	s.log.Log("Server is going down...")
 	s.log.Log("Unregistering service: %s", s.discovery.GetID())