From aab2321d58efdb4f6d8862ae0a7d0dd81a09da0d Mon Sep 17 00:00:00 2001 From: Carlo Mandelli Date: Mon, 4 Nov 2019 16:18:35 +0100 Subject: [PATCH] services: check config only for enabled services --- cmd/agola/cmd/serve.go | 2 +- internal/services/config/config.go | 134 ++++++------ internal/services/config/config_test.go | 257 ++++++++++++++++++++++++ 3 files changed, 336 insertions(+), 57 deletions(-) create mode 100644 internal/services/config/config_test.go diff --git a/cmd/agola/cmd/serve.go b/cmd/agola/cmd/serve.go index 55bdf1c..c672257 100644 --- a/cmd/agola/cmd/serve.go +++ b/cmd/agola/cmd/serve.go @@ -129,7 +129,7 @@ func serve(cmd *cobra.Command, args []string) error { } } - c, err := config.Parse(serveOpts.config) + c, err := config.Parse(serveOpts.config, serveOpts.components) if err != nil { return errors.Errorf("config error: %w", err) } diff --git a/internal/services/config/config.go b/internal/services/config/config.go index 1b3b5e1..358a652 100644 --- a/internal/services/config/config.go +++ b/internal/services/config/config.go @@ -19,6 +19,7 @@ import ( "time" "agola.io/agola/internal/util" + errors "golang.org/x/xerrors" yaml "gopkg.in/yaml.v2" ) @@ -231,7 +232,7 @@ var defaultConfig = Config{ }, } -func Parse(configFile string) (*Config, error) { +func Parse(configFile string, componentsNames []string) (*Config, error) { configData, err := ioutil.ReadFile(configFile) if err != nil { return nil, err @@ -242,7 +243,7 @@ func Parse(configFile string) (*Config, error) { return nil, err } - return c, Validate(c) + return c, Validate(c, componentsNames) } func validateWeb(w *Web) error { @@ -262,7 +263,7 @@ func validateWeb(w *Web) error { return nil } -func Validate(c *Config) error { +func Validate(c *Config, componentsNames []string) error { // Global if len(c.ID) > maxIDLength { return errors.Errorf("id too long") @@ -272,78 +273,99 @@ func Validate(c *Config) error { } // Gateway - if c.Gateway.APIExposedURL == "" { - return errors.Errorf("gateway apiExposedURL is empty") - } - if c.Gateway.WebExposedURL == "" { - return errors.Errorf("gateway webExposedURL is empty") - } - if c.Gateway.ConfigstoreURL == "" { - return errors.Errorf("gateway configstoreURL is empty") - } - if c.Gateway.RunserviceURL == "" { - return errors.Errorf("gateway runserviceURL is empty") - } - if err := validateWeb(&c.Gateway.Web); err != nil { - return errors.Errorf("gateway web configuration error: %w", err) + if isComponentEnabled(componentsNames, "gateway") { + if c.Gateway.APIExposedURL == "" { + return errors.Errorf("gateway apiExposedURL is empty") + } + if c.Gateway.WebExposedURL == "" { + return errors.Errorf("gateway webExposedURL is empty") + } + if c.Gateway.ConfigstoreURL == "" { + return errors.Errorf("gateway configstoreURL is empty") + } + if c.Gateway.RunserviceURL == "" { + return errors.Errorf("gateway runserviceURL is empty") + } + if err := validateWeb(&c.Gateway.Web); err != nil { + return errors.Errorf("gateway web configuration error: %w", err) + } } // Configstore - if c.Configstore.DataDir == "" { - return errors.Errorf("configstore dataDir is empty") - } - if err := validateWeb(&c.Configstore.Web); err != nil { - return errors.Errorf("configstore web configuration error: %w", err) + if isComponentEnabled(componentsNames, "configstore") { + if c.Configstore.DataDir == "" { + return errors.Errorf("configstore dataDir is empty") + } + if err := validateWeb(&c.Configstore.Web); err != nil { + return errors.Errorf("configstore web configuration error: %w", err) + } } // Runservice - if c.Runservice.DataDir == "" { - return errors.Errorf("runservice dataDir is empty") - } - if err := validateWeb(&c.Runservice.Web); err != nil { - return errors.Errorf("runservice web configuration error: %w", err) + if isComponentEnabled(componentsNames, "runservice") { + if c.Runservice.DataDir == "" { + return errors.Errorf("runservice dataDir is empty") + } + if err := validateWeb(&c.Runservice.Web); err != nil { + return errors.Errorf("runservice web configuration error: %w", err) + } } // Executor - if c.Executor.DataDir == "" { - return errors.Errorf("executor dataDir is empty") - } - if c.Executor.ToolboxPath == "" { - return errors.Errorf("git server toolboxPath is empty") - } - if c.Executor.RunserviceURL == "" { - return errors.Errorf("executor runserviceURL is empty") - } - if c.Executor.Driver.Type == "" { - return errors.Errorf("executor driver type is empty") - } - switch c.Executor.Driver.Type { - case DriverTypeDocker: - case DriverTypeK8s: - default: - return errors.Errorf("executor driver type %q unknown", c.Executor.Driver.Type) + if isComponentEnabled(componentsNames, "executor") { + if c.Executor.DataDir == "" { + return errors.Errorf("executor dataDir is empty") + } + if c.Executor.ToolboxPath == "" { + return errors.Errorf("git server toolboxPath is empty") + } + if c.Executor.RunserviceURL == "" { + return errors.Errorf("executor runserviceURL is empty") + } + if c.Executor.Driver.Type == "" { + return errors.Errorf("executor driver type is empty") + } + switch c.Executor.Driver.Type { + case DriverTypeDocker: + case DriverTypeK8s: + default: + return errors.Errorf("executor driver type %q unknown", c.Executor.Driver.Type) + } } // Scheduler - if c.Scheduler.RunserviceURL == "" { - return errors.Errorf("scheduler runserviceURL is empty") + if isComponentEnabled(componentsNames, "scheduler") { + if c.Scheduler.RunserviceURL == "" { + return errors.Errorf("scheduler runserviceURL is empty") + } } // Notification - if c.Notification.WebExposedURL == "" { - return errors.Errorf("notification webExposedURL is empty") - } - if c.Notification.ConfigstoreURL == "" { - return errors.Errorf("notification configstoreURL is empty") - } - if c.Notification.RunserviceURL == "" { - return errors.Errorf("notification runserviceURL is empty") + if isComponentEnabled(componentsNames, "notification") { + if c.Notification.WebExposedURL == "" { + return errors.Errorf("notification webExposedURL is empty") + } + if c.Notification.ConfigstoreURL == "" { + return errors.Errorf("notification configstoreURL is empty") + } + if c.Notification.RunserviceURL == "" { + return errors.Errorf("notification runserviceURL is empty") + } } // Git server - if c.Gitserver.DataDir == "" { - return errors.Errorf("git server dataDir is empty") + if isComponentEnabled(componentsNames, "gitserver") { + if c.Gitserver.DataDir == "" { + return errors.Errorf("git server dataDir is empty") + } } return nil } + +func isComponentEnabled(componentsNames []string, name string) bool { + if util.StringInSlice(componentsNames, "all-base") && name != "executor" { + return true + } + return util.StringInSlice(componentsNames, name) +} diff --git a/internal/services/config/config_test.go b/internal/services/config/config_test.go new file mode 100644 index 0000000..9b8b9aa --- /dev/null +++ b/internal/services/config/config_test.go @@ -0,0 +1,257 @@ +// Copyright 2019 Sorint.lab +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "io/ioutil" + "os" + "path" + "testing" + + errors "golang.org/x/xerrors" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + services []string + in string + err error + }{ + { + name: "test config for all-base components and executor", + services: []string{"all-base", "executor"}, + in: ` +gateway: + apiExposedURL: "http://localhost:8000" + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + gitserverURL: "http://localhost:4003" + + web: + listenAddress: ":8000" + tokenSigning: + method: hmac + key: supersecretsigningkey + adminToken: "admintoken" + +scheduler: + runserviceURL: "http://localhost:4000" + +notification: + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + etcd: + endpoints: "http://localhost:2379" + +configstore: + dataDir: /data/agola/configstore + etcd: + endpoints: "http://localhost:2379" + objectStorage: + type: posix + path: /agola/configstore/ost + web: + listenAddress: ":4002" + +runservice: + #debug: true + dataDir: /opt/data/agola/runservice + etcd: + endpoints: "http://localhost:2379" + objectStorage: + type: posix + path: /agola/runservice/ost + web: + listenAddress: ":4000" + +executor: + allowPrivilegedContainers: true + dataDir: /data/agola/executor + toolboxPath: ./bin + runserviceURL: "http://localhost:4000" + web: + listenAddress: ":4001" + activeTasksLimit: 5 + driver: + type: docker + +gitserver: + dataDir: /data/agola/gitserver + gatewayURL: "http://localhost:8000" + web: + listenAddress: ":4003"`, + }, + { + name: "test config for all-base components", + services: []string{"all-base"}, + in: ` +gateway: + apiExposedURL: "http://localhost:8000" + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + gitserverURL: "http://localhost:4003" + + web: + listenAddress: ":8000" + tokenSigning: + method: hmac + key: supersecretsigningkey + adminToken: "admintoken" + +scheduler: + runserviceURL: "http://localhost:4000" + +notification: + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + etcd: + endpoints: "http://localhost:2379" + +configstore: + dataDir: /data/agola/configstore + etcd: + endpoints: "http://localhost:2379" + objectStorage: + type: posix + path: /agola/configstore/ost + web: + listenAddress: ":4002" + +runservice: + dataDir: /data/agola/runservice + etcd: + endpoints: "http://localhost:2379" + objectStorage: + type: posix + path: /agola/runservice/ost + web: + listenAddress: ":4000" + +gitserver: + dataDir: /data/agola/gitserver + gatewayURL: "http://localhost:8000" + web: + listenAddress: ":4003"`, + }, + { + name: "test config for gateway, scheduler and notification", + services: []string{"gateway", "scheduler", "notification"}, + in: ` +gateway: + apiExposedURL: "http://localhost:8000" + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + gitserverURL: "http://localhost:4003" + + web: + listenAddress: ":8000" + tokenSigning: + method: hmac + key: supersecretsigningkey + adminToken: "admintoken" + +scheduler: + runserviceURL: "http://localhost:4000" + +notification: + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + etcd: + endpoints: "http://localhost:2379" + +configstore: + dataDir: + +runservice: + dataDir: + +gitserver: + dataDir:`, + }, + { + name: "test config for gateway, scheduler, notification and gitserver without dataDir", + services: []string{"gateway", "scheduler", "notification", "gitserver"}, + in: ` +gateway: + apiExposedURL: "http://localhost:8000" + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + gitserverURL: "http://localhost:4003" + + web: + listenAddress: ":8000" + tokenSigning: + method: hmac + key: supersecretsigningkey + adminToken: "admintoken" + +scheduler: + runserviceURL: "http://localhost:4000" + +notification: + webExposedURL: "http://localhost:8000" + runserviceURL: "http://localhost:4000" + configstoreURL: "http://localhost:4002" + etcd: + endpoints: "http://localhost:2379" + +configstore: + dataDir: + +runservice: + dataDir: + +gitserver: + dataDir:`, + err: errors.Errorf("git server dataDir is empty"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := ioutil.TempDir("", "ParseConfig") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + defer os.RemoveAll(dir) + + content := []byte(tt.in) + err = ioutil.WriteFile(path.Join(dir, "config.yml"), content, 0644) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if _, err := Parse(path.Join(dir, "config.yml"), tt.services); err != nil { + if tt.err == nil { + t.Fatalf("got error: %v, expected no error", err) + } + if err.Error() != tt.err.Error() { + t.Fatalf("got error: %v, want error: %v", err, tt.err) + } + } else { + if tt.err != nil { + t.Fatalf("got nil error, want error: %v", tt.err) + } + } + }) + } +}