From f6f267545aafa5c0aebd507b1244c324c29fc496 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Wed, 8 May 2019 15:42:42 +0200 Subject: [PATCH] examples: add k8s simple and distributed examples --- examples/kubernetes/common/rbac.yml | 64 ++++ examples/kubernetes/distributed/README.md | 15 + examples/kubernetes/distributed/agola.yml | 356 ++++++++++++++++++++++ examples/kubernetes/simple/README.md | 10 + examples/kubernetes/simple/agola.yml | 178 +++++++++++ 5 files changed, 623 insertions(+) create mode 100644 examples/kubernetes/common/rbac.yml create mode 100644 examples/kubernetes/distributed/README.md create mode 100644 examples/kubernetes/distributed/agola.yml create mode 100644 examples/kubernetes/simple/README.md create mode 100644 examples/kubernetes/simple/agola.yml diff --git a/examples/kubernetes/common/rbac.yml b/examples/kubernetes/common/rbac.yml new file mode 100644 index 0000000..61282fa --- /dev/null +++ b/examples/kubernetes/common/rbac.yml @@ -0,0 +1,64 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: agola +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: agola + namespace: default +rules: +- apiGroups: + - "" + - "coordination.k8s.io" + resources: + - nodes + - pods + - pods/exec + - configmaps + - leases + - secrets + verbs: + - "*" + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: agola + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: agola +subjects: +- kind: ServiceAccount + name: default + namespace: default + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: agola + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: agola +subjects: +- kind: ServiceAccount + name: default + namespace: default diff --git a/examples/kubernetes/distributed/README.md b/examples/kubernetes/distributed/README.md new file mode 100644 index 0000000..141fd87 --- /dev/null +++ b/examples/kubernetes/distributed/README.md @@ -0,0 +1,15 @@ +### Agola distributed k8s deployment + +This is a distributed deployment where all the components are replicated to achieve scaling and high availability + +Users should use it as an example base setup and change/improve it based on their needs (choosing which object storage to use). + +* point to an external etcd cluster +* points to an external s3 object storage. +* create 4 deployments for the various components with multiple replicas: + * runservice + * executor + * configstore + * gateway / scheduler + + diff --git a/examples/kubernetes/distributed/agola.yml b/examples/kubernetes/distributed/agola.yml new file mode 100644 index 0000000..888af58 --- /dev/null +++ b/examples/kubernetes/distributed/agola.yml @@ -0,0 +1,356 @@ + +# The client service. It's a node port for easier testing on minikube. Change +# it to become a LoadBalancer if needed. +apiVersion: v1 +kind: Service +metadata: + name: agola-gateway +spec: + ports: + - port: 8000 + nodePort: 30002 + selector: + app: agola + component: gateway-scheduler + type: NodePort + +--- + +# The service for internal components communication with the runservice. +apiVersion: v1 +kind: Service +metadata: + name: agola-runservice +spec: + ports: + - port: 4000 + selector: + app: agola + component: runservice + +--- + +# The service for internal components communication with the configstore. +apiVersion: v1 +kind: Service +metadata: + name: agola-configstore +spec: + ports: + - port: 4002 + selector: + app: agola + component: configstore + +--- + +# The service for internal components communication with the gitserver. +apiVersion: v1 +kind: Service +metadata: + name: agola-gitserver +spec: + ports: + - port: 4003 + selector: + app: agola + component: gitserver + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: agola +data: + config.yml: | + gateway: + # The api url that clients will call + # Change this to the exposed "agola" service IP or dns name + apiExposedURL: "http://192.168.39.188:30002" + # The web interface url that clients will use + # Change this to the exposed "agola" service IP or dns name + webExposedURL: "http://192.168.39.188:30002" + runserviceURL: "http://agola-runservice:4000" + configstoreURL: "http://agola-configstore:4002" + gitserverURL: "http://agola-gitserver:4003" + + web: + listenAddress: ":8000" + tokenSigning: + # hmac or rsa (it possible use rsa) + method: hmac + # key to use when signing with hmac + key: supersecretsigningkey + # paths to the private and public keys in pem encoding when using rsa signing + #privateKeyPath: /path/to/privatekey.pem + #publicKeyPath: /path/to/public.pem + adminToken: "admintoken" + + scheduler: + runserviceURL: "http://agola-runservice:4000" + + configstore: + dataDir: /mnt/agola/local/configstore + etcd: + endpoints: "http://etcd:2379" + objectStorage: + type: s3 + # example with minio + endpoint: "http://minio-service:9000" + bucket: configstore + accessKey: minio + secretAccessKey: minio123 + web: + listenAddress: ":4002" + + runservice: + #debug: true + dataDir: /mnt/agola/local/runservice + etcd: + endpoints: "http://etcd:2379" + objectStorage: + type: s3 + # example with minio + endpoint: "http://minio-service:9000" + bucket: runservice + accessKey: minio + secretAccessKey: minio123 + web: + listenAddress: ":4000" + + executor: + dataDir: /mnt/agola/local/executor + toolboxPath: ./bin/agola-toolbox + runserviceURL: "http://agola-runservice:4000" + web: + listenAddress: ":4001" + activeTasksLimit: 2 + driver: + type: kubernetes + + gitserver: + dataDir: /mnt/agola/local/gitserver + githookPath: ./bin/agola-git-hook + gatewayURL: "http://agola-gateway:8000" + web: + listenAddress: ":4003" + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agola-gateway-scheduler +spec: + replicas: 2 + selector: + matchLabels: + app: agola + component: gateway-scheduler + template: + metadata: + labels: + app: agola + component: gateway-scheduler + spec: + containers: + - name: agola + image: agola + command: + - /bin/agola + - serve + - "--config" + - /mnt/agola/config/config.yml + - "--components" + - gateway,scheduler + env: + ports: + - containerPort: 8000 + volumeMounts: + - name: config-volume + mountPath: /mnt/agola/config + - name: agola-localdata + mountPath: /mnt/agola/local + volumes: + - name: config-volume + configMap: + name: agola + - name: agola-localdata + emptyDir: {} +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agola-runservice +spec: + replicas: 2 + selector: + matchLabels: + app: agola + component: runservice + template: + metadata: + labels: + app: agola + component: runservice + spec: + containers: + - name: agola + image: agola + command: + - /bin/agola + - serve + - "--config" + - /mnt/agola/config/config.yml + - "--components" + - runservice + env: + ports: + - containerPort: 4000 + volumeMounts: + - name: config-volume + mountPath: /mnt/agola/config + - name: agola-localdata + mountPath: /mnt/agola/local + volumes: + - name: config-volume + configMap: + name: agola + - name: agola-localdata + emptyDir: {} + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agola-executor +spec: + replicas: 2 + selector: + matchLabels: + app: agola + component: executor + template: + metadata: + labels: + app: agola + component: executor + spec: + containers: + - name: agola + image: agola + command: + - /bin/agola + - serve + - "--config" + - /mnt/agola/config/config.yml + - "--components" + - executor + env: + ports: + - containerPort: 4001 + volumeMounts: + - name: config-volume + mountPath: /mnt/agola/config + - name: agola-localdata + mountPath: /mnt/agola/local + volumes: + - name: config-volume + configMap: + name: agola + - name: agola-localdata + emptyDir: {} + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agola-configstore +spec: + replicas: 2 + selector: + matchLabels: + app: agola + component: configstore + template: + metadata: + labels: + app: agola + component: configstore + spec: + containers: + - name: agola + image: agola + command: + - /bin/agola + - serve + - "--config" + - /mnt/agola/config/config.yml + - "--components" + - configstore + env: + ports: + - containerPort: 4002 + volumeMounts: + - name: config-volume + mountPath: /mnt/agola/config + - name: agola-localdata + mountPath: /mnt/agola/local + volumes: + - name: config-volume + configMap: + name: agola + - name: agola-localdata + emptyDir: {} + +--- + +# The gitserver. Since it'll primarily store temporary git build data the +# simple way to deploy it is to use a deployment with 1 replica and an emptyDir +# volume. A statefulset with 1 replica and a persistent volume will be a better +# alternative. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agola-gitserver +spec: + # Don't increase the replicas + replicas: 1 + selector: + matchLabels: + app: agola + component: gitserver + template: + metadata: + labels: + app: agola + component: gitserver + spec: + containers: + - name: agola + image: agola + command: + - /bin/agola + - serve + - "--config" + - /mnt/agola/config/config.yml + - "--components" + - gitserver + env: + ports: + - containerPort: 4003 + volumeMounts: + - name: config-volume + mountPath: /mnt/agola/config + - name: agola-localdata + mountPath: /mnt/agola/local + volumes: + - name: config-volume + configMap: + name: agola + - name: agola-localdata + emptyDir: {} diff --git a/examples/kubernetes/simple/README.md b/examples/kubernetes/simple/README.md new file mode 100644 index 0000000..1842210 --- /dev/null +++ b/examples/kubernetes/simple/README.md @@ -0,0 +1,10 @@ +### Agola simple k8s test deployment + +This is the simplest (and not production ready deployment). + +* uses an embedded etcd +* creates a `PersistentVolumeClaim` that will be used as the object storage container for all the components +* created a deployment with a single replica + +You must not increase the replicas or every pod will uses a different embedded etcd causing many issues and errors (and also the pods will fail if scheduled on different k8s node since the PV for the object storage cannot be mounted on multiple nodes) + diff --git a/examples/kubernetes/simple/agola.yml b/examples/kubernetes/simple/agola.yml new file mode 100644 index 0000000..eebf5e4 --- /dev/null +++ b/examples/kubernetes/simple/agola.yml @@ -0,0 +1,178 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: agola-vol +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + +--- + +# The client service. It's a node port for easier testing on minikube. Change +# it to become a LoadBalancer if needed. +apiVersion: v1 +kind: Service +metadata: + name: agola +spec: + ports: + - port: 8000 + name: api + nodePort: 30002 + selector: + app: agola + type: NodePort + +--- + +# The service for internal components communication. +# We are using an headless service since some k8s deployment doesn't have +# hairpin mode enabled and pods cannot communicate with themself via a +# service +apiVersion: v1 +kind: Service +metadata: + name: agola-internal +spec: + ports: + - port: 8000 + name: api + - port: 4000 + name: runservice + - port: 4002 + name: configstore + - port: 4003 + name: gitserver + selector: + app: agola + clusterIP: None + +--- + +# The agola config + +apiVersion: v1 +kind: ConfigMap +metadata: + name: agola +data: + config.yml: | + gateway: + # The api url that clients will call + # Change this to the exposed "agola" service IP + apiExposedURL: "http://192.168.39.188:30002" + # The web interface url that clients will use + # Change this to the exposed "agola" service IP + webExposedURL: "http://192.168.39.188:30002" + runserviceURL: "http://agola-internal:4000" + configstoreURL: "http://agola-internal:4002" + gitserverURL: "http://agola-internal:4003" + + web: + listenAddress: ":8000" + tokenSigning: + # hmac or rsa (it possible use rsa) + method: hmac + # key to use when signing with hmac + key: supersecretsigningkey + # paths to the private and public keys in pem encoding when using rsa signing + #privateKeyPath: /path/to/privatekey.pem + #publicKeyPath: /path/to/public.pem + adminToken: "admintoken" + + scheduler: + runserviceURL: "http://agola-internal:4000" + + configstore: + dataDir: /mnt/agola/local/configstore + etcd: + endpoints: "http://localhost:2379" + objectStorage: + type: posix + path: /mnt/agola/objectstorage/configstore/ost + web: + listenAddress: ":4002" + + runservice: + #debug: true + dataDir: /mnt/agola/local/runservice + etcd: + endpoints: "http://localhost:2379" + objectStorage: + type: posix + path: /mnt/agola/objectstorage/runservice/ost + web: + listenAddress: ":4000" + + executor: + dataDir: /mnt/agola/local/executor + toolboxPath: ./bin/agola-toolbox + runserviceURL: "http://agola-internal:4000" + web: + listenAddress: ":4001" + activeTasksLimit: 2 + driver: + type: kubernetes + + gitserver: + dataDir: /mnt/agola/local/gitserver + githookPath: ./bin/agola-git-hook + gatewayURL: "http://agola-internal:8000" + web: + listenAddress: ":4003" +--- + + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agola +spec: + # Do not increase replica count or everything will break since every pod will + # have its own etcd instance + replicas: 1 + selector: + matchLabels: + app: agola + template: + metadata: + labels: + app: agola + spec: + containers: + - name: agola + image: agola + command: + - /bin/agola + - serve + - --embedded-etcd + - "--config" + - /mnt/agola/config/config.yml + - "--components" + - all + env: + ports: + - containerPort: 8000 + - containerPort: 4000 + - containerPort: 4002 + - containerPort: 4003 + volumeMounts: + - name: config-volume + mountPath: /mnt/agola/config + - name: agola-localdata + mountPath: /mnt/agola/local + - name: agola-objectstorage + mountPath: /mnt/agola/objectstorage + volumes: + - name: config-volume + configMap: + name: agola + - name: agola-localdata + emptyDir: {} + - name: agola-objectstorage + persistentVolumeClaim: + claimName: agola-vol