change master service to headless service

2020-08-01 18:01:15 -07:00 · 2020-08-01 18:01:15 -07:00 · f65a532f78
parent bcd4d42404
commit f65a532f78
3 changed files with 200 additions and 28 deletions
--- a/README.md
+++ b/README.md
@ -0,0 +1,22 @@
+# SeaweedFS Operator
+
+## Installation
+
+## Development
+
+Follow the instructions in https://sdk.operatorframework.io/docs/golang/quickstart/
+
+```
+$ git clone https://github.com/seaweedfs/seaweedfs-operator
+$ cd seaweedfs-operator
+
+# register the CRD with the Kubernetes
+$ make install
+
+# run the operator locally outside the Kubernetes cluster
+$ make run ENABLE_WEBHOOKS=false 
+
+# From another terminal in the same directory
+$ kubectl apply -f config/samples/objectstore_v100_master.yaml
+
+```
--- a/controllers/objectstore/master_controller.go
+++ b/controllers/objectstore/master_controller.go
@ -23,10 +23,8 @@ import (
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"

@ -51,30 +49,17 @@ func (r *MasterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	// your logic here
 	log.Info("start Reconcile ...")

-	master, done, result, err := r.findMasterInstance(req, ctx, log)
+	master, done, result, err := r.findMasterCustomResourceInstance(ctx, log, req)
 	if done {
 		return result, err
 	}

-	// Check if the deployment already exists, if not create a new one
-	found := &appsv1.Deployment{}
-	err = r.Get(ctx, types.NamespacedName{Name: master.Name, Namespace: master.Namespace}, found)
-	if err != nil && errors.IsNotFound(err) {
-		// Define a new deployment
-		dep := r.deploymentForMaster(master)
-		log.Info("Creating a new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
-		err = r.Create(ctx, dep)
-		if err != nil {
-			log.Error(err, "Failed to create new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
-			return ctrl.Result{}, err
+	if done, result, err = r.ensureMasterStatefulSet(ctx, log, master); done {
+		return result, err
 	}
-		// Deployment created successfully - return and requeue
-		return ctrl.Result{Requeue: true}, nil
-	} else if err != nil {
-		log.Error(err, "Failed to get Deployment")
-		return ctrl.Result{}, err
+	if done, result, err = r.ensureMasterService(ctx, log, master); done {
+		return result, err
 	}
-	log.Info("Get deployment " + found.Name)

 	// Update the Memcached status with the pod names
 	// List the pods for this memcached's deployment
--- a/controllers/objectstore/master_controller_ensure.go
+++ b/controllers/objectstore/master_controller_ensure.go
@ -5,15 +5,20 @@ import (
 	"time"

 	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	"sigs.k8s.io/controller-runtime"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	ctrl "sigs.k8s.io/controller-runtime"

-	"github.com/seaweedfs/seaweedfs-operator/apis/objectstore/v100"
+	objectstorev100 "github.com/seaweedfs/seaweedfs-operator/apis/objectstore/v100"
 )

-func (r *MasterReconciler) findMasterInstance(req controllerruntime.Request, ctx context.Context, log logr.Logger) (*v100.Master, bool, controllerruntime.Result, error) {
+func (r *MasterReconciler) findMasterCustomResourceInstance(ctx context.Context, log logr.Logger, req ctrl.Request) (*objectstorev100.Master, bool, ctrl.Result, error) {
 	// fetch the master instance
-	master := &v100.Master{}
+	master := &objectstorev100.Master{}
 	err := r.Get(ctx, req.NamespacedName, master)
 	if err != nil {
 		if errors.IsNotFound(err) {
@ -21,12 +26,172 @@ func (r *MasterReconciler) findMasterInstance(req controllerruntime.Request, ctx
 			// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
 			// Return and don't requeue
 			log.Info("Master resource not found. Ignoring since object must be deleted")
-			return nil, true, controllerruntime.Result{RequeueAfter: time.Second * 5}, nil
+			return nil, true, ctrl.Result{RequeueAfter: time.Second * 5}, nil
 		}
 		// Error reading the object - requeue the request.
 		log.Error(err, "Failed to get Master")
-		return nil, true, controllerruntime.Result{}, err
+		return nil, true, ctrl.Result{}, err
 	}
 	log.Info("Get master " + master.Name)
-	return master, false, controllerruntime.Result{}, nil
+	return master, false, ctrl.Result{}, nil
+}
+
+func (r *MasterReconciler) ensureMasterStatefulSet(ctx context.Context, log logr.Logger, master *objectstorev100.Master) (bool, ctrl.Result, error) {
+
+	// fetch the master instance
+	masterCluster := &appsv1.StatefulSet{}
+	err := r.Get(ctx, types.NamespacedName{Name: master.Name, Namespace: master.Namespace}, masterCluster)
+	if err != nil && errors.IsNotFound(err) {
+		// Define a new deployment
+		dep := r.createMasterStatefulSet(master)
+		log.Info("Creating a new master cluster statefulset", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
+		err = r.Create(ctx, dep)
+		if err != nil {
+			log.Error(err, "Failed to create new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
+			return true, ctrl.Result{}, err
+		}
+		// Deployment created successfully - return and requeue
+		return true, ctrl.Result{Requeue: true}, nil
+	} else if err != nil {
+		log.Error(err, "Failed to get Deployment")
+		return true, ctrl.Result{}, err
+	}
+	log.Info("Get master cluster " + masterCluster.Name)
+	return false, ctrl.Result{}, nil
+}
+
+// deploymentForMaster returns a memcached Deployment object
+func (r *MasterReconciler) createMasterStatefulSet(m *objectstorev100.Master) *appsv1.StatefulSet {
+	labels := lablesForMaster(m.Name)
+	replicas := int32(3)
+	enableServiceLinks := false
+
+	dep := &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      m.Name,
+			Namespace: m.Namespace,
+		},
+		Spec: appsv1.StatefulSetSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					Volumes:        nil,
+					InitContainers: nil,
+					Containers: []corev1.Container{{
+						Image: "chrislusf/seaweedfs:latest",
+						Name:  "master",
+						Env: []corev1.EnvVar{
+							{
+								Name: "POD_IP",
+								ValueFrom: &corev1.EnvVarSource{
+									FieldRef: &corev1.ObjectFieldSelector{
+										FieldPath: "status.podIP",
+									},
+								},
+							},
+							{
+								Name: "POD_NAME",
+								ValueFrom: &corev1.EnvVarSource{
+									FieldRef: &corev1.ObjectFieldSelector{
+										FieldPath: "metadata.name",
+									},
+								},
+							},
+							{
+								Name: "NAMESPACE",
+								ValueFrom: &corev1.EnvVarSource{
+									FieldRef: &corev1.ObjectFieldSelector{
+										FieldPath: "metadata.namespace",
+									},
+								},
+							},
+						},
+						Command: []string{"weed", "master", "-volumeSizeLimitMB=1000", "-ip=${POD_NAME}"},
+						Ports: []corev1.ContainerPort{
+							{
+								ContainerPort: 9333,
+								Name:          "swfs-master",
+							},
+							{
+								ContainerPort: 19333,
+							},
+						},
+					}},
+					EnableServiceLinks:            &enableServiceLinks,
+				},
+			},
+		},
+	}
+	// Set master instance as the owner and controller
+	// ctrl.SetControllerReference(m, dep, r.Scheme)
+	return dep
+}
+
+func (r *MasterReconciler) ensureMasterService(ctx context.Context, log logr.Logger, master *objectstorev100.Master) (bool, ctrl.Result, error) {
+
+	// fetch the master instance
+	masterService := &corev1.Service{}
+	err := r.Get(ctx, types.NamespacedName{Name: master.Name, Namespace: master.Namespace}, masterService)
+	if err != nil && errors.IsNotFound(err) {
+		// Define a new deployment
+		dep := r.createMasterService(master)
+		log.Info("Creating a new master cluster statefulset", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
+		err = r.Create(ctx, dep)
+		if err != nil {
+			log.Error(err, "Failed to create new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
+			return true, ctrl.Result{}, err
+		}
+		// Deployment created successfully - return and requeue
+		return true, ctrl.Result{Requeue: true}, nil
+	} else if err != nil {
+		log.Error(err, "Failed to get Deployment")
+		return true, ctrl.Result{}, err
+	}
+	log.Info("Get master cluster " + masterService.Name)
+	return false, ctrl.Result{}, nil
+}
+
+// deploymentForMaster returns a memcached Deployment object
+func (r *MasterReconciler) createMasterService(m *objectstorev100.Master) *corev1.Service {
+	labels := lablesForMaster(m.Name)
+
+	dep := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      m.Name,
+			Namespace: m.Namespace,
+		},
+		Spec: corev1.ServiceSpec{
+			ClusterIP: "None",
+			Ports: []corev1.ServicePort{
+				{
+					Name:     "swfs-master",
+					Protocol: corev1.Protocol("TCP"),
+					Port:     9333,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 9333,
+					},
+				},
+				{
+					Name:     "swfs-master-grpc",
+					Protocol: corev1.Protocol("TCP"),
+					Port:     19333,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 19333,
+					},
+				},
+			},
+			Selector: labels,
+		},
+	}
+	// Set master instance as the owner and controller
+	// ctrl.SetControllerReference(m, dep, r.Scheme)
+	return dep
 }