agola/internal/objectstorage/s3.go

// Copyright 2019 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.

package objectstorage

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"agola.io/agola/internal/errors"
	minio "github.com/minio/minio-go/v6"
)

type S3Storage struct {
	bucket      string
	minioClient *minio.Client
	// minio core client user for low level api
	minioCore *minio.Core
}

func NewS3(bucket, location, endpoint, accessKeyID, secretAccessKey string, secure bool) (*S3Storage, error) {
	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, secure)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	minioCore, err := minio.NewCore(endpoint, accessKeyID, secretAccessKey, secure)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	exists, err := minioClient.BucketExists(bucket)
	if err != nil {
		return nil, errors.Wrapf(err, "cannot check if bucket %q in location %q exits", bucket, location)
	}
	if !exists {
		if err := minioClient.MakeBucket(bucket, location); err != nil {
			return nil, errors.Wrapf(err, "cannot create bucket %q in location %q", bucket, location)
		}
	}

	return &S3Storage{
		bucket:      bucket,
		minioClient: minioClient,
		minioCore:   minioCore,
	}, nil
}

func (s *S3Storage) Stat(p string) (*ObjectInfo, error) {
	oi, err := s.minioClient.StatObject(s.bucket, p, minio.StatObjectOptions{})
	if err != nil {
		merr := minio.ToErrorResponse(err)
		if merr.StatusCode == http.StatusNotFound {
			return nil, NewErrNotExist(errors.Errorf("object %q doesn't exist", p))
		}
		return nil, errors.WithStack(merr)
	}

	return &ObjectInfo{Path: p, LastModified: oi.LastModified, Size: oi.Size}, nil
}

func (s *S3Storage) ReadObject(filepath string) (ReadSeekCloser, error) {
	if _, err := s.minioClient.StatObject(s.bucket, filepath, minio.StatObjectOptions{}); err != nil {
		merr := minio.ToErrorResponse(err)
		if merr.StatusCode == http.StatusNotFound {
			return nil, NewErrNotExist(errors.Errorf("object %q doesn't exist", filepath))
		}
		return nil, errors.WithStack(merr)
	}

	o, err := s.minioClient.GetObject(s.bucket, filepath, minio.GetObjectOptions{})

	return o, errors.WithStack(err)
}

func (s *S3Storage) WriteObject(filepath string, data io.Reader, size int64, persist bool) error {
	// if size is not specified, limit max object size to defaultMaxObjectSize so
	// minio client will not calculate a very big part size using tons of ram.
	// An alternative is to write the file locally so we can calculate the size and
	// then put it. See commented out code below.
	if size >= 0 {
		lr := io.LimitReader(data, size)
		_, err := s.minioClient.PutObject(s.bucket, filepath, lr, size, minio.PutObjectOptions{ContentType: "application/octet-stream"})
		return errors.WithStack(err)
	}

	// hack to know the real file size or minio will do this in memory with big memory usage since s3 doesn't support real streaming of unknown sizes
	// TODO(sgotti) wait for minio client to expose an api to provide the max object size so we can remove this
	tmpfile, err := ioutil.TempFile(os.TempDir(), "s3")
	if err != nil {
		return errors.WithStack(err)
	}
	defer tmpfile.Close()
	defer os.Remove(tmpfile.Name())
	size, err = io.Copy(tmpfile, data)
	if err != nil {
		return errors.WithStack(err)
	}
	if _, err := tmpfile.Seek(0, 0); err != nil {
		return errors.WithStack(err)
	}
	_, err = s.minioClient.PutObject(s.bucket, filepath, tmpfile, size, minio.PutObjectOptions{ContentType: "application/octet-stream"})
	return errors.WithStack(err)
}

func (s *S3Storage) DeleteObject(filepath string) error {
	return errors.WithStack(s.minioClient.RemoveObject(s.bucket, filepath))
}

func (s *S3Storage) List(prefix, startWith, delimiter string, doneCh <-chan struct{}) <-chan ObjectInfo {
	objectCh := make(chan ObjectInfo, 1)

	if len(delimiter) > 1 {
		objectCh <- ObjectInfo{
			Err: errors.Errorf("wrong delimiter %q", delimiter),
		}
		return objectCh
	}

	// remove leading slash
	prefix = strings.TrimPrefix(prefix, "/")
	startWith = strings.TrimPrefix(startWith, "/")

	// Initiate list objects goroutine here.
	go func(objectCh chan<- ObjectInfo) {
		defer close(objectCh)
		// Save continuationToken for next request.
		var continuationToken string
		for {
			// Get list of objects a maximum of 1000 per request.
			result, err := s.minioCore.ListObjectsV2(s.bucket, prefix, continuationToken, false, delimiter, 1000, startWith)
			if err != nil {
				objectCh <- ObjectInfo{
					Err: err,
				}
				return
			}

			// If contents are available loop through and send over channel.
			for _, object := range result.Contents {
				select {
				// Send object content.
				case objectCh <- ObjectInfo{Path: object.Key, LastModified: object.LastModified, Size: object.Size}:
				// If receives done from the caller, return here.
				case <-doneCh:
					return
				}
			}

			// If continuation token present, save it for next request.
			if result.NextContinuationToken != "" {
				continuationToken = result.NextContinuationToken
			}

			// Listing ends result is not truncated, return right here.
			if !result.IsTruncated {
				return
			}
		}
	}(objectCh)

	return objectCh
}