harbor/api/utils.go

/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package api

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/vmware/harbor/dao"
	"github.com/vmware/harbor/models"
	"github.com/vmware/harbor/service/cache"
	"github.com/vmware/harbor/utils"
	"github.com/vmware/harbor/utils/log"
	"github.com/vmware/harbor/utils/registry"
	registry_error "github.com/vmware/harbor/utils/registry/error"
)

func checkProjectPermission(userID int, projectID int64) bool {
	roles, err := listRoles(userID, projectID)
	if err != nil {
		log.Errorf("error occurred in getProjectPermission: %v", err)
		return false
	}
	return len(roles) > 0
}

func hasProjectAdminRole(userID int, projectID int64) bool {
	roles, err := listRoles(userID, projectID)
	if err != nil {
		log.Errorf("error occurred in getProjectPermission: %v", err)
		return false
	}

	for _, role := range roles {
		if role.RoleID == models.PROJECTADMIN {
			return true
		}
	}

	return false
}

//sysadmin has all privileges to all projects
func listRoles(userID int, projectID int64) ([]models.Role, error) {
	roles := make([]models.Role, 0, 1)
	isSysAdmin, err := dao.IsAdminRole(userID)
	if err != nil {
		log.Errorf("failed to determine whether the user %d is system admin: %v", userID, err)
		return roles, err
	}
	if isSysAdmin {
		role, err := dao.GetRoleByID(models.PROJECTADMIN)
		if err != nil {
			log.Errorf("failed to get role %d: %v", models.PROJECTADMIN, err)
			return roles, err
		}
		roles = append(roles, *role)
		return roles, nil
	}

	rs, err := dao.GetUserProjectRoles(userID, projectID)
	if err != nil {
		log.Errorf("failed to get user %d 's roles for project %d: %v", userID, projectID, err)
		return roles, err
	}
	roles = append(roles, rs...)
	return roles, nil
}

func checkUserExists(name string) int {
	u, err := dao.GetUser(models.User{Username: name})
	if err != nil {
		log.Errorf("Error occurred in GetUser, error: %v", err)
		return 0
	}
	if u != nil {
		return u.UserID
	}
	return 0
}

// TriggerReplication triggers the replication according to the policy
func TriggerReplication(policyID int64, repository string,
	tags []string, operation string) error {
	data := struct {
		PolicyID  int64    `json:"policy_id"`
		Repo      string   `json:"repository"`
		Operation string   `json:"operation"`
		TagList   []string `json:"tags"`
	}{
		PolicyID:  policyID,
		Repo:      repository,
		TagList:   tags,
		Operation: operation,
	}

	b, err := json.Marshal(&data)
	if err != nil {
		return err
	}

	url := buildReplicationURL()

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(b))
	if err != nil {
		return err
	}
	addAuthentication(req)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		return nil
	}

	b, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	return fmt.Errorf("%d %s", resp.StatusCode, string(b))
}

// GetPoliciesByRepository returns policies according the repository
func GetPoliciesByRepository(repository string) ([]*models.RepPolicy, error) {
	repository = strings.TrimSpace(repository)
	repository = strings.TrimRight(repository, "/")
	projectName := repository[:strings.LastIndex(repository, "/")]

	project, err := dao.GetProjectByName(projectName)
	if err != nil {
		return nil, err
	}

	policies, err := dao.GetRepPolicyByProject(project.ProjectID)
	if err != nil {
		return nil, err
	}

	return policies, nil
}

// TriggerReplicationByRepository triggers the replication according to the repository
func TriggerReplicationByRepository(repository string, tags []string, operation string) {
	policies, err := GetPoliciesByRepository(repository)
	if err != nil {
		log.Errorf("failed to get policies for repository %s: %v", repository, err)
		return
	}

	for _, policy := range policies {
		if policy.Enabled == 0 {
			continue
		}
		if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil {
			log.Errorf("failed to trigger replication of policy %d for %s: %v", policy.ID, repository, err)
		} else {
			log.Infof("replication of policy %d for %s triggered", policy.ID, repository)
		}
	}
}

func postReplicationAction(policyID int64, acton string) error {
	data := struct {
		PolicyID int64  `json:"policy_id"`
		Action   string `json:"action"`
	}{
		PolicyID: policyID,
		Action:   acton,
	}

	b, err := json.Marshal(&data)
	if err != nil {
		return err
	}

	url := buildReplicationActionURL()

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(b))
	if err != nil {
		return err
	}

	addAuthentication(req)

	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		return nil
	}

	b, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	return fmt.Errorf("%d %s", resp.StatusCode, string(b))
}

func addAuthentication(req *http.Request) {
	if req != nil {
		req.AddCookie(&http.Cookie{
			Name: models.UISecretCookie,
			// TODO read secret from config
			Value: os.Getenv("UI_SECRET"),
		})
	}
}

// SyncRegistry syncs the repositories of registry with database.
func SyncRegistry() error {

	log.Debugf("Start syncing repositories from registry to DB... ")

	reposInRegistry, err := catalog()
	if err != nil {
		log.Error(err)
		return err
	}

	var repoRecordsInDB []models.RepoRecord
	repoRecordsInDB, err = dao.GetAllRepositories()
	if err != nil {
		log.Errorf("error occurred while getting all registories. %v", err)
		return err
	}

	var reposInDB []string
	for _, repoRecordInDB := range repoRecordsInDB {
		reposInDB = append(reposInDB, repoRecordInDB.Name)
	}

	var reposToAdd []string
	var reposToDel []string
	reposToAdd, reposToDel = diffRepos(reposInRegistry, reposInDB)
	if len(reposToAdd) > 0 {
		log.Debugf("Start adding repositories into DB... ")
		for _, repoToAdd := range reposToAdd {
			project, _ := utils.ParseRepository(repoToAdd)
			user, err := dao.GetAccessLogCreator(repoToAdd)
			if err != nil {
				log.Errorf("Error happens when getting the repository owner from access log: %v", err)
			}
			if len(user) == 0 {
				user = "anonymous"
			}
			pullCount, err := dao.CountPull(repoToAdd)
			if err != nil {
				log.Errorf("Error happens when counting pull count from access log: %v", err)
			}
			repoRecord := models.RepoRecord{Name: repoToAdd, OwnerName: user, ProjectName: project, PullCount: pullCount}
			if err := dao.AddRepository(repoRecord); err != nil {
				log.Errorf("Error happens when adding the missing repository: %v", err)
			}
			log.Debugf("Add repository: %s success.", repoToAdd)
		}
	}

	if len(reposToDel) > 0 {
		log.Debugf("Start deleting repositories from DB... ")
		for _, repoToDel := range reposToDel {
			if err := dao.DeleteRepository(repoToDel); err != nil {
				log.Errorf("Error happens when deleting the repository: %v", err)
			}
			log.Debugf("Delete repository: %s success.", repoToDel)
		}
	}

	log.Debugf("Sync repositories from registry to DB is done.")
	return nil
}

func catalog() ([]string, error) {
	repositories := []string{}

	rc, err := initRegistryClient()
	if err != nil {
		return repositories, err
	}

	repos, err := rc.Catalog()
	if err != nil {
		return repositories, err
	}

	for _, repo := range repos {
		// TODO remove the workaround when the bug of registry is fixed
		// TODO read it from config
		endpoint := os.Getenv("REGISTRY_URL")
		client, err := cache.NewRepositoryClient(endpoint, true,
			"admin", repo, "repository", repo)
		if err != nil {
			return repositories, err
		}

		exist, err := repositoryExist(repo, client)
		if err != nil {
			return repositories, err
		}

		if !exist {
			continue
		}

		repositories = append(repositories, repo)
	}

	return repositories, nil
}

func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string) {
	var needsAdd []string
	var needsDel []string

	sort.Strings(reposInRegistry)
	sort.Strings(reposInDB)

	i, j := 0, 0
	repoInR, repoInD := "", ""
	for i < len(reposInRegistry) && j < len(reposInDB) {
		repoInR = reposInRegistry[i]
		repoInD = reposInDB[j]
		d := strings.Compare(repoInR, repoInD)
		if d < 0 {
			i++
			exist, err := projectExists(repoInR)
			if err != nil {
				log.Errorf("failed to check the existence of project %s: %v", repoInR, err)
				continue
			}

			if !exist {
				continue
			}

			needsAdd = append(needsAdd, repoInR)
		} else if d > 0 {
			needsDel = append(needsDel, repoInD)
			j++
		} else {
			i++
			j++
		}
	}

	for i < len(reposInRegistry) {
		repoInR = reposInRegistry[i]
		i++
		exist, err := projectExists(repoInR)
		if err != nil {
			log.Errorf("failed to check whether project of %s exists: %v", repoInR, err)
			continue
		}

		if !exist {
			continue
		}
		needsAdd = append(needsAdd, repoInR)
	}

	for j < len(reposInDB) {
		needsDel = append(needsDel, reposInDB[j])
		j++
	}

	return needsAdd, needsDel
}

func projectExists(repository string) (bool, error) {
	project, _ := utils.ParseRepository(repository)
	return dao.ProjectExists(project)
}

func initRegistryClient() (r *registry.Registry, err error) {
	endpoint := os.Getenv("REGISTRY_URL")

	addr := endpoint
	if strings.Contains(endpoint, "/") {
		addr = endpoint[strings.LastIndex(endpoint, "/")+1:]
	}

	ch := make(chan int, 1)
	go func() {
		var err error
		var c net.Conn
		for {
			c, err = net.DialTimeout("tcp", addr, 20*time.Second)
			if err == nil {
				c.Close()
				ch <- 1
			} else {
				log.Errorf("failed to connect to registry client, retry after 2 seconds :%v", err)
				time.Sleep(2 * time.Second)
			}
		}
	}()
	select {
	case <-ch:
	case <-time.After(60 * time.Second):
		panic("Failed to connect to registry client after 60 seconds")
	}

	registryClient, err := cache.NewRegistryClient(endpoint, true, "admin",
		"registry", "catalog", "*")
	if err != nil {
		return nil, err
	}
	return registryClient, nil
}

func buildReplicationURL() string {
	url := getJobServiceURL()
	return fmt.Sprintf("%s/api/jobs/replication", url)
}

func buildJobLogURL(jobID string) string {
	url := getJobServiceURL()
	return fmt.Sprintf("%s/api/jobs/replication/%s/log", url, jobID)
}

func buildReplicationActionURL() string {
	url := getJobServiceURL()
	return fmt.Sprintf("%s/api/jobs/replication/actions", url)
}

func getJobServiceURL() string {
	url := os.Getenv("JOB_SERVICE_URL")
	url = strings.TrimSpace(url)
	url = strings.TrimRight(url, "/")

	if len(url) == 0 {
		url = "http://jobservice"
	}

	return url
}

func getReposByProject(name string, keyword ...string) ([]string, error) {
	repositories := []string{}

	repos, err := dao.GetRepositoryByProjectName(name)
	if err != nil {
		return repositories, err
	}

	needMatchKeyword := len(keyword) > 0 && len(keyword[0]) != 0

	for _, repo := range repos {
		if needMatchKeyword &&
			!strings.Contains(repo.Name, keyword[0]) {
			continue
		}

		repositories = append(repositories, repo.Name)
	}

	return repositories, nil
}

func getAllRepos() ([]string, error) {
	return cache.GetRepoFromCache()
}

func repositoryExist(name string, client *registry.Repository) (bool, error) {
	tags, err := client.ListTag()
	if err != nil {
		if regErr, ok := err.(*registry_error.Error); ok && regErr.StatusCode == http.StatusNotFound {
			return false, nil
		}
		return false, err
	}
	return len(tags) != 0, nil
}